From 23c9150699aea5176ffb1d65f7e3cc8d593a89ac Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 1 Mar 2026 00:11:56 +0800 Subject: [PATCH] feat(qihoo360)!: add qihoo360 driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(drivers): free space underflow if used larger than total space (#1407) * fix(189pc/189tv): `request` panic when login failed (#1428) * fix(ilanzou): wrong total capacity (#1433) * feat(pikpak): support disk usage (#1426) * feat(pikpak): support disk usage * fix(alias): cannot list with details * refactor: rename `NewDiskUsageFromUsedAndTotal` * fix(disk-usage): get details of storages that is not initialized * feat(offline_download): add 123 open (#1427) * feat(http3|quic): add http3|quic support (#1466) * feat(http3|quic): add http3|quic support * revert(ai): fix ai error * fix(shutdown): shutdown was using close * feat(http3|quic): add config if needs h3 * feat(http3|quic): add Alt-Svc to expose h3 * chore(frontend): optimize user operations (#1449) * feat(drivers): add halalcloud_open driver (#1430) * 新增清真云Open驱动，支持最新的轻量SDK * Change Go version in go.mod Downgrade Go version from 1.24.2 to 1.23.4 Signed-off-by: zzzhr1990 * Apply suggestions from code review * Removed unnecessary comments * Downgraded the Go version to 1.23.4. * Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety. Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: zzzhr1990 * feat(halalcloud_open): support disk usage * Set useSingleUpload to true for upload safety Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety. Signed-off-by: zzzhr1990 * Update meta.go Change required for RefreshToken, If using a personal API approach, the RefreshToken is not required. Signed-off-by: zzzhr1990 * remove debug logs * bump halalcloud SDK version * fix unnecessary params * Update drivers/halalcloud_open/driver_init.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: zzzhr1990 * Fixed spelling errors; changed hardcoded retry parameters to constants. * remove pointer in get link function in utils.go --------- Signed-off-by: zzzhr1990 Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: KirCute <951206789@qq.com> * feat(http3|quic): Set Alt-Svc header only for HTTPS requests (#1469) * feat(local): move deleted files to corresponding locations (#1281) * fix(fs):fix retry task after restart (#1467) * fix retry task after restart * fix: initialize SrcStorage and DstStorage in tasks to prevent nil pointer dereference * feat: implement storage load signal mechanism for improved synchronization * fix: update StoragesLoaded logic * refactor: reorganize storage loading logic and improve synchronization handling --------- Co-authored-by: j2rong4cn * feat(cache): improve cache management (#1339) * feat(cache): improve cache management * feat(disk-usage): add cache * feat(disk-usage): add refresh * fix(disk-usage): cache with ttl * feat(cache): implement KeyedCache and TypedCache for improved caching mechanism * fix(copy): update object retrieval to use Get instead of GetUnwrap * refactor(cache): simplify DirectoryCache structure and improve object management * fix(cache): correct cache entry initialization and key deletion logic in TypedCache * refactor(driver): remove GetObjInfo interface and simplify Link function logic https://github.com/OpenListTeam/OpenList/pull/888/files#r2430925783 * fix(link): optimize link retrieval and caching logic * refactor(cache): consolidate cache management and improve directory cache handling * fix(cache): add cache control based on storage configuration in List function * . * refactor: replace fmt.Sprintf with strconv for integer conversions * refactor(cache): enhance cache entry management with Expirable interface * fix(cache): improve link reference acquisition logic to handle expiration * refactor: replace OnlyLinkMFile with NoLinkSF in driver configurations and logic * refactor(link): enhance link caching logic with dynamic type keys based on IP and User-Agent * feat(drivers): add LinkCacheType to driver configurations for enhanced caching * refactor(cache): streamline directory object management in cache operations * refactor(cache): remove unnecessary 'dirty' field from CacheEntry structure * refactor(cache): replace 'dirty' field with bitwise flags * refactor(io): 调高SyncClosers.AcquireReference的优先级 * refactor(link): 优化链接获取逻辑，增加重 * refactor(link): 添加RequireReference字段以增强链接管理 * refactor(link): 移除MFile字段，改用RangeReader * refactor: 移除不必要的NoLinkSF字段 * refactor(cache): 修改目录缓存的脏标志定义和更新逻辑 * feat(cache): add expiration gc --------- Co-authored-by: KirCute <951206789@qq.com> Co-authored-by: KirCute Co-authored-by: j2rong4cn * feat(onedrive): add ref support (#1435) * feat(onedrive): add ref support * fix(onedrive): remove redundant token assignment from reference --------- Co-authored-by: j2rong4cn * feat(drivers): add ProtonDrive driver (#1368) * feat(drivers): add ProtonDrive driver - Implement complete ProtonDrive storage driver with end-to-end encryption support - Add authentication via username/password with credential caching and reusable login - Support all core operations: List, Link, Put, Copy, Move, Remove, Rename, MakeDir - Include encrypted file operations with PGP key management and node passphrase handling - Add temporary HTTP server for secure file downloads with range request support - Support media streaming using temp server range requests - Implement progress tracking for uploads and downloads - Support directory operations with circular move detection - Add proper error handling and panic recovery for external library integration - Support buffered upload for specific sequential and encrypted, but optimized transmission. * Update drivers/proton_drive/util.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: D@' 3z K!7 <99719341+Da3zKi7@users.noreply.github.com> * chore * feat(drivers): enhance ProtonDrive temp server - Implement separate listen and public port configuration for complex network deployments - Add intelligent port detection with 8080 as preferred default, fallback to auto-assignment - Support Container/NAT/VM environments through configurable external host and port mapping - Add port availability validation with graceful fallback to listen port - Enable users to specify external domain/IP for client connections (e.g., 192.168.1.5) - Follow FTP server configuration patterns for network flexibility - Maintain localhost development simplicity while supporting production deployments * feat(proton_drive): refactor directory handling and improve link retrieval * fix(proton_drive): add NoLinkURL configuration option * fix(proton_drive): update file size retrieval and enforce TwoFACode requirement * feat(proton_drive): add expiration to link response * fix(proton_drive): handle empty RootFolderID in Init method * fix(proton_drive): update credential handling to use email and reusable login * fix(proton_drive): update credential handling to use reusableCredential variable * fix(proton_drive): update DirectRename to use GetLink for source object retrieval * fix(proton_drive): refactor uploadFile to return model.Obj and handle errors correctly * fix(proton_drive): refactor DirectMove to use getLink for source retrieval and simplify destination handling * fix(proton_drive): simplify Copy method by removing temporary file creation and directly using FileStream * refactor(proton_drive): remove unused temporary server and related code * chore * fix(proton_drive): fix driver - Handle fresh login if ProtonDrive rejects AccessToken or RefreshToken - Update stored credentials * fix(proton_drive): simplify reusable login handling in Init method * fix(proton_drive): fix driver - Update stored credentials, now is failing * feat(proton_drive): improve authentication handling and remove unused variables * fix(proton_drive): fix driver - Update stored credentials, now is failing * fix(proton_drive): improve authentication handling * refactor(proton_drive): move client initialization to initClient method * feat(proton_drive): move addrs and addrKRs * feat(proton_drive): optimize upload threads - Change ConcurrentBlockUploadCount to user configured upload threads number - Comment ConcurrentFileCryptoCount, default is runtime.GOMAXPROCS(0) --------- Signed-off-by: D@' 3z K!7 <99719341+Da3zKi7@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: KirCute <951206789@qq.com> Co-authored-by: j2rong4cn Co-authored-by: KirCute * feat(openlist): add PassIPToUpsteam to driver (#1498) * fix(share): remove share when user delete (#1493) * fix(aliyundrive_open): use safe disk usage calculation (#1510) * feat(command): add --config flag to set custom config path (#1479) * fix(alias): nil panic in `ResolveLinkCacheMode` (#1527) * fix(alias): Check the driver path during initialization * fix(alias): Don't check the driver path during initialization anymore. * feat(123): allow modification of the platform header (#1542) * feat(drivers/123): Allow modification of the platform field * feat(drivers/123): Set login platfrom as web * fix(drivers/123): update platform field help value * feat(strm): strm local file (#1127) * feat(strm): strm local file * feat: 代码优化 * feat: 访问被strm挂载路径时也更新 * fix: 路径最后带/判断缺失 * fix: 路径最后带/判断缺失 * refactor * refactor * fix: close seekable-stream in `generateStrm` * refactor: lazy create local file * 优化路径判断 --------- Co-authored-by: KirCute * feat(thunder): allow setting space (#1219) allows access to files on remote devices via Thunder's tunneling service. * feat(net): support proxy configuration via config file (#1359) * support proxy * debug * debug2 * del debug * add proxy configuration with env var fallback * comments to en * refactor(env): fallback env --------- Co-authored-by: jyxjjj <773933146@qq.com> * fix(google_driver): fix google link file display size (#1335) * fix file link display size * fix performance and field * cn to en notes --------- Co-authored-by: ShenLin <773933146@qq.com> * chore(strm): Built-in file types support modification (#1483) * fix(baidu_netdisk): support resuming uploads when an error occurs (#1279) support resuming uploads when an error occurs * feat(stream): enhance GetRangeReaderFromLink rate limiting (#1528) * feat(stream): enhance GetRangeReaderFromLink rate limiting * refactor(stream): update GetRangeReaderFromMFile to return *model.FileRangeReader * refactor(stream): simplify context error handling in RateLimitReader, RateLimitWriter, and RateLimitFile * refactor(net): replace custom LimitedReadCloser with readers.NewLimitedReadCloser * fix(model): update Link.ContentLength JSON tag for correct serialization * docs(model): add clarification to FileRangeReader usage comment * fix(backup-restore): add shares (#1500) * perf(stream): optimize CacheFullAndWriter for better memory management (#1584) * perf(stream): optimize CacheFullAndWriter for better memory management * fix(stream): ensure proper seek handling in CacheFullAndWriter for improved data integrity * fix(baidu_netdisk): improve upload experience (#1562) * fix(baidu_netdisk): improve upload experience * fix(typo): URL should be uppercase, apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: ShenLin <773933146@qq.com> * fix(typo): URL should be uppercase, apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: ShenLin <773933146@qq.com> * fix(baidu_netdisk): use "UploadAPI" as a fallback when using dynamic upload api * fix(baidu_netdisk): all uploads share the same upload url cache * fix(drivers/baidu_netdisk): defer uploadUrlMu unlock * update driver.go to main --------- Signed-off-by: ShenLin <773933146@qq.com> Signed-off-by: jenfonro <799170122@qq.com> Co-authored-by: ShenLin <773933146@qq.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: jenfonro <799170122@qq.com> * feat(github_release): support Source code (zip/tar.gz) (#1581) * support Github Release Source code (zip/tar.gz) * fix TarballUrl and ZipballUrl * fix show source code by allversion --------- Co-authored-by: nibazshab <44338441+nibazshab@users.noreply.github.com> * fix(189): disk-usage unmarshal failed when used capacity overflow (#1577) * refactor(stream): simplify code (#1590) * refactor(stream): simplify Close method and update SeekableStream to use RangeReader interface * refactor(stream): improve RangeRead comments for clarity * fix(strm): non-specified type generates strm (#1585) * fix(strm): non-specified type generates strm * fix(strm): only insert to strmTrie if SaveStrmToLocal is enabled * fix(strm): update suffix handling in convert2strmObjs function * fix(strm): refactor generateStrm to use range reader --------- Co-authored-by: j2rong4cn * feat(onedrive): support frontend direct upload (#1532) * OneDrive添加直连上传 * refactor * fix: duplicate root path join --------- Co-authored-by: KirCute <951206789@qq.com> * refactor(fs): implement immediate retry within task execution cycle (#1575) * fix(baidu_netdisk): Fix Baidu Netdisk resume uploads sticking to the same upload host (#1609) Fix Baidu Netdisk resume uploads sticking to the same upload host * Initial plan * Initial plan for qihoo360 driver implementation Co-authored-by: Wodlie <86234536+Wodlie@users.noreply.github.com> * Implement qihoo360 driver with basic file operations Co-authored-by: Wodlie <86234536+Wodlie@users.noreply.github.com> * Remove OpenList binary from repository Co-authored-by: Wodlie <86234536+Wodlie@users.noreply.github.com> * Fix security issue: remove sensitive data from debug logs Co-authored-by: Wodlie <86234536+Wodlie@users.noreply.github.com> * Clean up: remove binary and update gitignore Co-authored-by: Wodlie <86234536+Wodlie@users.noreply.github.com> * fix(terabox): wrong return code used (#1547) fix(terabox): rename, delete, copy operations sometimes failed Signed-off-by: yuyamionini <46483865+yuyamionini@users.noreply.github.com> * fix(archive/zip): user specific encoding for non-EFS zips (#1599) * fix(archive/zip): user specific encoding for non-EFS zips * fix(stream): simplify head cache initialization and improve reader retrieval logic * fix: support multipart zips (.z01) * chore(deps): update github.com/KirCute/zip to v1.0.1 --------- Co-authored-by: j2rong4cn Co-authored-by: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> * feat(doubao): Add rate limiting (#1618) * chore(issue templates): require logs (#1626) * Fix infinite loop in authentication flow Move auth check inside method validation to prevent infinite recursion when getAuth() calls request(). Now request() only checks authentication when method is not empty, breaking the circular dependency. Co-authored-by: Wodlie <86234536+Wodlie@users.noreply.github.com> * feat(s3): support frontend direct upload (#1631) * feat(s3): support frontend direct upload * feat(s3): support custom direct upload host * fix: apply suggestions of Copilot * fix(link): correct link cache mode bitwise comparison (#1635) * fix(link): correct link cache mode bitwise comparison Signed-off-by: MadDogOwner * refactor(link): use explicit flag equality for link cache mode bitmask checks Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner * feat(upload): add optional system file filtering for uploads (#1634) * fix(driver/123): initialize Platform field (#1644) * fix(driver/123): initialization the Platform field Signed-off-by: MoYan <1561515308@qq.com> * Fix formatting of Platform field in Pan123 Signed-off-by: MoYan <1561515308@qq.com> --------- Signed-off-by: MoYan <1561515308@qq.com> * fix(openlist_driver): ensure UA is correctly propagated (#1679) * feat(fs): support manually trigger objs update hook (#1620) * feat(fs): support manually trigger objs update hook * fix: support driver internal copy & move case * fix * fix: apply suggestions of Copilot * fix(misskey): folderId format validation and root directory handling (#1647) fix(misskey): Fix folderId format validation and root directory handling * fix(strm): fix the name and type issue (#1630) * fix(strm): fix the name and type issue * fix(strm): update version * feat(strm): custom path prefixes (#1697) fix(strm): custom path prefixes Signed-off-by: ShenLin <773933146@qq.com> Co-authored-by: ShenLin <773933146@qq.com> * feat(baidu_netdisk): Add shard upload timeout setting (#1682) add timeout * fix(local): assign non-CoW copy requests to the task module (#1669) * fix(local): assign non-CoW copy requests to the task module * fix build * fix cross device * feat(fs): Add skipExisting option to move and copy, merge option to copy (#1556) * fix(fs): Add skipExisting option to move and copy. * feat(fs): Add merge option to copy. * feat(fs): Code smell. * feat(fs): Code smell. * feat(archive): support non-overwrite decompress (#1701) * feat(search): Add task queue for Meilisearch to prevent race conditions (#1423) * Add task queue for Meilisearch to prevent race conditions - Implement TaskQueueManager for async index operations - Queue update tasks and process them in batches every 30 seconds - Check pending task status before executing new operations - Optimize batch indexing and deletion logic - Fix type assertion bug in buildSearchDocumentFromResults * fix(search): re-enqueue skipped tasks to prevent task loss When tasks are skipped due to pending dependencies, they are now re-enqueued if not already in queue. This prevents task loss while avoiding overwriting newer snapshots for the same parent. * fix(copilot-comment): Invoke Stop() & err of SliceConvert --------- Co-authored-by: ImoutoHeaven Co-authored-by: jyxjjj <773933146@qq.com> * fix(search): fix duplicated variable init (#1703) * chore(archive): fix struct literal uses unkeyed fields (#1704) * chore(pkg/aria2): simplify context cancellation handling in RPC calls (#1705) * fix(pkg/aria2): use pointer receivers for Call methods (#1706) * feat(strm): keep local download file (#1707) * fix(onedrive_shareurl): Reduce temporary file errors (#1686) * fix onedrive_shareurl * . * chore(strm): avoid generating empty folders (#1720) chore(strm): empty folders are not generated locally * refactor: improve upload handling (#1455) * fix(quark): refactor upPart to use http.NewRequest * fix(quark): improved upload handling * fix(quark_open): improved upload handling * fix: add retry context to multiple upload functions * fix: optimize hash calculation in multipart upload to avoid blocking * fix: update error handling in lifecycle functions for better clarity * fix: update upload progress calculation to improve accuracy * fix: simplify error handling in lifecycle functions for improved readability * fix: remove unnecessary mutex for part uploads to simplify code * fix(stream): simplify file handling in NewStreamSectionReader and improve error messages * fix(terabox): optimize chunk count calculation in Put method * perf(chaoxing): 表单上传文件0拷贝 * fix(cnb_releases): improve file upload progress tracking * fix(baidu_netdisk): improve upload handling * fix(upload): optimize buffer initialization for file uploads * fix(baidu_netdisk): add retry condition to skip ErrUploadIDExpired in upload loop * fix(drivers/crypt): remove hard dependency on RemotePath (#1713) * fix(mediafire): enable automatic session token acquisition and fix gzip parsing (#1661) * fix(mediafire): enable automatic session token acquisition and fix gzip parsing - Fix Init() method to allow automatic session token retrieval from cookie - Change SessionToken from required to optional in configuration - Add proper gzip decompression support for API responses - Improve error handling for session token acquisition failures - Update help text to clarify authentication requirements Resolves initialization failure and JSON parsing errors when session token can be automatically obtained from browser cookie. * fix(mediafire): ensure driver files end with newline * chore: gofmt drivers/mediafire/*.go * refactor(op): remove automatic Path assignment (#1734) * refactor: 移除 ObjResp 中的 Id 和 Path 字段 * 移除op.List的自动设置Path Path和Id只在驱动内使用，不应由op.List设置Path * cnb_releases：将 Addition 结构体中的 RootPath 字段为 RootID 当List方法加载二级目录时，若使用的是Id，应对使用driver.RootID * doubao_share: 添加潜在bug注释 * 添加 GetRootPath 方法到多个驱动 * refactor!(userAgent): merge most userAgent into base (#1722) refactor!(userAgent): merge all userAgent into base 1. change var to const 2. remove duplicated ua definetion after original Resty R 3. upgrade Chrome and OS versions * feat(fs): Support customizing the cache time for a specific path (#1533) * feat(fs): Support customizing the cache time for a specific path * feat(fs): Get the cache rule for driver information. * feat(fs): Support globbing. * feat(fs): Add log. --------- Signed-off-by: ShenLin <773933146@qq.com> Co-authored-by: ShenLin <773933146@qq.com> * feat(driver): add AList v3 (#1721) * feat(driver/openlist): compatible with AList v3 * Revert "feat(driver/openlist): compatible with AList v3" This reverts commit 90f3f80186af485aeb7251c3dfdc8b8fc98c870d. * feat(driver): add AList v3 * Revert "feat(patch): add migration from Alist V3 driver to OpenList (#919)" Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner Co-authored-by: MadDogOwner * feat(utils): add support for ignoring '@eaDir' system files (#1779) * fix(task): tasks keep being cancelled (#1745) * fix_cancel * update(go.mod): update tache version * tache v0.2.2 --------- Co-authored-by: j2rong4cn * feat(model): add object mask support and enhance cache/task handling (#1743) * fix(ilanzou): parse vip size (#1792) * fix(halalcloud_open): halal-cloud upload issues (#1800) fix halal-cloud upload issues * refactor(bootstrap): move booting logic to bootstrap package (#1773) * refactor(bootstrap): move booting to bootstrap package * chore(log): reduce level of some callings of `utils.Log.Fatal` * fix(s3): no shutdown after SIGTERM received * fix: add handle hook * fix(openlist): disable status check for openlist driver (#1757) * fix(openlist): disable status check to avoid network stability issues * fix(alist_v3): disable status check to avoid network stability issues Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner * feat(ldap): support webdav, ftp and sftp login (#1746) * feat(ldap): support webdav, ftp and sftp login * fix: apply suggestions of Copilot * feat(ldap) support ftp, sftp and webdav auto-register * feat(drivers/139): user authentication and file batch operations (#1534) * feat(139): Enhance 139 driver with password login and root path handling - Added support for password-based login in the 139 driver. - Introduced RootPath field to store the root directory path. - Updated Init method to handle family and group types more effectively. - Implemented new methods for handling file operations in family and group contexts. - Enhanced error handling and logging for better debugging. - Added new request and response structures for batch operations and document modifications. - Improved encryption and decryption methods for secure communication. * Update drivers/139/util.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: UcnacDx2 <127503808+UcnacDx2@users.noreply.github.com> * Update drivers/139/util.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: UcnacDx2 <127503808+UcnacDx2@users.noreply.github.com> * Update drivers/139/util.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: UcnacDx2 <127503808+UcnacDx2@users.noreply.github.com> * Update drivers/139/util.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: UcnacDx2 <127503808+UcnacDx2@users.noreply.github.com> * Update drivers/139/util.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: UcnacDx2 <127503808+UcnacDx2@users.noreply.github.com> --------- Signed-off-by: UcnacDx2 <127503808+UcnacDx2@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix(139): disk-usage unmarshal failed when used capacity overflow (#1718) Co-authored-by: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> * feat(driver): add personal / business wps drive support (#1802) * feat(driver): add wps drive support * feat(driver): add wps drive support * fix(wps): update personal mode string to English Signed-off-by: MadDogOwner * fix(wps): remove trailing slash from drive origin URL Signed-off-by: MadDogOwner * fix(wps): correct order of options in mode selection Signed-off-by: MadDogOwner * fix(wps): enable local sort and upload overwrite Signed-off-by: MadDogOwner * fix(wps): resolve put bugs, fix file op problems and optimize list logic - Fix uploading bugs. Support all uploading methods based on 8825.85d3c864.js - Fix issues in delete/copy/move while opearting big folders. - Use cache to optimize performance of list, especially in a deep path. --------- Signed-off-by: MadDogOwner Co-authored-by: MadDogOwner * feat(cloudreve_v4): add ks3 support (#1828) Signed-off-by: MadDogOwner * fix(strm): support generate strm with sign (#1832) * feat(115_open): support custom pagesize (#1822) * docs: add VPS.Town as sponsor to all README (#1842) Co-authored-by: XZB-1248 * fix(s3): incorrect copy key with plus sign (#1820) * docs: update README for zh-CN (#1844) * fix(weiyun): update sdk and support getDetails (#1845) * feat(strm): add save local mode (#1814) * feat(strm): add KeepSameNameOnly logic * chore(strm): skip update strm file when keepLocalDownloadFile * feat(strm): add save local mode * docs(readme): add demo site (#1850) Last Sync: 2025-12-22 12:39 * refactor(bootstrap): fix OpenList-Mobile compile failed (#1857) * feat(alias): support load balance (#1767) * feat(alias): support load balance * feat(alias): support storage match for load balance * feat(patch): add alias addition upgrade patch * fix bugs * fix(op/balance): optimize compatibility * chore: change default read conflict policy * feat(alias): refactor Alias initialization and enhance path handling * feat(alias): enhance object masking and add support for operation restrictions * feat(alias): enhance object masking * feat(fs): add permission checks * improve parsing * update object masks * feat(fs): enhance virtual file handling * feat(storage): enhance virtual file retrieval and path handling * refactor(alias): rename path handling functions for clarity and consistency * fix(alias): update path handling in Other method to use balanced path * fix bug * feat(alias): add file size validation * feat(alias): add hash consistency check * 移除哈希合并， * fix(alias): wrong behavior for all_strict/deterministic_or_all * Revert "fix(alias): wrong behavior for all_strict/deterministic_or_all" This reverts commit f001f2dcd76d5711d2b1f44d04edf646f32c75a7. * fix(alias): wrong behavior for all_strict/deterministic_or_all * feat(alias): support part-based read load balance * fix(alias): list panic when leak conflict path * fix(alias): remove Other load balance * fix(alias): 修复 Link 方法中 resultLink 的返回类型和内容复制问题 * fix(alias): 更好的下载并发？ * chore(alias): all tips * fix(alias): moving paths mismatch --------- Co-authored-by: j2rong4cn Co-authored-by: ShenLin <773933146@qq.com> * fix(s3): use current time as default modified time (#1860) * fix(driver): fix file copy failure to 123pan due to incorrect etag (#1874) * ci(lang): sync only new fields (#1881) * fix(123_open): infinite recursive call (#1854) fix(123_open): token refresh logic Fix token handling logic to avoid deadlock. Token method took reference of Alist's implementation. * feat(qihoo360): fix bugs made by copliot * fix(doubao_share): use new download info (#1890) fix(doubao_share): update file URL retrieval to use new download info structure Signed-off-by: MadDogOwner * feat(qihoo360): implement file upload functionality and enhance request handling * fix(qihoo360): update user-agent * feat(qihoo360): add User-Agent header for Sync.getVerifiedDownLoadUrl request * feat(drivers/mega): support getting disk usage (#1903) * feat(drivers/mega): support getting disk usage * fix * feat(qihoo360): implement GetDetails and user detail retrieval functions * feat(drivers/mega): support permanently deleting files (#1913) * feat(123pan): add offline download (#1911) * feat(123网盘): 添加123网盘离线下载功能 - 新增123网盘离线下载实现 - 添加相关API接口和常量配置 - 在路由和工具集中集成123网盘支持 * refactor(offline_download): 重构123网盘离线下载状态处理和类型定义 - 将离线下载相关类型定义从util.go移至types.go - 更新状态获取api * 移除了备选方案（/offline_download/task/status） * feat(qihoo360): enhance GetDetails function and update UserDetailResp structure * chore(strm): remove redundant param (#1929) * feat(drivers): support getting disk usage of some drivers (#1905) * feat(drivers): support getting disk usage of some drivers * feat(drivers/degoo): implement GetDetails * fix(fs/storage-details): fill used space rather than free space * fix mega * fix bsize type * fix(cloudreve): Manually set path to avoid empty path (#1933) * fix(drivers/alist_v3): failed to unmarshal MeResp (#1931) * feat(drivers/webdav): add support for 302 redirects (#1952) * Remove the `OnlyProxy` restriction and obtain the redirected link to support 302 * Add `driver.Config` `PreferProxy` to recommend user to enable the proxy by default --------- Signed-off-by: MadDogOwner * fix(fs): panic when failed to get storage details (#1964) * chore(.github): update issue templates [no ci] (#1976) chore(.github): update issue templates * fix(drivers/ftp): failed to get objs (#1948) fix(drivers/ftp): failed get objs * feat(drivers/cloudreve_v4): implement Getter interface (#1937) feat(cloudreve_v4): implement Getter interface Signed-off-by: MadDogOwner * fix(drivers/github): failed to get objs (#1915) fix(drivers/github): failed get objs * fix(drivers/strm): delete extra local directories in sync mode (#1980) * fix(ci): change unchecked regex (#1993) fix(ci/issue): change unchecked regex Signed-off-by: KirCute <951206789@qq.com> * feat(drivers): add autoindex driver (#1978) * feat(drivers): add autoindex driver * fix * add NoUpload Signed-off-by: MadDogOwner * add TestParseSize Signed-off-by: MadDogOwner * go mod tidy Signed-off-by: MadDogOwner * use base.RestyClient Signed-off-by: MadDogOwner * fix: support evaluate size and modified time * fix apache * perf * feat: support ignore size and modified time * rename driver --------- Signed-off-by: MadDogOwner Co-authored-by: MadDogOwner * fix(fs): handle non-existent destination directory in file transfer (#1898) * fix(FileTransferTask): skip copying if destination directory does not exist * pass only object not found error --------- Co-authored-by: cyk Co-authored-by: KirCute <951206789@qq.com> * feat(drivers/cloudreve): implement GetDetails (#1960) * fix(driver/quark_uc): Fix display of non-compliant filenames (#2000) * fix(webdav/move): fix source file still exist after moving file by webdav (#1979) * perf(baidu_netdisk): reduce the number of file listing requests (#2016) * 本优化减少了百度网盘驱动下文件列表的请求次数，能加快文件浏览速度。此前只要文件夹下有至少一个文件，都会至少发出2次列表请求。 Signed-off-by: hcrgm * fix(drivers/seafile): object not found when RootFolderPath != "/" (#2010) * fix(driver/seafile): object not found when RootFolderPath != "/" * refactor(seafile): restructure Seafile driver for improved library handling and error management * add IsDir method to LibraryInfo type * improve initialization * add repoID to RepoItemResp and update List method to set repoID --------- Co-authored-by: Khoray Co-authored-by: j2rong4cn * fix(drivers/alias): default sort & substitute link (#1917) * fix(drivers/alias): default sort & substitute link * fix * fix * fix(drivers/cloudreve_v4): add IsFolder attribute to Getter response (#2035) * fix(drivers/cloudreve_v4): add IsFolder attribute to Getter response Signed-off-by: MadDogOwner * refactor(drivers/cloudreve_v4): implement File.fileToObject method Signed-off-by: MadDogOwner * fix(drivers/cloudreve_v4): implement 404 not found for getter Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner * fix(drivers/quark): apply html escaping in quark (#2046) * fix(drivers/quark): apply html escaping in quark * fix(api/remove): add validation for empty items in delete file list (#1617) * fix(FsRemove): add validation for empty items in delete file list If Req.Names contains an empty string item, the whole directory will be removed. As a result we need add a simple guard to prevent such cases. Signed-off-by: huyuantao * fix(FsRemove): enhance validation to prevent unintended directory deletion 1. Use `utils.FixAndCleanPath` to correctly identify and block invalid names. 2. Change error handling from `return` to `continue`. Signed-off-by: huyuantao --------- Signed-off-by: huyuantao Co-authored-by: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> * Merge commit from fork Co-authored-by: KirCute <951206789@qq.com> * Merge commit from fork Co-authored-by: KirCute <951206789@qq.com> * fix(drivers/teldrive): enhance file listing and upload functionality with pagination and random chunk naming (#2034) * fix(drivers/teldrive): enhance file listing and upload functionality with pagination and random chunk naming * fix(drivers/teldrive): optimize file listing by removing unnecessary mutex and restructuring data handling * Update drivers/teldrive/meta.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Chaloemchai --------- Signed-off-by: Chaloemchai Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix(115_share): adjust 115 share driver for official API update (#2068) * fix(115_share): add user agent support and update driver dependency * fix(115): fix download error * feat: add thumbnail support for 115 driver and 115 share - Add Thumb() method to FileObj in 115 driver to return thumbnail URL - Add ThumbURL field to FileObj struct in 115 share utility - Update 115driver dependency from v1.2.2 to v1.2.3 to support thumbnail functionality - Implement Thumb() method for 115 share FileObj to return thumbnail URL * chore(ci): add breaking change guideline to PR title check (#2087) * feat(ci): add PR title validation for breaking changes Updated regex to allow '!' for breaking changes in PR titles. Signed-off-by: MadDogOwner * chore(pr): Update PR template Add bilingual instructions for PR title formatting. Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner * feat(drivers/123open): support sha1 reuse api (#2089) * feat(drivers/123open): support sha1 reuse api * fix(drivers/123open): fix typos * fix(deps): update go4.org digest to a507140 (#2095) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix(‎drivers/quark_uc_tv) : Update error code judgment (#2080) * fix error code * add ErrorInfo check * chore(typo): fix typo in UpdateFileReq field name (#2133) Fix typo in UpdateFileReq field name ref: https://github.com/OpenListTeam/115-sdk-go/pull/3 Signed-off-by: LXY <767763591@qq.com> * fix(driver/wps): fetch all files via multiple API invocations (#2139) fix(driver/wps): wps list all files in one request * feat(security): add SECURITY.md (#2147) [skip ci] Add SECURITY.md Signed-off-by: MadDogOwner * chore(handles/auth): improve error response (#2148) * chore(handles/auth): improve error response Signed-off-by: MadDogOwner * Apply suggestion from @xrgzs Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner * feat(drivers/thunder*): implement GetDetails (#2113) Signed-off-by: MadDogOwner * Fix critical auth bug: treat access_token_expire as absolute timestamp The API returns access_token_expire as an absolute Unix timestamp, not a duration. The old code was adding it to current time, causing tokens to "expire" in year 2082. This meant tokens appeared valid even after actual expiration (1 hour), causing all API calls to fail after the program ran for some time. Also added retry limit protection to prevent infinite recursion in error handling. Co-authored-by: Wodlie <86234536+Wodlie@users.noreply.github.com> --------- Signed-off-by: zzzhr1990 Signed-off-by: D@' 3z K!7 <99719341+Da3zKi7@users.noreply.github.com> Signed-off-by: ShenLin <773933146@qq.com> Signed-off-by: jenfonro <799170122@qq.com> Signed-off-by: yuyamionini <46483865+yuyamionini@users.noreply.github.com> Signed-off-by: MadDogOwner Signed-off-by: MoYan <1561515308@qq.com> Signed-off-by: UcnacDx2 <127503808+UcnacDx2@users.noreply.github.com> Signed-off-by: Wodlie <86234536+Wodlie@users.noreply.github.com> Signed-off-by: KirCute <951206789@qq.com> Signed-off-by: hcrgm Signed-off-by: huyuantao Signed-off-by: Chaloemchai Signed-off-by: LXY <767763591@qq.com> Co-authored-by: NewbieOrange Co-authored-by: KirCute <951206789@qq.com> Co-authored-by: ShenLin <773933146@qq.com> Co-authored-by: zzzhr1990 Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Tursom K. Ulefits <33368546+tursom@users.noreply.github.com> Co-authored-by: jenfonro <799170122@qq.com> Co-authored-by: j2rong4cn Co-authored-by: ILoveScratch Co-authored-by: KirCute Co-authored-by: jerrita Co-authored-by: D@' 3z K!7 <99719341+Da3zKi7@users.noreply.github.com> Co-authored-by: MadDogOwner Co-authored-by: walloo <51254954+varg1714@users.noreply.github.com> Co-authored-by: MoYan <1561515308@qq.com> Co-authored-by: Seven <53081179+Seven66677731@users.noreply.github.com> Co-authored-by: ex-hentai <96158157+ex-hentai@users.noreply.github.com> Co-authored-by: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Co-authored-by: hcrgm Co-authored-by: nibazshab <44338441+nibazshab@users.noreply.github.com> Co-authored-by: ASLant <77436463+Y-ASLant@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Wodlie <86234536+Wodlie@users.noreply.github.com> Co-authored-by: yuyamionini <46483865+yuyamionini@users.noreply.github.com> Co-authored-by: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> Co-authored-by: VXTLS <187420201+vxtls@users.noreply.github.com> Co-authored-by: ImoutoHeaven Co-authored-by: ImoutoHeaven Co-authored-by: UcnacDx2 <127503808+UcnacDx2@users.noreply.github.com> Co-authored-by: wongz Co-authored-by: Mako (XSpy) <127767602+x-spy@users.noreply.github.com> Co-authored-by: hshpy Co-authored-by: XZB-1248 <28593573+XZB-1248@users.noreply.github.com> Co-authored-by: XZB-1248 Co-authored-by: TwoOnefour Co-authored-by: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Co-authored-by: Seven <53081179+sevxn007@users.noreply.github.com> Co-authored-by: 绎泽 Co-authored-by: 我怎么就不是一只猫呢? <26274059+dezhishen@users.noreply.github.com> Co-authored-by: Tron Co-authored-by: Edward <73746306+WangEdward@users.noreply.github.com> Co-authored-by: mcxiedidi <113833419+mcxiedidi@users.noreply.github.com> Co-authored-by: Ljcbaby <46277145+ljcbaby@users.noreply.github.com> Co-authored-by: cyk Co-authored-by: Cp0204 Co-authored-by: mkitsdts <136291922+mkitsdts@users.noreply.github.com> Co-authored-by: Rui Huang <1251433731@qq.com> Co-authored-by: Khoray Co-authored-by: Hu Yuantao <130338111+datao2001@users.noreply.github.com> Co-authored-by: Chaloemchai Co-authored-by: Shelton Zhu <498220739@qq.com> Co-authored-by: gdm257 Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: LXY <767763591@qq.com> --- .github/ISSUE_TEMPLATE/00-bug_report_zh.yml | 18 +- .github/ISSUE_TEMPLATE/01-bug_report_en.yml | 18 +- .github/PULL_REQUEST_TEMPLATE.md | 2 + .github/workflows/issue_pr_comment.yml | 6 +- .gitignore | 3 +- README.md | 10 +- README_cn.md | 10 +- README_ja.md | 8 +- README_nl.md | 8 +- SECURITY.md | 89 ++ cmd/admin.go | 13 +- cmd/cancel2FA.go | 5 +- cmd/common.go | 12 +- cmd/crypt.go | 6 +- cmd/flags/config.go | 1 + cmd/lang.go | 23 +- cmd/root.go | 3 +- cmd/server.go | 213 +--- cmd/storage.go | 13 +- drivers/115/driver.go | 7 +- drivers/115/meta.go | 7 +- drivers/115/types.go | 4 + drivers/115/util.go | 62 +- drivers/115_open/driver.go | 35 +- drivers/115_open/meta.go | 6 +- drivers/115_open/upload.go | 4 +- drivers/115_open/util.go | 14 +- drivers/115_share/driver.go | 21 +- drivers/115_share/utils.go | 11 +- drivers/123/driver.go | 9 +- drivers/123/meta.go | 5 +- drivers/123/types.go | 51 +- drivers/123/upload.go | 23 +- drivers/123/util.go | 125 +- drivers/123_link/driver.go | 4 + drivers/123_open/driver.go | 49 +- drivers/123_open/meta.go | 3 +- drivers/123_open/token.go | 115 ++ drivers/123_open/types.go | 37 +- drivers/123_open/upload.go | 56 +- drivers/123_open/util.go | 107 +- drivers/123_share/meta.go | 1 + drivers/139/driver.go | 153 ++- drivers/139/meta.go | 3 + drivers/139/types.go | 59 + drivers/139/util.go | 710 ++++++++++- drivers/189/driver.go | 2 +- drivers/189/types.go | 14 +- drivers/189/util.go | 2 +- drivers/189_tv/driver.go | 8 +- drivers/189_tv/types.go | 14 +- drivers/189_tv/utils.go | 3 + drivers/189pc/driver.go | 8 +- drivers/189pc/types.go | 14 +- drivers/189pc/utils.go | 45 +- drivers/alias/driver.go | 677 ++++++----- drivers/alias/meta.go | 27 +- drivers/alias/types.go | 77 ++ drivers/alias/util.go | 479 ++++++-- drivers/alist_v3/driver.go | 379 ++++++ drivers/alist_v3/meta.go | 32 + drivers/alist_v3/types.go | 182 +++ drivers/alist_v3/util.go | 65 + drivers/aliyundrive/driver.go | 6 +- drivers/aliyundrive_open/driver.go | 7 +- drivers/aliyundrive_open/upload.go | 4 +- drivers/aliyundrive_open/util.go | 1 - drivers/all.go | 7 +- drivers/autoindex/driver.go | 169 +++ drivers/autoindex/meta.go | 29 + drivers/autoindex/types.go | 13 + drivers/autoindex/util.go | 116 ++ drivers/autoindex/util_test.go | 49 + drivers/azure_blob/driver.go | 3 +- drivers/azure_blob/meta.go | 10 +- drivers/baidu_netdisk/driver.go | 269 +++-- drivers/baidu_netdisk/meta.go | 14 + drivers/baidu_netdisk/types.go | 46 +- drivers/baidu_netdisk/util.go | 82 +- drivers/baidu_photo/driver.go | 3 +- drivers/baidu_photo/meta.go | 5 +- drivers/base/client.go | 8 +- drivers/chaoxing/driver.go | 26 +- drivers/chaoxing/types.go | 2 +- drivers/chaoxing/util.go | 3 +- drivers/chunk/driver.go | 35 +- drivers/cloudreve/driver.go | 12 + drivers/cloudreve/types.go | 7 + drivers/cloudreve/util.go | 9 +- drivers/cloudreve_v4/driver.go | 27 +- drivers/cloudreve_v4/types.go | 16 +- drivers/cloudreve_v4/util.go | 29 +- drivers/cnb_releases/driver.go | 66 +- drivers/cnb_releases/meta.go | 7 +- drivers/crypt/driver.go | 287 ++--- drivers/crypt/meta.go | 6 +- drivers/crypt/util.go | 25 +- drivers/degoo/driver.go | 21 + drivers/degoo/types.go | 23 + drivers/degoo/util.go | 16 + drivers/doubao/driver.go | 41 + drivers/doubao/meta.go | 13 +- drivers/doubao/util.go | 56 +- drivers/doubao_share/driver.go | 10 +- drivers/doubao_share/types.go | 12 +- drivers/doubao_share/util.go | 2 +- drivers/dropbox/util.go | 30 +- drivers/febbox/meta.go | 7 +- drivers/ftp/driver.go | 8 +- drivers/ftp/meta.go | 10 +- drivers/github/types.go | 3 + drivers/github_releases/driver.go | 12 +- drivers/github_releases/meta.go | 3 +- drivers/github_releases/types.go | 65 +- drivers/google_drive/driver.go | 8 +- drivers/google_drive/util.go | 83 +- drivers/halalcloud_open/common.go | 111 ++ drivers/halalcloud_open/driver.go | 29 + drivers/halalcloud_open/driver_curd_impl.go | 127 ++ drivers/halalcloud_open/driver_get_link.go | 108 ++ drivers/halalcloud_open/driver_init.go | 50 + drivers/halalcloud_open/driver_interface.go | 48 + drivers/halalcloud_open/halalcloud_upload.go | 258 ++++ drivers/halalcloud_open/meta.go | 40 + drivers/halalcloud_open/obj_file.go | 60 + drivers/halalcloud_open/utils.go | 185 +++ drivers/ilanzou/driver.go | 12 +- drivers/ilanzou/util.go | 5 +- drivers/lanzou/driver.go | 2 +- drivers/lanzou/meta.go | 2 +- drivers/lenovonas_share/driver.go | 13 +- drivers/local/copy_namedpipes.go | 16 + drivers/local/copy_namedpipes_x.go | 9 + drivers/local/driver.go | 58 +- drivers/local/meta.go | 12 +- drivers/local/util.go | 81 +- drivers/local/util_unix.go | 12 +- drivers/local/util_windows.go | 8 +- drivers/mediafire/driver.go | 22 +- drivers/mediafire/meta.go | 19 +- drivers/mediafire/types.go | 1 + drivers/mediafire/util.go | 40 +- drivers/mega/driver.go | 15 +- drivers/mega/meta.go | 1 + drivers/misskey/util.go | 34 +- drivers/mopan/driver.go | 15 +- drivers/netease_music/driver.go | 11 +- drivers/onedrive/driver.go | 34 +- drivers/onedrive/meta.go | 27 +- drivers/onedrive/types.go | 6 +- drivers/onedrive/util.go | 38 +- drivers/onedrive_app/driver.go | 20 +- drivers/onedrive_app/meta.go | 17 +- drivers/onedrive_app/types.go | 6 +- drivers/onedrive_app/util.go | 34 +- drivers/onedrive_sharelink/driver.go | 141 ++- drivers/openlist/driver.go | 35 +- drivers/openlist/meta.go | 3 +- drivers/openlist/types.go | 1 + drivers/openlist_share/driver.go | 1 + drivers/pikpak/driver.go | 29 +- drivers/pikpak/types.go | 14 +- drivers/proton_drive/driver.go | 286 +++++ drivers/proton_drive/meta.go | 56 + drivers/proton_drive/types.go | 38 + drivers/proton_drive/util.go | 670 +++++++++++ drivers/qihoo360/driver.go | 678 +++++++++++ drivers/qihoo360/meta.go | 24 + drivers/qihoo360/types.go | 167 +++ drivers/qihoo360/util.go | 377 ++++++ drivers/quark_open/driver.go | 56 +- drivers/quark_open/util.go | 4 +- drivers/quark_uc/driver.go | 63 +- drivers/quark_uc/types.go | 8 +- drivers/quark_uc/util.go | 45 +- drivers/quark_uc_tv/util.go | 13 +- drivers/s3/driver.go | 47 +- drivers/s3/meta.go | 2 + drivers/s3/util.go | 43 +- drivers/seafile/driver.go | 209 ++-- drivers/seafile/types.go | 51 +- drivers/seafile/util.go | 76 +- drivers/sftp/driver.go | 17 +- drivers/sftp/meta.go | 12 +- drivers/sftp/types.go | 4 +- drivers/smb/driver.go | 36 +- drivers/smb/meta.go | 12 +- drivers/strm/driver.go | 81 +- drivers/strm/hook.go | 237 ++++ drivers/strm/meta.go | 33 +- drivers/strm/types.go | 36 - drivers/strm/util.go | 34 +- drivers/teldrive/driver.go | 54 +- drivers/teldrive/meta.go | 1 + drivers/teldrive/upload.go | 68 +- drivers/template/meta.go | 1 - drivers/terabox/driver.go | 7 +- drivers/terabox/util.go | 2 +- drivers/thunder/driver.go | 65 +- drivers/thunder/meta.go | 5 +- drivers/thunder/types.go | 18 + drivers/thunder/util.go | 1 + drivers/thunder_browser/driver.go | 29 +- drivers/thunder_browser/types.go | 7 + drivers/thunderx/driver.go | 26 + drivers/thunderx/types.go | 7 + drivers/url_tree/driver.go | 4 + drivers/url_tree/meta.go | 5 +- drivers/uss/driver.go | 1 + drivers/virtual/driver.go | 12 +- drivers/virtual/meta.go | 10 +- drivers/webdav/driver.go | 19 + drivers/webdav/meta.go | 2 +- drivers/weiyun/driver.go | 20 +- drivers/wopan/driver.go | 19 +- drivers/wps/driver.go | 91 ++ drivers/wps/meta.go | 26 + drivers/wps/types.go | 107 ++ drivers/wps/util.go | 1072 +++++++++++++++++ drivers/yandex_disk/driver.go | 4 +- drivers/yandex_disk/types.go | 2 +- drivers/yandex_disk/util.go | 1 - go.mod | 74 +- go.sum | 121 +- internal/archive/rardecode/rardecode.go | 3 +- internal/archive/sevenzip/sevenzip.go | 3 +- internal/archive/tool/base.go | 3 +- internal/archive/zip/utils.go | 166 +-- internal/archive/zip/zip.go | 46 +- internal/bootstrap/config.go | 32 +- internal/bootstrap/data/setting.go | 8 + internal/bootstrap/patch.go | 1 - internal/bootstrap/patch/all.go | 14 +- .../bootstrap/patch/v3_24_0/hash_password.go | 5 +- .../bootstrap/patch/v3_32_0/update_authn.go | 5 +- internal/bootstrap/patch/v3_all/rename.go | 33 - internal/bootstrap/patch/v4_1_8/alias.go | 81 ++ internal/bootstrap/patch/v4_1_9/skip_tls.go | 32 + internal/bootstrap/patch/v4_1_9/webdav.go | 30 + internal/bootstrap/run.go | 404 +++++++ internal/bootstrap/storage.go | 2 +- internal/cache/keyed_cache.go | 101 ++ internal/cache/type.go | 18 + internal/cache/typed_cache.go | 113 ++ internal/cache/utils.go | 24 + internal/conf/config.go | 5 +- internal/conf/const.go | 40 +- internal/conf/var.go | 44 +- internal/db/sharing.go | 30 +- internal/db/tasks.go | 2 + internal/driver/config.go | 34 +- internal/driver/driver.go | 28 +- internal/driver/utils.go | 2 +- internal/errs/errors.go | 13 +- internal/errs/object.go | 8 +- internal/errs/unwrap.go | 11 +- internal/fs/archive.go | 49 +- internal/fs/copy_move.go | 104 +- internal/fs/fs.go | 36 +- internal/fs/get.go | 9 +- internal/fs/list.go | 2 +- internal/fs/other.go | 13 +- internal/fs/put.go | 24 +- internal/model/args.go | 7 +- internal/model/direct_upload.go | 8 + internal/model/file.go | 3 + internal/model/obj.go | 119 +- internal/model/object.go | 5 + internal/model/sharing.go | 2 +- internal/model/storage.go | 63 +- internal/model/user.go | 101 +- internal/net/request.go | 3 +- internal/net/serve.go | 14 +- internal/net/util.go | 58 +- internal/offline_download/123/client.go | 142 +++ internal/offline_download/123_open/client.go | 119 ++ internal/offline_download/all.go | 2 + internal/offline_download/tool/add.go | 23 +- internal/offline_download/tool/download.go | 11 +- internal/offline_download/tool/transfer.go | 47 +- internal/op/archive.go | 292 +++-- internal/op/cache.go | 228 ++++ internal/op/driver.go | 44 +- internal/op/fs.go | 968 ++++++++------- internal/op/hook.go | 7 +- internal/op/recursive_list.go | 125 ++ internal/op/setting.go | 39 +- internal/op/sharing.go | 4 + internal/op/storage.go | 140 ++- internal/op/user.go | 18 +- internal/search/build.go | 47 +- internal/search/meilisearch/init.go | 5 + internal/search/meilisearch/search.go | 128 +- internal/search/meilisearch/task_queue.go | 265 ++++ internal/search/meilisearch/utils.go | 17 +- internal/setting/setting.go | 8 + internal/stream/limit.go | 45 +- internal/stream/stream.go | 257 ++-- internal/stream/stream_test.go | 49 +- internal/stream/util.go | 153 ++- internal/task/base.go | 16 +- internal/task_group/group.go | 7 +- internal/task_group/transfer.go | 61 +- pkg/aria2/rpc/call.go | 25 +- pkg/errgroup/errgroup.go | 7 +- pkg/gowebdav/errors.go | 4 +- pkg/singleflight/singleflight.go | 5 +- pkg/utils/file.go | 17 + pkg/utils/file_test.go | 43 + pkg/utils/io.go | 65 +- pkg/utils/path.go | 6 +- server/common/ldap.go | 107 ++ server/common/proxy.go | 12 +- server/ftp.go | 28 +- server/ftp/afero.go | 4 +- server/ftp/fsup.go | 12 + server/handles/archive.go | 37 +- server/handles/auth.go | 19 +- server/handles/direct_upload.go | 54 + server/handles/down.go | 3 +- server/handles/fsmanage.go | 115 +- server/handles/fsread.go | 62 +- server/handles/fsup.go | 24 +- server/handles/ldap_login.go | 116 +- server/handles/offline_download.go | 96 +- server/handles/scan.go | 47 + server/handles/sharing.go | 70 +- server/handles/storage.go | 45 +- server/middlewares/check.go | 9 +- server/middlewares/search.go | 2 +- server/router.go | 9 + server/s3/backend.go | 12 +- server/sftp.go | 23 +- server/sftp/hostkey.go | 12 +- server/utils.go | 14 + server/webdav.go | 24 +- server/webdav/webdav.go | 7 +- 337 files changed, 15587 insertions(+), 3873 deletions(-) create mode 100644 SECURITY.md create mode 100644 drivers/123_open/token.go create mode 100644 drivers/alist_v3/driver.go create mode 100644 drivers/alist_v3/meta.go create mode 100644 drivers/alist_v3/types.go create mode 100644 drivers/alist_v3/util.go create mode 100644 drivers/autoindex/driver.go create mode 100644 drivers/autoindex/meta.go create mode 100644 drivers/autoindex/types.go create mode 100644 drivers/autoindex/util.go create mode 100644 drivers/autoindex/util_test.go create mode 100644 drivers/halalcloud_open/common.go create mode 100644 drivers/halalcloud_open/driver.go create mode 100644 drivers/halalcloud_open/driver_curd_impl.go create mode 100644 drivers/halalcloud_open/driver_get_link.go create mode 100644 drivers/halalcloud_open/driver_init.go create mode 100644 drivers/halalcloud_open/driver_interface.go create mode 100644 drivers/halalcloud_open/halalcloud_upload.go create mode 100644 drivers/halalcloud_open/meta.go create mode 100644 drivers/halalcloud_open/obj_file.go create mode 100644 drivers/halalcloud_open/utils.go create mode 100644 drivers/local/copy_namedpipes.go create mode 100644 drivers/local/copy_namedpipes_x.go create mode 100644 drivers/proton_drive/driver.go create mode 100644 drivers/proton_drive/meta.go create mode 100644 drivers/proton_drive/types.go create mode 100644 drivers/proton_drive/util.go create mode 100644 drivers/qihoo360/driver.go create mode 100644 drivers/qihoo360/meta.go create mode 100644 drivers/qihoo360/types.go create mode 100644 drivers/qihoo360/util.go create mode 100644 drivers/strm/hook.go delete mode 100644 drivers/strm/types.go create mode 100644 drivers/wps/driver.go create mode 100644 drivers/wps/meta.go create mode 100644 drivers/wps/types.go create mode 100644 drivers/wps/util.go delete mode 100644 internal/bootstrap/patch/v3_all/rename.go create mode 100644 internal/bootstrap/patch/v4_1_8/alias.go create mode 100644 internal/bootstrap/patch/v4_1_9/skip_tls.go create mode 100644 internal/bootstrap/patch/v4_1_9/webdav.go create mode 100644 internal/bootstrap/run.go create mode 100644 internal/cache/keyed_cache.go create mode 100644 internal/cache/type.go create mode 100644 internal/cache/typed_cache.go create mode 100644 internal/cache/utils.go create mode 100644 internal/model/direct_upload.go create mode 100644 internal/offline_download/123/client.go create mode 100644 internal/offline_download/123_open/client.go create mode 100644 internal/op/cache.go create mode 100644 internal/op/recursive_list.go create mode 100644 internal/search/meilisearch/task_queue.go create mode 100644 pkg/utils/file_test.go create mode 100644 server/common/ldap.go create mode 100644 server/handles/direct_upload.go create mode 100644 server/handles/scan.go create mode 100644 server/utils.go diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index 06752f537..7dadfc3f4 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -13,7 +13,7 @@ body: attributes: label: 请确认以下事项 description: | - 您必须勾选以下内容，否则您的问题可能会被直接关闭。 + 您必须阅读并检查以下内容，否则您的问题一定会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | @@ -34,6 +34,8 @@ body: 我认为此问题必须由`OpenList`处理，而非第三方。 - label: | 我已确认这个问题在最新版本中没有被修复。 + - label: | + 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue - type: input id: version @@ -60,19 +62,21 @@ body: validations: required: true - type: textarea - id: config + id: logs attributes: - label: 配置文件内容（必填） + label: 日志（必填） description: | - 请提供您的`OpenList`应用的配置文件，并截图相关存储配置。(可隐藏隐私字段) + 请复制粘贴错误日志，或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug) validations: required: true - type: textarea - id: logs + id: config attributes: - label: 日志（可选） + label: 配置文件内容（必填） description: | - 请复制粘贴错误日志，或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug) + 请提供您的`OpenList`应用的配置文件，并截图相关存储配置。(可隐藏隐私字段) + validations: + required: true - type: textarea id: reproduction attributes: diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index 5d263c3c5..5d5d91d00 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -13,7 +13,7 @@ body: attributes: label: Please confirm the following description: | - You must check all the following, otherwise your issue may be closed directly. + You must read and check all the following, otherwise your issue will definitely be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | @@ -34,6 +34,8 @@ body: I believe this issue must be handled by `OpenList` and not by a third party. - label: | I confirm this issue is not fixed in the latest version. + - label: | + I have not read these checkboxes and therefore I just ticked them all, Please close this issue - type: input id: version @@ -60,19 +62,21 @@ body: validations: required: true - type: textarea - id: config + id: logs attributes: - label: Configuration File Content (required) + label: Logs (required) description: | - Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields) + Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs) validations: required: true - type: textarea - id: logs + id: config attributes: - label: Logs (optional) + label: Configuration File Content (required) description: | - Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs) + Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields) + validations: + required: true - type: textarea id: reproduction attributes: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e74e59631..f1687eabf 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,11 +2,13 @@ Provide a general summary of your changes in the Title above. The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`. If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body. + For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`. --> ## Description / 描述 diff --git a/.github/workflows/issue_pr_comment.yml b/.github/workflows/issue_pr_comment.yml index 44f6e0aff..f15acd424 100644 --- a/.github/workflows/issue_pr_comment.yml +++ b/.github/workflows/issue_pr_comment.yml @@ -20,7 +20,7 @@ jobs: with: script: | const issueBody = context.payload.issue.body || ""; - const unchecked = /- \[ \] /.test(issueBody); + const unchecked = /- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); let comment = "感谢您联系OpenList。我们会尽快回复您。\n"; comment += "Thanks for contacting OpenList. We will reply to you as soon as possible.\n\n"; if (unchecked) { @@ -47,12 +47,14 @@ jobs: with: script: | const title = context.payload.pull_request.title || ""; - const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\): /i.test(title); + const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\)!?: /i.test(title); if (!ok) { let comment = "⚠️ PR 标题需以 `feat(): `, `docs(): `, `fix(): `, `style(): `, `refactor(): `, `chore(): ` 其中之一开头，例如：`feat(component): 新增功能`。\n"; comment += "⚠️ The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`.\n\n"; comment += "如果跨多个组件，请使用主要组件作为前缀，并在标题中枚举、描述中说明。\n"; comment += "If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body.\n\n"; + comment += "如果是破坏性变更，请在类型后添加 `!`，例如 `feat(component)!: 破坏性变更`。\n"; + comment += "For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`.\n\n"; await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, diff --git a/.gitignore b/.gitignore index 1d71f0d60..d56e0e161 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ output/ /public/dist/* /!public/dist/README.md -.VSCodeCounter \ No newline at end of file +.VSCodeCounter/OpenList +/OpenList diff --git a/README.md b/README.md index 405377f18..c4f29462e 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,9 @@ Thank you for your support and understanding of the OpenList project. - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([China](https://www.teambition.com), [International](https://us.teambition.com)) - - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [MediaFire](https://www.mediafire.com) + - [x] [Mediatrack](https://www.mediatrack.cn) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) @@ -121,12 +122,17 @@ Thank you for your support and understanding of the OpenList project. ## Demo -N/A (to be rebuilt) +- 🌎 [Global Demo](https://demo.oplist.org) +- 🇨🇳 [CN Demo](https://demo.oplist.org.cn) ## Discussion Please refer to [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) for raising general questions, ***Issues* is for bug reports and feature requests only.** +## Sponsor + +[![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") + ## License The `OpenList` is open-source software licensed under the [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) license. diff --git a/README_cn.md b/README_cn.md index 4d58f31f6..adac3d0b9 100644 --- a/README_cn.md +++ b/README_cn.md @@ -64,8 +64,9 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 - [x] [又拍云对象存储](https://www.upyun.com/products/file-storage) - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([中国](https://www.teambition.com), [国际](https://us.teambition.com)) - - [x] [分秒帧](https://www.mediatrack.cn) - [x] [MediaFire](https://www.mediafire.com) + - [x] [分秒帧](https://www.mediatrack.cn) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [和彩云](https://yun.139.com)（个人、家庭、群组） - [x] [YandexDisk](https://disk.yandex.com) - [x] [百度网盘](http://pan.baidu.com) @@ -121,12 +122,17 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 ## 演示 -N/A（待重建） +- 🇨🇳 [国内演示站](https://demo.oplist.org.cn) +- 🌎 [海外演示站](https://demo.oplist.org) ## 讨论 如有一般性问题请前往 [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) 讨论区，***Issues* 仅用于错误报告和功能请求。** +## 赞助者 + +[![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") + ## 许可证 `OpenList` 是基于 [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) 许可证的开源软件。 diff --git a/README_ja.md b/README_ja.md index 3eac9a691..52c1a01c0 100644 --- a/README_ja.md +++ b/README_ja.md @@ -65,6 +65,7 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([中国](https://www.teambition.com), [国際](https://us.teambition.com)) - [x] [Mediatrack](https://www.mediatrack.cn) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com)（個人、家族、グループ） - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) @@ -121,12 +122,17 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい ## デモ -N/A（再構築中） +- 🌎 [グローバルデモ](https://demo.oplist.org) +- 🇨🇳 [CNデモ](https://demo.oplist.org.cn) ## ディスカッション 一般的な質問は [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) をご利用ください。***Issues* はバグ報告と機能リクエスト専用です。** +## スポンサー + +[![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") + ## ライセンス 「OpenList」は [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) ライセンスの下で公開されているオープンソースソフトウェアです。 diff --git a/README_nl.md b/README_nl.md index 469e86028..8b9e62ec7 100644 --- a/README_nl.md +++ b/README_nl.md @@ -66,6 +66,7 @@ Dank u voor uw ondersteuning en begrip - [x] Teambition([China](https://www.teambition.com), [Internationaal](https://us.teambition.com)) - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com) (Persoonlijk, Familie, Groep) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) @@ -121,12 +122,17 @@ Dank u voor uw ondersteuning en begrip ## Demo -N.v.t. (wordt opnieuw opgebouwd) +- 🌎 [Global Demo](https://demo.oplist.org) +- 🇨🇳 [CN Demo](https://demo.oplist.org.cn) ## Discussie Stel algemene vragen in [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions), ***Issues* zijn alleen voor bugmeldingen en feature requests.** +## Sponsoren + +[![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") + ## Licentie `OpenList` is open-source software onder de [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) licentie. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..ff558d64e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,89 @@ +# Security Policy + +## Supported Versions + +Only the latest stable release receives security patches. We strongly recommend always keeping OpenList up to date. + +| Version | Supported | +| -------------------- | ------------------ | +| Latest stable (v4.x) | :white_check_mark: | +| Older versions | :x: | + +## Reporting a Vulnerability + +**Please do NOT report security vulnerabilities through public GitHub Issues.** + +If you discover a security vulnerability in OpenList, please report it responsibly by using one of the following channels: + +- **GitHub Private Security Advisory** (preferred): [Submit here](https://github.com/OpenListTeam/OpenList/security/advisories/new) +- **Telegram**: Contact a maintainer privately via [@OpenListTeam](https://t.me/OpenListTeam) + +When reporting, please include as much of the following as possible: + +- A description of the vulnerability and its potential impact +- The affected version(s) +- Step-by-step instructions to reproduce the issue +- Any proof-of-concept code or screenshots (if applicable) +- Suggested mitigation or fix (optional but appreciated) + +## Security Best Practices for Users + +To keep your OpenList instance secure: + +- Always update to the latest release. +- Use a strong, unique admin password and change it after first login. +- Enable HTTPS (TLS) for your deployment — do **not** expose OpenList over plain HTTP on the public internet. +- Limit exposed ports using a reverse proxy (e.g., Nginx, Caddy). +- Set up access controls and avoid enabling guest access unless necessary. +- Regularly review mounted storage permissions and revoke unused API tokens. +- When using Docker, avoid running the container as root if possible. + +## Acknowledgments + +We sincerely thank all security researchers and community members who responsibly disclose vulnerabilities and help make OpenList safer for everyone. + +--- + +# 安全政策 + +## 支持的版本 + +我们仅对最新稳定版本提供安全补丁。强烈建议始终保持 OpenList 为最新版本。 + +| 版本 | 是否支持 | +| ------------------ | ------------------ | +| 最新稳定版（v4.x） | :white_check_mark: | +| 旧版本 | :x: | + +## 报告漏洞 + +**请勿通过公开的 GitHub Issues 报告安全漏洞。** + +如果您在 OpenList 中发现安全漏洞，请通过以下渠道之一负责任地进行报告： + +- **GitHub 私密安全公告**（推荐）：[点击提交](https://github.com/OpenListTeam/OpenList/security/advisories/new) +- **Telegram**：通过 [@OpenListTeam](https://t.me/OpenListTeam) 私信联系维护者 + +报告时，请尽量提供以下信息： + +- 漏洞描述及其潜在影响 +- 受影响的版本 +- 复现问题的详细步骤 +- 概念验证代码或截图（如有） +- 建议的缓解措施或修复方案（可选，但非常欢迎） + +## 用户安全最佳实践 + +为保障您的 OpenList 实例安全： + +- 始终更新至最新版本。 +- 使用强且唯一的管理员密码，并在首次登录后立即修改。 +- 为您的部署启用 HTTPS（TLS）—— **请勿**在公网上以明文 HTTP 方式暴露 OpenList。 +- 使用反向代理（如 Nginx、Caddy）限制对外暴露的端口。 +- 配置访问控制，非必要情况下不要开启访客访问。 +- 定期检查已挂载存储的权限，并撤销未使用的 API 令牌。 +- 使用 Docker 部署时，尽可能避免以 root 用户运行容器。 + +## 致谢 + +我们衷心感谢所有负责任地披露漏洞、帮助 OpenList 变得更加安全的安全研究人员和社区成员。 diff --git a/cmd/admin.go b/cmd/admin.go index 447c49703..5e09959cf 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -20,8 +21,8 @@ var AdminCmd = &cobra.Command{ Aliases: []string{"password"}, Short: "Show admin user's info and some operations about admin user's password", Run: func(cmd *cobra.Command, args []string) { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("failed get admin user: %+v", err) @@ -61,8 +62,8 @@ var ShowTokenCmd = &cobra.Command{ Use: "token", Short: "Show admin token", Run: func(cmd *cobra.Command, args []string) { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() token := setting.GetStr(conf.Token) utils.Log.Infof("show admin token from CLI") fmt.Println("Admin token:", token) @@ -70,8 +71,8 @@ var ShowTokenCmd = &cobra.Command{ } func setAdminPassword(pwd string) { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("failed get admin user: %+v", err) diff --git a/cmd/cancel2FA.go b/cmd/cancel2FA.go index 809d32e37..3ddd7f838 100644 --- a/cmd/cancel2FA.go +++ b/cmd/cancel2FA.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/spf13/cobra" @@ -16,8 +17,8 @@ var Cancel2FACmd = &cobra.Command{ Use: "cancel2fa", Short: "Delete 2FA of admin user", Run: func(cmd *cobra.Command, args []string) { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("failed to get admin user: %+v", err) diff --git a/cmd/common.go b/cmd/common.go index 6835a0f5c..e334fc1fa 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -6,24 +6,16 @@ import ( "strconv" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" - "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/data" - "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) func Init() { - bootstrap.InitConfig() - bootstrap.Log() - bootstrap.InitDB() - data.InitData() - bootstrap.InitStreamLimit() - bootstrap.InitIndex() - bootstrap.InitUpgradePatch() + bootstrap.Init() } func Release() { - db.Close() + bootstrap.Release() } var pid = -1 diff --git a/cmd/crypt.go b/cmd/crypt.go index 10b3352cd..12f51f5cd 100644 --- a/cmd/crypt.go +++ b/cmd/crypt.go @@ -1,19 +1,17 @@ package cmd import ( - log "github.com/sirupsen/logrus" - "io" "os" "path" "path/filepath" "strings" - "github.com/spf13/cobra" - rcCrypt "github.com/rclone/rclone/backend/crypt" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/obscure" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) // encryption and decryption command format for Crypt driver diff --git a/cmd/flags/config.go b/cmd/flags/config.go index f74e2cb4b..0409731e8 100644 --- a/cmd/flags/config.go +++ b/cmd/flags/config.go @@ -2,6 +2,7 @@ package flags var ( DataDir string + ConfigPath string Debug bool NoPrefix bool Dev bool diff --git a/cmd/lang.go b/cmd/lang.go index a03b3c561..c18877cd7 100644 --- a/cmd/lang.go +++ b/cmd/lang.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "os" - "reflect" "strings" _ "github.com/OpenListTeam/OpenList/v4/drivers" @@ -69,15 +68,33 @@ func writeFile(name string, data interface{}) { log.Errorf("failed to unmarshal json: %+v", err) return } - if reflect.DeepEqual(oldData, newData) { + if mergeJson(newData, oldData) { log.Infof("%s.json no changed, skip", name) } else { log.Infof("%s.json changed, update file", name) //log.Infof("old: %+v\nnew:%+v", oldData, data) - utils.WriteJsonToFile(fmt.Sprintf("lang/%s.json", name), newData, true) + utils.WriteJsonToFile(fmt.Sprintf("lang/%s.json", name), oldData, true) } } +func mergeJson(source, target map[string]interface{}) bool { + equal := true + for k, v := range source { + tgtV, tgtOk := target[k] + if !tgtOk { + equal = false + target[k] = v + } else { + srcMap, srcIsMap := v.(map[string]interface{}) + tgtMap, tgtIsMap := tgtV.(map[string]interface{}) + if srcIsMap && tgtIsMap { + equal = mergeJson(srcMap, tgtMap) && equal + } + } + } + return equal +} + func generateDriversJson() { drivers := make(Drivers) drivers["drivers"] = make(KV[interface{}]) diff --git a/cmd/root.go b/cmd/root.go index 86e45a356..42121aac7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,7 +27,8 @@ func Execute() { } func init() { - RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data folder") + RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data directory (relative paths are resolved against the current working directory)") + RootCmd.PersistentFlags().StringVar(&flags.ConfigPath, "config", "", "path to config.json (relative to current working directory; defaults to [data directory]/config.json, where [data directory] is set by --data)") RootCmd.PersistentFlags().BoolVar(&flags.Debug, "debug", false, "start with debug mode") RootCmd.PersistentFlags().BoolVar(&flags.NoPrefix, "no-prefix", false, "disable env prefix") RootCmd.PersistentFlags().BoolVar(&flags.Dev, "dev", false, "start with dev mode") diff --git a/cmd/server.go b/cmd/server.go index 3758009f6..26441f916 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -1,32 +1,13 @@ package cmd import ( - "context" - "errors" - "fmt" - "net" - "net/http" "os" "os/signal" - "strconv" - "sync" "syscall" "time" - "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" - "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/fs" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server" - "github.com/OpenListTeam/OpenList/v4/server/middlewares" - "github.com/OpenListTeam/sftpd-openlist" - ftpserver "github.com/fclairamb/ftpserverlib" - "github.com/gin-gonic/gin" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" ) // ServerCmd represents the server command @@ -36,142 +17,9 @@ var ServerCmd = &cobra.Command{ Long: `Start the server at the specified address the address is defined in config file`, Run: func(cmd *cobra.Command, args []string) { - Init() - if conf.Conf.DelayedStart != 0 { - utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart) - time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second) - } - bootstrap.InitOfflineDownloadTools() - bootstrap.LoadStorages() - bootstrap.InitTaskManager() - if !flags.Debug && !flags.Dev { - gin.SetMode(gin.ReleaseMode) - } - r := gin.New() - - // gin log - if conf.Conf.Log.Filter.Enable { - r.Use(middlewares.FilteredLogger()) - } else { - r.Use(gin.LoggerWithWriter(log.StandardLogger().Out)) - } - r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out)) - - server.Init(r) - var httpHandler http.Handler = r - if conf.Conf.Scheme.EnableH2c { - httpHandler = h2c.NewHandler(r, &http2.Server{}) - } - var httpSrv, httpsSrv, unixSrv *http.Server - if conf.Conf.Scheme.HttpPort != -1 { - httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort) - fmt.Printf("start HTTP server @ %s\n", httpBase) - utils.Log.Infof("start HTTP server @ %s", httpBase) - httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler} - go func() { - err := httpSrv.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - utils.Log.Fatalf("failed to start http: %s", err.Error()) - } - }() - } - if conf.Conf.Scheme.HttpsPort != -1 { - httpsBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpsPort) - fmt.Printf("start HTTPS server @ %s\n", httpsBase) - utils.Log.Infof("start HTTPS server @ %s", httpsBase) - httpsSrv = &http.Server{Addr: httpsBase, Handler: r} - go func() { - err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) - if err != nil && !errors.Is(err, http.ErrServerClosed) { - utils.Log.Fatalf("failed to start https: %s", err.Error()) - } - }() - } - if conf.Conf.Scheme.UnixFile != "" { - fmt.Printf("start unix server @ %s\n", conf.Conf.Scheme.UnixFile) - utils.Log.Infof("start unix server @ %s", conf.Conf.Scheme.UnixFile) - unixSrv = &http.Server{Handler: httpHandler} - go func() { - listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile) - if err != nil { - utils.Log.Fatalf("failed to listen unix: %+v", err) - } - // set socket file permission - mode, err := strconv.ParseUint(conf.Conf.Scheme.UnixFilePerm, 8, 32) - if err != nil { - utils.Log.Errorf("failed to parse socket file permission: %+v", err) - } else { - err = os.Chmod(conf.Conf.Scheme.UnixFile, os.FileMode(mode)) - if err != nil { - utils.Log.Errorf("failed to chmod socket file: %+v", err) - } - } - err = unixSrv.Serve(listener) - if err != nil && !errors.Is(err, http.ErrServerClosed) { - utils.Log.Fatalf("failed to start unix: %s", err.Error()) - } - }() - } - if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { - s3r := gin.New() - s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) - server.InitS3(s3r) - s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) - fmt.Printf("start S3 server @ %s\n", s3Base) - utils.Log.Infof("start S3 server @ %s", s3Base) - go func() { - var err error - if conf.Conf.S3.SSL { - httpsSrv = &http.Server{Addr: s3Base, Handler: s3r} - err = httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) - } - if !conf.Conf.S3.SSL { - httpSrv = &http.Server{Addr: s3Base, Handler: s3r} - err = httpSrv.ListenAndServe() - } - if err != nil && !errors.Is(err, http.ErrServerClosed) { - utils.Log.Fatalf("failed to start s3 server: %s", err.Error()) - } - }() - } - var ftpDriver *server.FtpMainDriver - var ftpServer *ftpserver.FtpServer - if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable { - var err error - ftpDriver, err = server.NewMainDriver() - if err != nil { - utils.Log.Fatalf("failed to start ftp driver: %s", err.Error()) - } else { - fmt.Printf("start ftp server on %s\n", conf.Conf.FTP.Listen) - utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen) - go func() { - ftpServer = ftpserver.NewFtpServer(ftpDriver) - err = ftpServer.ListenAndServe() - if err != nil { - utils.Log.Fatalf("problem ftp server listening: %s", err.Error()) - } - }() - } - } - var sftpDriver *server.SftpDriver - var sftpServer *sftpd.SftpServer - if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable { - var err error - sftpDriver, err = server.NewSftpDriver() - if err != nil { - utils.Log.Fatalf("failed to start sftp driver: %s", err.Error()) - } else { - fmt.Printf("start sftp server on %s", conf.Conf.SFTP.Listen) - utils.Log.Infof("start sftp server on %s", conf.Conf.SFTP.Listen) - go func() { - sftpServer = sftpd.NewSftpServer(sftpDriver) - err = sftpServer.RunServer() - if err != nil { - utils.Log.Fatalf("problem sftp server listening: %s", err.Error()) - } - }() - } - } + bootstrap.Init() + defer bootstrap.Release() + bootstrap.Start() // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) @@ -180,60 +28,7 @@ the address is defined in config file`, // kill -9 is syscall. SIGKILL but can"t be catch, so don't need add it signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) <-quit - utils.Log.Println("Shutdown server...") - fs.ArchiveContentUploadTaskManager.RemoveAll() - Release() - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - var wg sync.WaitGroup - if conf.Conf.Scheme.HttpPort != -1 { - wg.Add(1) - go func() { - defer wg.Done() - if err := httpSrv.Shutdown(ctx); err != nil { - utils.Log.Fatal("HTTP server shutdown err: ", err) - } - }() - } - if conf.Conf.Scheme.HttpsPort != -1 { - wg.Add(1) - go func() { - defer wg.Done() - if err := httpsSrv.Shutdown(ctx); err != nil { - utils.Log.Fatal("HTTPS server shutdown err: ", err) - } - }() - } - if conf.Conf.Scheme.UnixFile != "" { - wg.Add(1) - go func() { - defer wg.Done() - if err := unixSrv.Shutdown(ctx); err != nil { - utils.Log.Fatal("Unix server shutdown err: ", err) - } - }() - } - if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil && ftpDriver != nil { - wg.Add(1) - go func() { - defer wg.Done() - ftpDriver.Stop() - if err := ftpServer.Stop(); err != nil { - utils.Log.Fatal("FTP server shutdown err: ", err) - } - }() - } - if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable && sftpServer != nil && sftpDriver != nil { - wg.Add(1) - go func() { - defer wg.Done() - if err := sftpServer.Close(); err != nil { - utils.Log.Fatal("SFTP server shutdown err: ", err) - } - }() - } - wg.Wait() - utils.Log.Println("Server exit") + bootstrap.Shutdown(1 * time.Second) }, } diff --git a/cmd/storage.go b/cmd/storage.go index c744dace5..6190feb38 100644 --- a/cmd/storage.go +++ b/cmd/storage.go @@ -8,6 +8,7 @@ import ( "os" "strconv" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/charmbracelet/bubbles/table" @@ -30,8 +31,8 @@ var disableStorageCmd = &cobra.Command{ return fmt.Errorf("mount path is required") } mountPath := args[0] - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() storage, err := db.GetStorageByMountPath(mountPath) if err != nil { return fmt.Errorf("failed to query storage: %+v", err) @@ -69,8 +70,8 @@ var deleteStorageCmd = &cobra.Command{ } } - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() err = db.DeleteStorageById(uint(id)) if err != nil { return fmt.Errorf("failed to delete storage by id: %+v", err) @@ -123,8 +124,8 @@ var listStorageCmd = &cobra.Command{ Use: "list", Short: "List all storages", RunE: func(cmd *cobra.Command, args []string) error { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() storages, _, err := db.GetStorages(1, -1) if err != nil { return fmt.Errorf("failed to query storages: %+v", err) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 22db74e38..d4f5741d0 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -68,8 +68,7 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, err } userAgent := args.Header.Get("User-Agent") - downloadInfo, err := d. - DownloadWithUA(file.(*FileObj).PickCode, userAgent) + downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } @@ -252,8 +251,8 @@ func (d *Pan115) GetDetails(ctx context.Context) (*model.StorageDetails, error) } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: uint64(info.SpaceInfo.AllTotal.Size), - FreeSpace: uint64(info.SpaceInfo.AllRemain.Size), + TotalSpace: info.SpaceInfo.AllTotal.Size, + UsedSpace: info.SpaceInfo.AllUse.Size, }, }, nil } diff --git a/drivers/115/meta.go b/drivers/115/meta.go index 6a32715e7..02c180c30 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -15,10 +15,9 @@ type Addition struct { } var config = driver.Config{ - Name: "115 Cloud", - DefaultRoot: "0", - // OnlyProxy: true, - // NoOverwriteUpload: true, + Name: "115 Cloud", + DefaultRoot: "0", + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/115/types.go b/drivers/115/types.go index 28a8ced30..3477ffed0 100644 --- a/drivers/115/types.go +++ b/drivers/115/types.go @@ -22,6 +22,10 @@ func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + type UploadResult struct { driver.BasicResp Data struct { diff --git a/drivers/115/util.go b/drivers/115/util.go index b000436b2..7ae375b75 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "net/url" "strconv" "strings" @@ -22,11 +21,9 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/aliyun/aliyun-oss-go-sdk/oss" - cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" - crypto "github.com/SheltonZhu/115driver/pkg/crypto/m115" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/pkg/errors" ) @@ -108,60 +105,6 @@ func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } -func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { - key := crypto.GenerateKey() - result := driver115.DownloadResp{} - params, err := utils.Json.Marshal(map[string]string{"pick_code": pickCode}) - if err != nil { - return nil, err - } - - data := crypto.Encode(params, key) - - bodyReader := strings.NewReader(url.Values{"data": []string{data}}.Encode()) - reqUrl := fmt.Sprintf("%s?t=%s", driver115.AndroidApiDownloadGetUrl, driver115.Now().String()) - req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Cookie", d.Cookie) - req.Header.Set("User-Agent", ua) - - resp, err := d.client.Client.GetClient().Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if err := utils.Json.Unmarshal(body, &result); err != nil { - return nil, err - } - - if err = result.Err(string(body)); err != nil { - return nil, err - } - - b, err := crypto.Decode(string(result.EncodedData), key) - if err != nil { - return nil, err - } - - downloadInfo := struct { - Url string `json:"url"` - }{} - if err := utils.Json.Unmarshal(b, &downloadInfo); err != nil { - return nil, err - } - - info := &driver115.DownloadInfo{} - info.PickCode = pickCode - info.Header = resp.Request.Header - info.Url.Url = downloadInfo.Url - return info, nil -} - func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { userID := strconv.FormatInt(c.client.UserID, 10) userIDMd5 := md5.Sum([]byte(userID)) @@ -309,7 +252,8 @@ func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSPar // UploadByMultipart upload by mutipart blocks func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.UploadOSSParams, fileSize int64, s model.FileStreamer, - dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption) (*UploadResult, error) { + dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption, +) (*UploadResult, error) { var ( chunks []oss.FileChunk parts []oss.UploadPart diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index edab65abe..ec76a6bc8 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -53,6 +53,12 @@ func (d *Open115) Init(ctx context.Context) error { if d.Addition.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.Addition.LimitRate), 1) } + if d.PageSize <= 0 { + d.PageSize = 200 + } else if d.PageSize > 1150 { + d.PageSize = 1150 + } + return nil } @@ -69,7 +75,7 @@ func (d *Open115) Drop(ctx context.Context) error { func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var res []model.Obj - pageSize := int64(200) + pageSize := int64(d.PageSize) offset := int64(0) for { if err := d.WaitLimit(ctx); err != nil { @@ -131,23 +137,6 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) }, nil } -func (d *Open115) GetObjInfo(ctx context.Context, path string) (model.Obj, error) { - if err := d.WaitLimit(ctx); err != nil { - return nil, err - } - resp, err := d.client.GetFolderInfoByPath(ctx, path) - if err != nil { - return nil, err - } - return &Obj{ - Fid: resp.FileID, - Fn: resp.FileName, - Fc: resp.FileCategory, - Sha1: resp.Sha1, - Pc: resp.PickCode, - }, nil -} - func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err @@ -187,7 +176,7 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ FileID: srcObj.GetID(), - FileNma: newName, + FileName: newName, }) if err != nil { return nil, err @@ -342,18 +331,18 @@ func (d *Open115) GetDetails(ctx context.Context) (*model.StorageDetails, error) if err != nil { return nil, err } - total, err := userInfo.RtSpaceInfo.AllTotal.Size.Int64() + total, err := ParseInt64(userInfo.RtSpaceInfo.AllTotal.Size) if err != nil { return nil, err } - free, err := userInfo.RtSpaceInfo.AllRemain.Size.Int64() + used, err := ParseInt64(userInfo.RtSpaceInfo.AllUse.Size) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: uint64(total), - FreeSpace: uint64(free), + TotalSpace: total, + UsedSpace: used, }, }, nil } diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go index c24b9993a..ed908e2e6 100644 --- a/drivers/115_open/meta.go +++ b/drivers/115_open/meta.go @@ -12,13 +12,15 @@ type Addition struct { OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` + PageSize int64 `json:"page_size" type:"number" default:"200" help:"list api per page size of 115open driver"` AccessToken string `json:"access_token" required:"true"` RefreshToken string `json:"refresh_token" required:"true"` } var config = driver.Config{ - Name: "115 Open", - DefaultRoot: "0", + Name: "115 Open", + DefaultRoot: "0", + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index 3c847e057..3575678c2 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -107,16 +107,16 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, if err != nil { return err } - rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd) err = retry.Do(func() error { rd.Seek(0, io.SeekStart) - part, err := bucket.UploadPart(imur, rateLimitedRd, partSize, int(i)) + part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), partSize, int(i)) if err != nil { return err } parts[i-1] = part return nil }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) diff --git a/drivers/115_open/util.go b/drivers/115_open/util.go index 1d7ab0fff..a7a333c91 100644 --- a/drivers/115_open/util.go +++ b/drivers/115_open/util.go @@ -1,3 +1,15 @@ package _115_open -// do others that not defined in Driver interface \ No newline at end of file +import "encoding/json" + +func ParseInt64(v json.Number) (int64, error) { + i, err := v.Int64() + if err == nil { + return i, nil + } + f, e1 := v.Float64() + if e1 == nil { + return int64(f), nil + } + return int64(0), err +} diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index 00fa623e6..fe8b7733a 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -3,6 +3,7 @@ package _115_share import ( "context" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -49,9 +50,16 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr if err := d.WaitLimit(ctx); err != nil { return nil, err } - + var ua string + // TODO: will use user agent from header + // if args.Header != nil { + // ua = args.Header.Get("User-Agent") + // } + if ua == "" { + ua = base.UserAgentNT + } files := make([]driver115.ShareFile, 0) - fileResp, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) + fileResp, err := d.client.GetShareSnapWithUA(ua, d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) if err != nil { return nil, err } @@ -77,7 +85,14 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if err := d.WaitLimit(ctx); err != nil { return nil, err } - downloadInfo, err := d.client.DownloadByShareCode(d.ShareCode, d.ReceiveCode, file.GetID()) + var ua string + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + downloadInfo, err := d.client.DownloadByShareCodeWithUA(ua, d.ShareCode, d.ReceiveCode, file.GetID()) if err != nil { return nil, err } diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go index 082d9d462..f9575d493 100644 --- a/drivers/115_share/utils.go +++ b/drivers/115_share/utils.go @@ -5,6 +5,7 @@ import ( "strconv" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" driver115 "github.com/SheltonZhu/115driver/pkg/driver" @@ -20,6 +21,7 @@ type FileObj struct { FileName string isDir bool FileID string + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -54,6 +56,10 @@ func (f *FileObj) GetPath() string { return "" } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + func transFunc(sf driver115.ShareFile) (model.Obj, error) { timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) if err != nil { @@ -74,15 +80,14 @@ func transFunc(sf driver115.ShareFile) (model.Obj, error) { FileName: string(sf.FileName), isDir: isDir, FileID: fileID, + ThumbURL: sf.ThumbURL, }, nil } -var UserAgent = driver115.UA115Browser - func (d *Pan115Share) login() error { var err error opts := []driver115.Option{ - driver115.UA(UserAgent), + driver115.UA(base.UserAgentNT), } d.client = driver115.New(opts...) if _, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, ""); err != nil { diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 135ebc51a..abf0724db 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -41,7 +41,9 @@ func (d *Pan123) GetAddition() driver.Additional { } func (d *Pan123) Init(ctx context.Context) error { - _, err := d.Request(UserInfo, http.MethodGet, nil, nil) + _, err := d.Request(UserInfo, http.MethodGet, func(req *resty.Request) { + req.SetHeader("platform", "web") + }, nil) return err } @@ -258,11 +260,10 @@ func (d *Pan123) GetDetails(ctx context.Context) (*model.StorageDetails, error) if err != nil { return nil, err } - total := userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: total, - FreeSpace: total - userInfo.Data.SpaceUsed, + TotalSpace: userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp, + UsedSpace: userInfo.Data.SpaceUsed, }, }, nil } diff --git a/drivers/123/meta.go b/drivers/123/meta.go index 613ba3ad6..5b19ae82e 100644 --- a/drivers/123/meta.go +++ b/drivers/123/meta.go @@ -12,13 +12,15 @@ type Addition struct { //OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"` //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` AccessToken string - UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"` + UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"` + Platform string `json:"platform" type:"string" default:"web" help:"the platform header value, sent with API requests"` } var config = driver.Config{ Name: "123Pan", DefaultRoot: "0", LocalSort: true, + PreferProxy: true, } func init() { @@ -27,6 +29,7 @@ func init() { return &Pan123{ Addition: Addition{ UploadThread: 3, + Platform: "web", }, } }) diff --git a/drivers/123/types.go b/drivers/123/types.go index dc77b77f1..dc76c26fa 100644 --- a/drivers/123/types.go +++ b/drivers/123/types.go @@ -127,9 +127,54 @@ type UserInfoResp struct { Data struct { Uid int64 `json:"UID"` Nickname string `json:"Nickname"` - SpaceUsed uint64 `json:"SpaceUsed"` - SpacePermanent uint64 `json:"SpacePermanent"` - SpaceTemp uint64 `json:"SpaceTemp"` + SpaceUsed int64 `json:"SpaceUsed"` + SpacePermanent int64 `json:"SpacePermanent"` + SpaceTemp int64 `json:"SpaceTemp"` FileCount int `json:"FileCount"` } `json:"data"` } + +type offlineResolveResp struct { + Data struct { + List []struct { + Result int `json:"result"` + ID int64 `json:"id"` + ErrCode int `json:"err_code"` + ErrMsg string `json:"err_msg"` + Files []struct { + ID int64 `json:"id"` + } `json:"files"` + } `json:"list"` + } `json:"data"` +} + +type offlineSubmitResp struct { + Data struct { + TaskList []struct { + TaskID int64 `json:"task_id"` + Result int `json:"result"` + } `json:"task_list"` + } `json:"data"` +} + +type offlineTaskListResp struct { + Data struct { + HasRun bool `json:"has_run"` + List []offlineTask `json:"list"` + Total int `json:"total"` + } `json:"data"` +} + +type offlineTask struct { + TaskID int64 `json:"task_id"` + Name string `json:"name"` + Status int `json:"status"` + Size int64 `json:"size"` + ThirdTask string `json:"third_task_id"` + Downloaded int64 `json:"downloaded"` + Progress float64 `json:"progress"` + UploadIDR int64 `json:"upload_idr"` + UploadName string `json:"upload_name"` + Type string `json:"type"` + Speed int64 `json:"speed"` +} diff --git a/drivers/123/upload.go b/drivers/123/upload.go index a4ae988d9..cc149cfe3 100644 --- a/drivers/123/upload.go +++ b/drivers/123/upload.go @@ -125,27 +125,18 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi curSize = lastChunkSize } var reader io.ReadSeeker - var rateLimitedRd io.Reader threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { - if reader == nil { - var err error - reader, err = ss.GetSectionReader(offset, curSize) - if err != nil { - return err - } - rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader) - } - return nil + Before: func(ctx context.Context) (err error) { + reader, err = ss.GetSectionReader(offset, curSize) + return }, - Do: func(ctx context.Context) error { + Do: func(ctx context.Context) (err error) { reader.Seek(0, io.SeekStart) uploadUrl := s3PreSignedUrls.Data.PreSignedUrls[strconv.Itoa(cur)] if uploadUrl == "" { return fmt.Errorf("upload url is empty, s3PreSignedUrls: %+v", s3PreSignedUrls) } - reader.Seek(0, io.SeekStart) - req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, rateLimitedRd) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } @@ -157,7 +148,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi } defer res.Body.Close() if res.StatusCode == http.StatusForbidden { - singleflight.AnyGroup.Do(fmt.Sprintf("Pan123.newUpload_%p", threadG), func() (any, error) { + _, err, _ = singleflight.AnyGroup.Do(fmt.Sprintf("Pan123.newUpload_%p", threadG), func() (any, error) { newS3PreSignedUrls, err := getS3UploadUrl(ctx, upReq, cur, end) if err != nil { return nil, err @@ -177,7 +168,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi } return fmt.Errorf("upload s3 chunk %d failed, status code: %d, body: %s", cur, res.StatusCode, body) } - progress := 10.0 + 85.0*float64(threadG.Success())/float64(chunkCount) + progress := 100 * float64(threadG.Success()+1) / float64(chunkCount+1) up(progress) return nil }, diff --git a/drivers/123/util.go b/drivers/123/util.go index 9afe46e36..1a2ff0173 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -16,6 +16,7 @@ import ( "encoding/json" "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" @@ -47,9 +48,16 @@ const ( S3Auth = MainApi + "/file/s3_upload_object/auth" UploadCompleteV2 = MainApi + "/file/upload_complete/v2" S3Complete = MainApi + "/file/s3_complete_multipart_upload" + + OfflineResolve = MainApi + "/v2/offline_download/task/resolve" + OfflineSubmit = MainApi + "/v2/offline_download/task/submit" + OfflineTaskList = MainApi + "/offline_download/task/list" + OfflineTaskDelete = MainApi + "/offline_download/task/delete" // AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) +var ErrOfflineTaskNotFound = errors.New("offline task not found") + func signPath(path string, os string, version string) (k string, v string) { table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'} random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64())) @@ -242,10 +250,10 @@ do: "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, - "user-agent": "123pan/v2.4.7(Android_10.0;Oppo)", - "platform": "android", - "app-version": "69", - "x-app-version": "2.4.7", + "platform": d.Platform, + "user-agent": "123pan/v2.4.7(Android_10.0;Oppo)", + "app-version": "69", + "x-app-version": "2.4.7", //"user-agent": base.UserAgent, }) if callback != nil { @@ -306,6 +314,115 @@ do: return body, nil } +func (d *Pan123) OfflineDownload(ctx context.Context, uri string, dstDir model.Obj) (int64, error) { + var resolveResp offlineResolveResp + _, err := d.Request(OfflineResolve, http.MethodPost, func(req *resty.Request) { + req.SetContext(ctx).SetBody(base.Json{ + "urls": uri, + }) + }, &resolveResp) + if err != nil { + return 0, err + } + if len(resolveResp.Data.List) == 0 { + return 0, fmt.Errorf("offline resolve failed: empty response") + } + if resolveResp.Data.List[0].Result != 0 { + msg := resolveResp.Data.List[0].ErrMsg + if msg == "" { + msg = "offline resolve failed" + } + return 0, fmt.Errorf("%s", msg) + } + resourceID := resolveResp.Data.List[0].ID + if resourceID == 0 { + return 0, fmt.Errorf("offline resolve failed: empty resource id") + } + selectFileIDs := make([]int64, 0, len(resolveResp.Data.List[0].Files)) + for _, f := range resolveResp.Data.List[0].Files { + if f.ID > 0 { + selectFileIDs = append(selectFileIDs, f.ID) + } + } + if len(selectFileIDs) == 0 { + return 0, fmt.Errorf("offline resolve failed: empty file list") + } + uploadDir, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid destination dir id: %s", dstDir.GetID()) + } + + var submitResp offlineSubmitResp + _, err = d.Request(OfflineSubmit, http.MethodPost, func(req *resty.Request) { + req.SetContext(ctx).SetBody(base.Json{ + "resource_list": []base.Json{ + { + "resource_id": resourceID, + "select_file_id": selectFileIDs, + }, + }, + "upload_dir": uploadDir, + }) + }, &submitResp) + if err != nil { + return 0, err + } + if len(submitResp.Data.TaskList) == 0 { + return 0, fmt.Errorf("offline submit failed: empty task list") + } + if submitResp.Data.TaskList[0].Result != 0 { + return 0, fmt.Errorf("offline submit failed") + } + if submitResp.Data.TaskList[0].TaskID == 0 { + return 0, fmt.Errorf("offline submit failed: empty task id") + } + return submitResp.Data.TaskList[0].TaskID, nil +} + +func (d *Pan123) GetOfflineTask(ctx context.Context, taskID int64) (*offlineTask, error) { + if taskID == 0 { + return nil, fmt.Errorf("invalid task id") + } + page := 1 + pageSize := 100 + statusArr := []int{0, 1, 2, 3} + for { + var listResp offlineTaskListResp + _, err := d.Request(OfflineTaskList, http.MethodPost, func(req *resty.Request) { + req.SetContext(ctx).SetBody(base.Json{ + "current_page": page, + "page_size": pageSize, + "status_arr": statusArr, + }) + }, &listResp) + if err != nil { + return nil, err + } + for i := range listResp.Data.List { + if listResp.Data.List[i].TaskID == taskID { + return &listResp.Data.List[i], nil + } + } + if len(listResp.Data.List) == 0 || page*pageSize >= listResp.Data.Total { + break + } + page++ + } + return nil, ErrOfflineTaskNotFound +} + +func (d *Pan123) DeleteOfflineTasks(ctx context.Context, taskIDs []int64) error { + if len(taskIDs) == 0 { + return nil + } + _, err := d.Request(OfflineTaskDelete, http.MethodPost, func(req *resty.Request) { + req.SetContext(ctx).SetBody(base.Json{ + "task_ids": taskIDs, + }) + }, nil) + return err +} + func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([]File, error) { page := 1 total := 0 diff --git a/drivers/123_link/driver.go b/drivers/123_link/driver.go index af8cc2164..dec09816b 100644 --- a/drivers/123_link/driver.go +++ b/drivers/123_link/driver.go @@ -39,6 +39,10 @@ func (d *Pan123Link) Drop(ctx context.Context) error { return nil } +func (Addition) GetRootPath() string { + return "/" +} + func (d *Pan123Link) Get(ctx context.Context, path string) (model.Obj, error) { node := GetNodeFromRootByPath(d.root, path) return nodeToObj(node, path) diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 62b4b95c5..e20140277 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -18,6 +18,7 @@ type Open123 struct { model.Storage Addition UID uint64 + tm *tokenManager } func (d *Open123) Config() driver.Config { @@ -33,6 +34,24 @@ func (d *Open123) Init(ctx context.Context) error { d.UploadThread = 3 } + if d.RefreshToken != "" { + // refresh token 直接主动刷新 + d.AccessToken = "" + d.tm = &tokenManager{} + } else { + // 避免个人 token 刷新产生的多个登录，被动刷新 + // 默认过期时间90天，jwt exp 不可靠 + d.tm = &tokenManager{ + // accessToken: d.AccessToken, + expiredAt: time.Now().Add(90 * 24 * time.Hour), + } + } + + _, err := d.getAccessToken(false) + if err != nil { + return fmt.Errorf("init get access token error: %w", err) + } + return nil } @@ -162,6 +181,22 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return nil, fmt.Errorf("parse parentFileID error: %v", err) } + + // 尝试 SHA1 秒传 + sha1Hash := file.GetHash().GetHash(utils.SHA1) + if len(sha1Hash) == utils.SHA1.Width { + resp, err := d.sha1Reuse(parentFileId, file.GetName(), sha1Hash, file.GetSize(), 2) + if err == nil && resp.Data.Reuse { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: resp.Data.FileID, + Type: 2, + SHA1: sha1Hash, + }, nil + } + } + // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) if len(etag) < utils.MD5.Width { @@ -219,16 +254,22 @@ func (d *Open123) GetDetails(ctx context.Context) (*model.StorageDetails, error) if err != nil { return nil, err } - total := userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp - free := total - userInfo.Data.SpaceUsed return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: total, - FreeSpace: free, + TotalSpace: userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp, + UsedSpace: userInfo.Data.SpaceUsed, }, }, nil } +func (d *Open123) OfflineDownload(ctx context.Context, url string, dir model.Obj, callback string) (int, error) { + return d.createOfflineDownloadTask(ctx, url, dir.GetID(), callback) +} + +func (d *Open123) OfflineDownloadProcess(ctx context.Context, taskID int) (float64, int, error) { + return d.queryOfflineDownloadStatus(ctx, taskID) +} + var ( _ driver.Driver = (*Open123)(nil) _ driver.PutResult = (*Open123)(nil) diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go index db4ccc187..5481ef356 100644 --- a/drivers/123_open/meta.go +++ b/drivers/123_open/meta.go @@ -13,7 +13,7 @@ type Addition struct { ClientID string `json:"ClientID" required:"false"` ClientSecret string `json:"ClientSecret" required:"false"` - // 直接写入AccessToken + // 直接写入AccessToken, AccessToken有过期时间，不建议直接填写 AccessToken string `json:"AccessToken" required:"false"` // 用户名+密码方式登录的AccessToken可以兼容 @@ -35,6 +35,7 @@ var config = driver.Config{ Name: "123 Open", DefaultRoot: "0", LocalSort: true, + PreferProxy: true, } func init() { diff --git a/drivers/123_open/token.go b/drivers/123_open/token.go new file mode 100644 index 000000000..3c5c416c9 --- /dev/null +++ b/drivers/123_open/token.go @@ -0,0 +1,115 @@ +package _123_open + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +var ( + AccessToken = "https://open-api.123pan.com/api/v1/access_token" + RefreshToken = "https://open-api.123pan.com/api/v1/oauth2/access_token" +) + +type tokenManager struct { + // accessToken string + expiredAt time.Time + mu sync.Mutex + blockRefresh bool +} + +func (d *Open123) getAccessToken(forceRefresh bool) (string, error) { + tm := d.tm + tm.mu.Lock() + defer tm.mu.Unlock() + if tm.blockRefresh { + return "", errors.New("Authentication expired") + } + if !forceRefresh && d.AccessToken != "" && time.Now().Before(tm.expiredAt.Add(-5*time.Minute)) { + return d.AccessToken, nil + } + if err := d.flushAccessToken(); err != nil { + // token expired and failed to refresh, block further refresh attempts + tm.blockRefresh = true + return "", err + } + return d.AccessToken, nil +} + +func (d *Open123) flushAccessToken() error { + // directly send request to avoid deadlock + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "authorization": "Bearer " + d.AccessToken, + "platform": "open_platform", + "Content-Type": "application/json", + }) + + if d.ClientID != "" { + if d.RefreshToken != "" { + var resp RefreshTokenResp + req.SetQueryParam("client_id", d.ClientID) + if d.ClientSecret != "" { + req.SetQueryParam("client_secret", d.ClientSecret) + } + req.SetQueryParam("grant_type", "refresh_token") + req.SetQueryParam("refresh_token", d.RefreshToken) + req.SetResult(&resp) + res, err := req.Execute(http.MethodPost, RefreshToken) + if err != nil { + return err + } + body := res.Body() + var baseResp BaseResp + if err = json.Unmarshal(body, &baseResp); err != nil { + return err + } + if baseResp.Code != 0 { + return fmt.Errorf("get access token failed: %s", baseResp.Message) + } + + d.AccessToken = resp.AccessToken + // add token expire time + d.tm.expiredAt = time.Now().Add(time.Duration(resp.ExpiresIn) * time.Second) + d.RefreshToken = resp.RefreshToken + op.MustSaveDriverStorage(d) + d.tm.blockRefresh = false + return nil + } else if d.ClientSecret != "" { + var resp AccessTokenResp + req.SetBody(base.Json{ + "clientID": d.ClientID, + "clientSecret": d.ClientSecret, + }) + req.SetResult(&resp) + res, err := req.Execute(http.MethodPost, AccessToken) + if err != nil { + return err + } + body := res.Body() + var baseResp BaseResp + if err = json.Unmarshal(body, &baseResp); err != nil { + return err + } + if baseResp.Code != 0 { + return fmt.Errorf("get access token failed: %s", baseResp.Message) + } + d.AccessToken = resp.Data.AccessToken + // parse token expire time + d.tm.expiredAt, err = time.Parse(time.RFC3339, resp.Data.ExpiredAt) + if err != nil { + return fmt.Errorf("parse expire time failed: %w", err) + } + op.MustSaveDriverStorage(d) + d.tm.blockRefresh = false + return nil + } + } + return errors.New("no valid authentication method available") +} diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go index 70257d84f..7d586c8b0 100644 --- a/drivers/123_open/types.go +++ b/drivers/123_open/types.go @@ -19,6 +19,7 @@ func (a *ApiInfo) Require() { a.token <- struct{}{} } } + func (a *ApiInfo) Release() { if a.qps > 0 { time.AfterFunc(time.Second, func() { @@ -26,13 +27,16 @@ func (a *ApiInfo) Release() { }) } } + func (a *ApiInfo) SetQPS(qps int) { a.qps = qps a.token = make(chan struct{}, qps) } + func (a *ApiInfo) NowLen() int { return len(a.token) } + func InitApiInfo(url string, qps int) *ApiInfo { return &ApiInfo{ url: url, @@ -54,9 +58,13 @@ type File struct { Category int `json:"category"` Status int `json:"status"` Trashed int `json:"trashed"` + SHA1 string } func (f File) GetHash() utils.HashInfo { + if len(f.SHA1) == utils.SHA1.Width && len(f.Etag) != utils.MD5.Width { + return utils.NewHashInfo(utils.SHA1, f.SHA1) + } return utils.NewHashInfo(utils.MD5, f.Etag) } @@ -133,9 +141,9 @@ type UserInfoResp struct { // HeadImage string `json:"headImage"` // Passport string `json:"passport"` // Mail string `json:"mail"` - SpaceUsed uint64 `json:"spaceUsed"` - SpacePermanent uint64 `json:"spacePermanent"` - SpaceTemp uint64 `json:"spaceTemp"` + SpaceUsed int64 `json:"spaceUsed"` + SpacePermanent int64 `json:"spacePermanent"` + SpaceTemp int64 `json:"spaceTemp"` // SpaceTempExpr int64 `json:"spaceTempExpr"` // Vip bool `json:"vip"` // DirectTraffic int64 `json:"directTraffic"` @@ -185,3 +193,26 @@ type UploadCompleteResp struct { FileID int64 `json:"fileID"` } `json:"data"` } + +type SHA1ReuseResp struct { + BaseResp + Data struct { + FileID int64 `json:"fileID"` + Reuse bool `json:"reuse"` + } `json:"data"` +} + +type OfflineDownloadResp struct { + BaseResp + Data struct { + TaskID int `json:"taskID"` + } `json:"data"` +} + +type OfflineDownloadProcessResp struct { + BaseResp + Data struct { + Process float64 `json:"process"` + Status int `json:"status"` + } `json:"data"` +} diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go index 73395fcfd..0e03684e9 100644 --- a/drivers/123_open/upload.go +++ b/drivers/123_open/upload.go @@ -73,25 +73,20 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes // 表单 b := bytes.NewBuffer(make([]byte, 0, 2048)) threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { - if reader == nil { - var err error - // 每个分片一个reader - reader, err = ss.GetSectionReader(offset, size) - if err != nil { - return err - } - // 计算当前分片的MD5 + Before: func(ctx context.Context) (err error) { + reader, err = ss.GetSectionReader(offset, size) + return + }, + Do: func(ctx context.Context) (err error) { + reader.Seek(0, io.SeekStart) + if sliceMD5 == "" { + // 把耗时的计算放在这里，避免阻塞其他协程 sliceMD5, err = utils.HashReader(utils.MD5, reader) if err != nil { return err } + reader.Seek(0, io.SeekStart) } - return nil - }, - Do: func(ctx context.Context) error { - // 重置分片reader位置，因为HashReader、上一次失败已经读取到分片EOF - reader.Seek(0, io.SeekStart) b.Reset() w := multipart.NewWriter(b) @@ -121,6 +116,10 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes head := bytes.NewReader(b.Bytes()[:headSize]) tail := bytes.NewReader(b.Bytes()[headSize:]) rateLimitedRd = driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail)) + token, err := d.getAccessToken(false) + if err != nil { + return err + } // 创建请求并设置header req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadDomain+"/upload/v2/file/slice", rateLimitedRd) if err != nil { @@ -128,7 +127,7 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes } // 设置请求头 - req.Header.Add("Authorization", "Bearer "+d.AccessToken) + req.Header.Add("Authorization", "Bearer "+token) req.Header.Add("Content-Type", w.FormDataContentType()) req.Header.Add("Platform", "open_platform") @@ -140,12 +139,13 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes if res.StatusCode != 200 { return fmt.Errorf("slice %d upload failed, status code: %d", partNumber, res.StatusCode) } - var resp BaseResp - respBody, err := io.ReadAll(res.Body) + b.Reset() + _, err = b.ReadFrom(res.Body) if err != nil { return err } - err = json.Unmarshal(respBody, &resp) + var resp BaseResp + err = json.Unmarshal(b.Bytes(), &resp) if err != nil { return err } @@ -153,7 +153,7 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes return fmt.Errorf("slice %d upload failed: %s", partNumber, resp.Message) } - progress := 10.0 + 85.0*float64(threadG.Success())/float64(uploadNums) + progress := 100 * float64(threadG.Success()+1) / float64(uploadNums+1) up(progress) return nil }, @@ -183,3 +183,21 @@ func (d *Open123) complete(preuploadID string) (*UploadCompleteResp, error) { } return &resp, nil } + +// SHA1 秒传 +func (d *Open123) sha1Reuse(parentFileID int64, filename string, sha1Hash string, size int64, duplicate int) (*SHA1ReuseResp, error) { + var resp SHA1ReuseResp + _, err := d.Request(UploadSHA1Reuse, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "parentFileID": parentFileID, + "filename": filename, + "sha1": strings.ToLower(sha1Hash), + "size": size, + "duplicate": duplicate, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} diff --git a/drivers/123_open/util.go b/drivers/123_open/util.go index 17da9a66f..1b6eea2da 100644 --- a/drivers/123_open/util.go +++ b/drivers/123_open/util.go @@ -13,7 +13,6 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/go-resty/resty/v2" "github.com/google/uuid" log "github.com/sirupsen/logrus" @@ -22,26 +21,31 @@ import ( var ( // 不同情况下获取的AccessTokenQPS限制不同 如下模块化易于拓展 Api = "https://open-api.123pan.com" - AccessToken = InitApiInfo(Api+"/api/v1/access_token", 1) - RefreshToken = InitApiInfo(Api+"/api/v1/oauth2/access_token", 1) - UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) - FileList = InitApiInfo(Api+"/api/v2/file/list", 3) - DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 5) - DirectLink = InitApiInfo(Api+"/api/v1/direct-link/url", 5) - Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) - Move = InitApiInfo(Api+"/api/v1/file/move", 1) - Rename = InitApiInfo(Api+"/api/v1/file/name", 1) - Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) - UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) - UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) + UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) + FileList = InitApiInfo(Api+"/api/v2/file/list", 3) + DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 5) + DirectLink = InitApiInfo(Api+"/api/v1/direct-link/url", 5) + Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) + Move = InitApiInfo(Api+"/api/v1/file/move", 1) + Rename = InitApiInfo(Api+"/api/v1/file/name", 1) + Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) + UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) + UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) + UploadSHA1Reuse = InitApiInfo(Api+"/upload/v2/file/sha1_reuse", 2) + + OfflineDownload = InitApiInfo(Api+"/api/v1/offline/download", 1) + OfflineDownloadProcess = InitApiInfo(Api+"/api/v1/offline/download/process", 5) ) func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - retryToken := true for { + token, err := d.getAccessToken(false) + if err != nil { + return nil, err + } req := base.RestyClient.R() req.SetHeaders(map[string]string{ - "authorization": "Bearer " + d.AccessToken, + "authorization": "Bearer " + token, "platform": "open_platform", "Content-Type": "application/json", }) @@ -71,9 +75,9 @@ func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCall if baseResp.Code == 0 { return body, nil - } else if baseResp.Code == 401 && retryToken { - retryToken = false - if err := d.flushAccessToken(); err != nil { + } else if baseResp.Code == 401 { + // 强制刷新Token, 有小概率会 race condition 导致多次刷新Token，但不影响正确运行 + if _, err := d.getAccessToken(true); err != nil { return nil, err } } else if baseResp.Code == 429 { @@ -85,42 +89,6 @@ func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCall } } -func (d *Open123) flushAccessToken() error { - if d.ClientID != "" { - if d.RefreshToken != "" { - var resp RefreshTokenResp - _, err := d.Request(RefreshToken, http.MethodPost, func(req *resty.Request) { - req.SetQueryParam("client_id", d.ClientID) - if d.ClientSecret != "" { - req.SetQueryParam("client_secret", d.ClientSecret) - } - req.SetQueryParam("grant_type", "refresh_token") - req.SetQueryParam("refresh_token", d.RefreshToken) - }, &resp) - if err != nil { - return err - } - d.AccessToken = resp.AccessToken - d.RefreshToken = resp.RefreshToken - op.MustSaveDriverStorage(d) - } else if d.ClientSecret != "" { - var resp AccessTokenResp - _, err := d.Request(AccessToken, http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "clientID": d.ClientID, - "clientSecret": d.ClientSecret, - }) - }, &resp) - if err != nil { - return err - } - d.AccessToken = resp.Data.AccessToken - op.MustSaveDriverStorage(d) - } - } - return nil -} - func (d *Open123) SignURL(originURL, privateKey string, uid uint64, validDuration time.Duration) (newURL string, err error) { // 生成Unix时间戳 ts := time.Now().Add(validDuration).Unix() @@ -277,3 +245,34 @@ func (d *Open123) trash(fileId int64) error { return nil } + +func (d *Open123) createOfflineDownloadTask(ctx context.Context, url string, dirID, callback string) (taskID int, err error) { + body := base.Json{ + "url": url, + "dirID": dirID, + } + if len(callback) > 0 { + body["callBackUrl"] = callback + } + var resp OfflineDownloadResp + _, err = d.Request(OfflineDownload, http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &resp) + if err != nil { + return 0, err + } + return resp.Data.TaskID, nil +} + +func (d *Open123) queryOfflineDownloadStatus(ctx context.Context, taskID int) (process float64, status int, err error) { + var resp OfflineDownloadProcessResp + _, err = d.Request(OfflineDownloadProcess, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "taskID": strconv.Itoa(taskID), + }) + }, &resp) + if err != nil { + return .0, 0, err + } + return resp.Data.Process, resp.Data.Status, nil +} diff --git a/drivers/123_share/meta.go b/drivers/123_share/meta.go index 12c620d7a..e40965fe9 100644 --- a/drivers/123_share/meta.go +++ b/drivers/123_share/meta.go @@ -19,6 +19,7 @@ var config = driver.Config{ LocalSort: true, NoUpload: true, DefaultRoot: "0", + PreferProxy: true, } func init() { diff --git a/drivers/139/driver.go b/drivers/139/driver.go index fbbc7f8e0..4e3ea3e92 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -14,6 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -28,6 +29,7 @@ type Yun139 struct { Account string ref *Yun139 PersonalCloudHost string + RootPath string } func (d *Yun139) Config() driver.Config { @@ -41,7 +43,16 @@ func (d *Yun139) GetAddition() driver.Additional { func (d *Yun139) Init(ctx context.Context) error { if d.ref == nil { if len(d.Authorization) == 0 { - return fmt.Errorf("authorization is empty") + if d.Username != "" && d.Password != "" { + log.Infof("139yun: authorization is empty, trying to login with password.") + newAuth, err := d.loginWithPassword() + log.Debugf("newAuth: Ok: %s", newAuth) + if err != nil { + return fmt.Errorf("login with password failed: %w", err) + } + } else { + return fmt.Errorf("authorization is empty and username/password is not provided") + } } err := d.refreshToken() if err != nil { @@ -92,7 +103,22 @@ func (d *Yun139) Init(ctx context.Context) error { if len(d.Addition.RootFolderID) == 0 { d.RootFolderID = d.CloudID } + _, err := d.groupGetFiles(d.RootFolderID) + if err != nil { + return err + } case MetaFamily: + if len(d.Addition.RootFolderID) == 0 { + // Attempt to obtain data.path as the root via a query and persist it. + if root, err := d.getFamilyRootPath(d.CloudID); err == nil && root != "" { + d.RootFolderID = root + op.MustSaveDriverStorage(d) + } + } + _, err := d.familyGetFiles(d.RootFolderID) + if err != nil { + return err + } default: return errs.NotImplement } @@ -279,6 +305,42 @@ func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, return nil, err } return srcObj, nil + case MetaFamily: + pathname := "/isbo/openApi/createBatchOprTask" + var contentList []string + var catalogList []string + if srcObj.IsDir() { + catalogList = append(catalogList, path.Join(srcObj.GetPath(), srcObj.GetID())) + } else { + contentList = append(contentList, path.Join(srcObj.GetPath(), srcObj.GetID())) + } + + body := base.Json{ + "catalogList": catalogList, + "accountInfo": base.Json{ + "accountName": d.getAccount(), + "accountType": "1", + }, + "contentList": contentList, + "destCatalogID": dstDir.GetID(), + "destGroupID": d.CloudID, + "destPath": path.Join(dstDir.GetPath(), dstDir.GetID()), + "destType": 0, + "srcGroupID": d.CloudID, + "srcType": 0, + "taskType": 3, + } + + var resp CreateBatchOprTaskResp + _, err := d.isboPost(pathname, body, &resp) + if err != nil { + return nil, err + } + log.Debugf("[139] Move MetaFamily CreateBatchOprTaskResp.Result.ResultCode: %s", resp.Result.ResultCode) + if resp.Result.ResultCode != "0" { + return nil, fmt.Errorf("failed to move in family cloud: %s", resp.Result.ResultDesc) + } + return srcObj, nil default: return nil, errs.NotImplement } @@ -353,19 +415,27 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e var data base.Json var pathname string if srcObj.IsDir() { - // 网页接口不支持重命名家庭云文件夹 - // data = base.Json{ - // "catalogType": 3, - // "catalogID": srcObj.GetID(), - // "catalogName": newName, - // "commonAccountInfo": base.Json{ - // "account": d.getAccount(), - // "accountType": 1, - // }, - // "path": srcObj.GetPath(), - // } - // pathname = "/orchestration/familyCloud-rebuild/photoContent/v1.0/modifyCatalogInfo" - return errs.NotImplement + pathname = "/modifyCloudDocV2" + data = base.Json{ + "catalogType": 3, + "cloudID": d.CloudID, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": "1", + }, + "docLibName": newName, + "docLibraryID": srcObj.GetID(), + "path": path.Join(srcObj.GetPath(), srcObj.GetID()), + } + var resp ModifyCloudDocV2Resp + _, err = d.andAlbumRequest(pathname, data, &resp) + if err != nil { + return err + } + if resp.Result.ResultCode != "0" { + return fmt.Errorf("failed to rename family folder: %s", resp.Result.ResultDesc) + } + return nil } else { data = base.Json{ "contentID": srcObj.GetID(), @@ -421,6 +491,33 @@ func (d *Yun139) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" _, err = d.post(pathname, data, nil) + case MetaGroup: + err = d.handleMetaGroupCopy(ctx, srcObj, dstDir) + case MetaFamily: + pathname := "/copyContentCatalog" + var sourceContentIDs []string + var sourceCatalogIDs []string + if srcObj.IsDir() { + sourceCatalogIDs = append(sourceCatalogIDs, srcObj.GetID()) + } else { + sourceContentIDs = append(sourceContentIDs, srcObj.GetID()) + } + + body := base.Json{ + "commonAccountInfo": base.Json{ + "accountType": "1", + "accountUserId": d.ref.UserDomainID, + }, + "destCatalogID": dstDir.GetID(), + "destCloudID": d.CloudID, + "sourceCatalogIDs": sourceCatalogIDs, + "sourceCloudID": d.CloudID, + "sourceContentIDs": sourceContentIDs, + } + + var resp base.Json // Assuming a generic JSON response for success/failure + _, err = d.andAlbumRequest(pathname, body, &resp) + // For now, we assume no error means success. default: err = errs.NotImplement } @@ -680,6 +777,8 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return nil case MetaPersonal: fallthrough + case MetaGroup: + fallthrough case MetaFamily: // 处理冲突 // 获取文件列表 @@ -727,12 +826,17 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr }, } pathname := "/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest" - if d.isFamily() { + if d.isFamily() || d.Addition.Type == MetaGroup { + uploadPath := path.Join(dstDir.GetPath(), dstDir.GetID()) + // if dstDir is root folder + if dstDir.GetID() == d.RootFolderID { + uploadPath = d.RootPath + } data = d.newJson(base.Json{ "fileCount": 1, "manualRename": 2, "operation": 0, - "path": path.Join(dstDir.GetPath(), dstDir.GetID()), + "path": uploadPath, "seqNo": random.String(32), // 序列号不能为空 "totalSize": reportSize, "uploadContentList": []base.Json{{ @@ -744,6 +848,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr pathname = "/orchestration/familyCloud-rebuild/content/v1.0/getFileUploadURL" } var resp UploadResp + log.Debugf("[139] upload request body: %+v", data) _, err = d.post(pathname, data, &resp) if err != nil { return err @@ -839,42 +944,42 @@ func (d *Yun139) GetDetails(ctx context.Context) (*model.StorageDetails, error) if d.UserDomainID == "" { return nil, errs.NotImplement } - var total, free uint64 + var total, used int64 if d.isFamily() { diskInfo, err := d.getFamilyDiskInfo(ctx) if err != nil { return nil, err } - totalMb, err := strconv.ParseUint(diskInfo.Data.DiskSize, 10, 64) + totalMb, err := strconv.ParseInt(diskInfo.Data.DiskSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert disk size into integer: %+v", err) } - usedMb, err := strconv.ParseUint(diskInfo.Data.UsedSize, 10, 64) + usedMb, err := strconv.ParseInt(diskInfo.Data.UsedSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert used size into integer: %+v", err) } total = totalMb * 1024 * 1024 - free = total - (usedMb * 1024 * 1024) + used = usedMb * 1024 * 1024 } else { diskInfo, err := d.getPersonalDiskInfo(ctx) if err != nil { return nil, err } - totalMb, err := strconv.ParseUint(diskInfo.Data.DiskSize, 10, 64) + totalMb, err := strconv.ParseInt(diskInfo.Data.DiskSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert disk size into integer: %+v", err) } - freeMb, err := strconv.ParseUint(diskInfo.Data.FreeDiskSize, 10, 64) + freeMb, err := strconv.ParseInt(diskInfo.Data.FreeDiskSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert free size into integer: %+v", err) } total = totalMb * 1024 * 1024 - free = freeMb * 1024 * 1024 + used = total - (freeMb * 1024 * 1024) } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: used, }, }, nil } diff --git a/drivers/139/meta.go b/drivers/139/meta.go index b226e240f..91d54fd30 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -8,6 +8,9 @@ import ( type Addition struct { //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true" secret:"true"` + MailCookies string `json:"mail_cookies" required:"true" type:"text" help:"Cookies from mail.139.com used for login authentication."` driver.RootID Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` CloudID string `json:"cloud_id"` diff --git a/drivers/139/types.go b/drivers/139/types.go index 118655deb..499cdbc4d 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -329,3 +329,62 @@ type FamilyDiskInfoResp struct { DiskSize string `json:"diskSize"` } `json:"data"` } + +type AndAlbumUploadResp struct { + Result struct { + ResultCode string `json:"resultCode"` + ResultDesc string `json:"resultDesc"` + } `json:"result"` + UploadResult struct { + UploadTaskID string `json:"uploadTaskID"` + RedirectionURL string `json:"redirectionUrl"` + NewContentIDList []struct { + ContentID string `json:"contentID"` + ContentName string `json:"contentName"` + } `json:"newContentIDList"` + } `json:"uploadResult"` +} + +type ModifyCloudDocV2Req struct { + CatalogType int `json:"catalogType"` + CloudID string `json:"cloudID"` + CommonAccountInfo struct { + Account string `json:"account"` + AccountType string `json:"accountType"` + } `json:"commonAccountInfo"` + DocLibName string `json:"docLibName"` + DocLibraryID string `json:"docLibraryID"` + Path string `json:"path"` +} + +type ModifyCloudDocV2Resp struct { + Result struct { + ResultCode string `json:"resultCode"` + ResultDesc string `json:"resultDesc"` + } `json:"result"` +} + +type CreateBatchOprTaskReq struct { + CatalogList []string `json:"catalogList"` + CommonAccountInfo struct { + Account string `json:"account"` + AccountType string `json:"accountType"` + } `json:"commonAccountInfo"` + ContentList []string `json:"contentList"` + DestCatalogID string `json:"destCatalogID"` + DestGroupID string `json:"destGroupID"` + DestPath string `json:"destPath"` + DestType int `json:"destType"` + SourceCatalogType int `json:"sourceCatalogType"` + SourceCloudID string `json:"sourceCloudID"` + SourceType int `json:"sourceType"` + TaskType int `json:"taskType"` +} + +type CreateBatchOprTaskResp struct { + Result struct { + ResultCode string `json:"resultCode"` + ResultDesc string `json:"resultDesc"` + } `json:"result"` + TaskID string `json:"taskID"` +} diff --git a/drivers/139/util.go b/drivers/139/util.go index 9a9d2c6d0..2c7ac242d 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -1,14 +1,22 @@ package _139 import ( + "bytes" "context" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + crypto_rand "crypto/rand" + "crypto/sha1" "encoding/base64" + "encoding/hex" "errors" "fmt" "io" "net/http" "net/url" "path" + "regexp" "sort" "strconv" "strings" @@ -25,6 +33,11 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + KEY_HEX_1 = "73634235495062495331515373756c734e7253306c673d3d" // 第一层 AES 解密密钥 + KEY_HEX_2 = "7150714477323633586746674c337538" // 第二层 AES 解密密钥 +) + // do others that not defined in Driver interface func (d *Yun139) isFamily() bool { return d.Type == "family" @@ -96,12 +109,16 @@ func (d *Yun139) refreshToken() error { SetBody(reqBody). SetResult(&resp). Post(url) - if err != nil { - return err - } - if resp.Return != "0" { - return fmt.Errorf("failed to refresh token: %s", resp.Desc) + if err != nil || resp.Return != "0" { + log.Warnf("139yun: failed to refresh token with old token: %v, desc: %s. trying to login with password.", err, resp.Desc) + newAuth, loginErr := d.loginWithPassword() + log.Debugf("newAuth: Ok: %s", newAuth) + if loginErr != nil { + return fmt.Errorf("failed to login with password after refresh failed: %w", loginErr) + } + return nil } + d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + ":" + splits[1] + ":" + resp.Token)) op.MustSaveDriverStorage(d) return nil @@ -146,10 +163,29 @@ func (d *Yun139) request(url string, method string, callback base.ReqCallback, r var e BaseResp req.SetResult(&e) + log.Debugf("[139] request: %s %s, body: %s", method, url, string(body)) res, err := req.Execute(method, url) - log.Debugln(res.String()) + if err != nil { + log.Debugf("[139] request error: %v", err) + return nil, err + } + log.Debugf("[139] response body: %s", res.String()) if !e.Success { - return nil, errors.New(e.Message) + // Always try to unmarshal to the specific response type first if 'resp' is provided. + if resp != nil { + err = utils.Json.Unmarshal(res.Body(), resp) + if err != nil { + log.Debugf("[139] failed to unmarshal response to specific type: %v", err) + return nil, err // Return unmarshal error + } + if createBatchOprTaskResp, ok := resp.(*CreateBatchOprTaskResp); ok { + log.Debugf("[139] CreateBatchOprTaskResp.Result.ResultCode: %s", createBatchOprTaskResp.Result.ResultCode) + if createBatchOprTaskResp.Result.ResultCode == "0" { + goto SUCCESS_PROCESS + } + } + } + return nil, errors.New(e.Message) // Fallback to original error if not handled } if resp != nil { err = utils.Json.Unmarshal(res.Body(), resp) @@ -157,6 +193,7 @@ func (d *Yun139) request(url string, method string, callback base.ReqCallback, r return nil, err } } +SUCCESS_PROCESS: return res.Body(), nil } @@ -311,6 +348,9 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { return nil, err } path := resp.Data.Path + if catalogID == d.RootFolderID { + d.RootPath = path + } for _, catalog := range resp.Data.CloudCatalogList { f := model.Object{ ID: catalog.CatalogID, @@ -366,6 +406,9 @@ func (d *Yun139) groupGetFiles(catalogID string) ([]model.Obj, error) { return nil, err } path := resp.Data.GetGroupContentResult.ParentCatalogID + if catalogID == d.RootFolderID { + d.RootPath = path + } for _, catalog := range resp.Data.GetGroupContentResult.CatalogList { f := model.Object{ ID: catalog.CatalogID, @@ -494,11 +537,13 @@ func (d *Yun139) personalRequest(pathname string, method string, callback base.R var e BaseResp req.SetResult(&e) + log.Debugf("[139] personal request: %s %s, body: %s", method, url, string(body)) res, err := req.Execute(method, url) if err != nil { + log.Debugf("[139] personal request error: %v", err) return nil, err } - log.Debugln(res.String()) + log.Debugf("[139] personal response body: %s", res.String()) if !e.Success { return nil, errors.New(e.Message) } @@ -517,6 +562,13 @@ func (d *Yun139) personalPost(pathname string, data interface{}, resp interface{ }, resp) } +func (d *Yun139) isboPost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + url := "https://group.yun.139.com/hcy/mutual/adapter" + pathname + return d.request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, resp) +} + func getPersonalTime(t string) time.Time { stamp, err := time.ParseInLocation("2006-01-02T15:04:05.999-07:00", t, utils.CNLoc) if err != nil { @@ -703,3 +755,645 @@ func (d *Yun139) getFamilyDiskInfo(ctx context.Context) (*FamilyDiskInfoResp, er } return &resp, nil } + +func getMd5(dataStr string) string { + hash := md5.Sum([]byte(dataStr)) + return fmt.Sprintf("%x", hash) +} + +func (d *Yun139) step1_password_login() (string, error) { + log.Debugf("--- 执行步骤 1: 登录 API ---") + loginURL := "https://mail.10086.cn/Login/Login.ashx" + + // 密码 SHA1 哈希 + hashedPassword := sha1Hash(fmt.Sprintf("fetion.com.cn:%s", d.Password)) + log.Debugf("DEBUG: 原始密码: %s", d.Password) + log.Debugf("DEBUG: SHA1 输入: fetion.com.cn:%s", d.Password) + log.Debugf("DEBUG: 生成的 Password 哈希: %s", hashedPassword) + + cguid := strconv.FormatInt(time.Now().UnixMilli(), 10) // 随机生成 cguid + + loginHeaders := map[string]string{ + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "accept-language": "zh-CN,zh;q=0.9,zh-TW;q=0.8,en-US;q=0.7,en;q=0.6,en-GB;q=0.5", + "cache-control": "max-age=0", + "content-type": "application/x-www-form-urlencoded", + "dnt": "1", + "origin": "https://mail.10086.cn", + "priority": "u=0, i", + "referer": fmt.Sprintf("https://mail.10086.cn/default.html?&s=1&v=0&u=%s&m=1&ec=S001&resource=indexLogin&clientid=1003&auto=on&cguid=%s&mtime=45", base64.StdEncoding.EncodeToString([]byte(d.Username)), cguid), + "sec-ch-ua": "\"Microsoft Edge\";v=\"141\", \"Not?A_Brand\";v=\"8\", \"Chromium\";v=\"141\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Windows\"", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "same-origin", + "sec-fetch-user": "?1", + "upgrade-insecure-requests": "1", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0", + "Cookie": d.MailCookies, + } + + loginData := url.Values{} + loginData.Set("UserName", d.Username) + loginData.Set("passOld", "") + loginData.Set("auto", "on") + loginData.Set("Password", hashedPassword) + loginData.Set("webIndexPagePwdLogin", "1") + loginData.Set("pwdType", "1") + loginData.Set("clientId", "1003") + loginData.Set("authType", "2") + + log.Debugf("DEBUG: 登录请求 URL: %s", loginURL) + log.Debugf("DEBUG: 登录请求 Headers: %+v", loginHeaders) + log.Debugf("DEBUG: 登录请求 Body: %s", loginData.Encode()) + + // 设置客户端不跟随重定向 + client := base.RestyClient.SetRedirectPolicy(resty.NoRedirectPolicy()) + res, err := client.R(). + SetHeaders(loginHeaders). + SetFormDataFromValues(loginData). + Post(loginURL) + + if err != nil { + // 如果是重定向错误，则不作为失败处理，因为我们禁止了自动重定向 + if res != nil && res.StatusCode() >= 300 && res.StatusCode() < 400 { + log.Debugf("DEBUG: 登录响应 Status Code: %d (Redirect)", res.StatusCode()) + } else { + return "", fmt.Errorf("step1 login request failed: %w", err) + } + } else { + log.Debugf("DEBUG: 登录响应 Status Code: %d", res.StatusCode()) + } + // 恢复客户端的默认重定向策略，以免影响后续请求 + base.RestyClient.SetRedirectPolicy(resty.FlexibleRedirectPolicy(10)) + log.Debugf("DEBUG: 登录响应 Headers: %+v", res.Header()) + + var sid, extractedCguid string + + // 从 Location 头部提取 sid 和 cguid + locationHeader := res.Header().Get("Location") + if locationHeader != "" { + sidMatch := regexp.MustCompile(`sid=([^&]+)`).FindStringSubmatch(locationHeader) + cguidMatch := regexp.MustCompile(`cguid=([^&]+)`).FindStringSubmatch(locationHeader) + if len(sidMatch) > 1 { + sid = sidMatch[1] + log.Debugf("DEBUG: 从 Location 提取到 sid: %s", sid) + } + if len(cguidMatch) > 1 { + extractedCguid = cguidMatch[1] + log.Debugf("DEBUG: 从 Location 提取到 cguid: %s", extractedCguid) + } + } + + // 如果 Location 中没有，尝试从 Set-Cookie 中提取 + if sid == "" || extractedCguid == "" { + setCookieHeaders := res.Header().Values("Set-Cookie") + for _, cookieStr := range setCookieHeaders { + ssoSidMatch := regexp.MustCompile(`Os_SSo_Sid=([^;]+)`).FindStringSubmatch(cookieStr) + cookieCguidMatch := regexp.MustCompile(`cguid=([^;]+)`).FindStringSubmatch(cookieStr) + if len(ssoSidMatch) > 1 && sid == "" { + sid = ssoSidMatch[1] + log.Debugf("DEBUG: 从 Set-Cookie 提取到 sid: %s", sid) + } + if len(cookieCguidMatch) > 1 && extractedCguid == "" { + extractedCguid = cookieCguidMatch[1] + log.Debugf("DEBUG: 从 Set-Cookie 提取到 cguid: %s", extractedCguid) + } + } + } + + if sid == "" || extractedCguid == "" { + return "", errors.New("failed to extract sid or cguid from login response") + } + + // 提取并记录 cookies + loginUrlObj, _ := url.Parse(loginURL) + cookies := base.RestyClient.GetClient().Jar.Cookies(loginUrlObj) + var cookieStrings []string + for _, cookie := range cookies { + cookieStrings = append(cookieStrings, cookie.Name+"="+cookie.Value) + } + cookieStr := strings.Join(cookieStrings, "; ") + log.Debugf("DEBUG: 提取到的 Cookies: %s", cookieStr) + d.MailCookies = cookieStr + + return sid, nil +} + +func (d *Yun139) step2_get_single_token(sid string) (string, error) { + log.Debugf("\n--- 执行步骤 2: 换artifact API ---") + cguid := strconv.FormatInt(time.Now().UnixMilli(), 10) + + exchangeArtifactURL := fmt.Sprintf("https://smsrebuild1.mail.10086.cn/setting/s?func=%s&sid=%s&cguid=%s", url.QueryEscape("umc:getArtifact"), sid, cguid) + + // 从 MailCookies 中提取 RMKEY + var rmkey string + cookies := strings.Split(d.MailCookies, ";") + for _, cookie := range cookies { + cookie = strings.TrimSpace(cookie) + if strings.HasPrefix(cookie, "RMKEY=") { + rmkey = cookie + break + } + } + if rmkey == "" { + return "", errors.New("RMKEY not found in MailCookies") + } + + exchangePassidHeaders := map[string]string{ + "Host": "smsrebuild1.mail.10086.cn", + "Cookie": rmkey, + "Content-Type": "text/xml; charset=utf-8", + "Accept-Encoding": "gzip", + "User-Agent": "okhttp/4.12.0", + } + + log.Debugf("DEBUG: 换passid 请求 URL: %s", exchangeArtifactURL) + log.Debugf("DEBUG: 换passid 请求 Headers: %+v", exchangePassidHeaders) + + res, err := base.RestyClient.R(). + SetHeaders(exchangePassidHeaders). + Post(exchangeArtifactURL) + + if err != nil { + return "", fmt.Errorf("step2 exchange artifact request failed: %w", err) + } + + log.Debugf("DEBUG: 换passid 响应 Status Code: %d", res.StatusCode()) + log.Debugf("DEBUG: 换passid 响应 Headers: %+v", res.Header()) + log.Debugf("DEBUG: 换passid 响应 Body: %s...", res.String()[:min(len(res.String()), 500)]) + + dycpwd := jsoniter.Get(res.Body(), "var", "artifact").ToString() + if dycpwd == "" { + return "", errors.New("failed to extract dycpwd from artifact exchange response") + } + log.Debugf("DEBUG: 提取到 dycpwd: %s", dycpwd) + + return dycpwd, nil +} + +// --- 辅助函数：加密/解密 --- + +// sha1Hash 计算 SHA1 哈希值，返回十六进制字符串。 +func sha1Hash(data string) string { + h := sha1.New() + h.Write([]byte(data)) + return hex.EncodeToString(h.Sum(nil)) +} + +// pkcs7_pad PKCS7 填充 +func pkcs7_pad(data []byte, blockSize int) []byte { + padding := blockSize - len(data)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + return append(data, padtext...) +} + +// pkcs7_unpad PKCS7 去填充 +func pkcs7_unpad(data []byte) ([]byte, error) { + length := len(data) + if length == 0 { + return nil, errors.New("pkcs7: data is empty") + } + unpadding := int(data[length-1]) + if unpadding > length { + return nil, errors.New("pkcs7: invalid padding") + } + return data[:(length - unpadding)], nil +} + +// aes_ecb_decrypt AES/ECB/Pkcs7 解密，输入为十六进制字符串。 +func aes_ecb_decrypt(ciphertext []byte, key []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + if len(ciphertext)%block.BlockSize() != 0 { + return nil, errors.New("AES ECB decrypt: ciphertext is not a multiple of the block size") + } + + decrypted := make([]byte, len(ciphertext)) + blockSize := block.BlockSize() + + for bs, be := 0, blockSize; bs < len(ciphertext); bs, be = bs+blockSize, be+blockSize { + block.Decrypt(decrypted[bs:be], ciphertext[bs:be]) + } + + return pkcs7_unpad(decrypted) +} + +// 以下提供 camelCase 的 AES CBC 加解密，供文件中其它位置调用（并支持传入 IV）。 +func aesCbcEncrypt(plaintext []byte, key []byte, iv []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + if len(iv) != block.BlockSize() { + return nil, fmt.Errorf("aesCbcEncrypt: iv length %d does not match block size %d", len(iv), block.BlockSize()) + } + padded := pkcs7_pad(plaintext, block.BlockSize()) + ciphertext := make([]byte, len(padded)) + mode := cipher.NewCBCEncrypter(block, iv) + mode.CryptBlocks(ciphertext, padded) + return ciphertext, nil +} + +func aesCbcDecrypt(ciphertext []byte, key []byte, iv []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + if len(iv) != block.BlockSize() { + return nil, fmt.Errorf("aesCbcDecrypt: iv length %d does not match block size %d", len(iv), block.BlockSize()) + } + if len(ciphertext)%block.BlockSize() != 0 { + return nil, errors.New("aesCbcDecrypt: ciphertext is not a multiple of the block size") + } + decrypted := make([]byte, len(ciphertext)) + mode := cipher.NewCBCDecrypter(block, iv) + mode.CryptBlocks(decrypted, ciphertext) + return pkcs7_unpad(decrypted) +} + +// sortedJsonStringify 对 JSON 对象进行排序并字符串化。 +func sortedJsonStringify(obj interface{}) (string, error) { + if obj == nil { + return "null", nil + } + + switch v := obj.(type) { + case string: + // 尝试解析为 JSON，如果成功则递归处理 + var parsed interface{} + if err := jsoniter.Unmarshal([]byte(v), &parsed); err == nil { + return sortedJsonStringify(parsed) + } + // 如果不是 JSON 字符串，则直接返回 JSON 字符串化的结果 + return jsoniter.MarshalToString(v) + case int, float64, bool: + return fmt.Sprintf("%v", v), nil + case []interface{}: + var items []string + for _, item := range v { + s, err := sortedJsonStringify(item) + if err != nil { + return "", err + } + items = append(items, s) + } + return fmt.Sprintf("[%s]", strings.Join(items, ",")), nil + case map[string]interface{}: + sortedKeys := make([]string, 0, len(v)) + for key := range v { + sortedKeys = append(sortedKeys, key) + } + sort.Strings(sortedKeys) + + var pairs []string + for _, key := range sortedKeys { + value := v[key] + s, err := sortedJsonStringify(value) + if err != nil { + return "", err + } + // Use jsoniter.MarshalToString for the key to ensure it's quoted correctly + keyStr, err := jsoniter.MarshalToString(key) + if err != nil { + return "", err + } + pairs = append(pairs, fmt.Sprintf("%s:%s", keyStr, s)) + } + return fmt.Sprintf("{%s}", strings.Join(pairs, ",")), nil + default: + // Fallback for other types, e.g., numbers, booleans, or unhandled complex types + // Use jsoniter's default marshalling for these + return jsoniter.MarshalToString(v) + } +} + +// yun139EncryptedRequest handles the common encrypted request/response flow. +func (d *Yun139) yun139EncryptedRequest(url string, body interface{}, headers map[string]string, aesKeyHex string, resp interface{}) ([]byte, error) { + // 1. Decode AES key + aesKey, err := hex.DecodeString(aesKeyHex) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to decode AES key: %w", err) + } + + // 2. Marshal and sort the request body + sortedJson, err := sortedJsonStringify(body) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to marshal and sort body: %w", err) + } + log.Debugf("yun139EncryptedRequest: Request Body (plaintext): %s", sortedJson) + + // 3. Encrypt the body using AES/CBC + iv := make([]byte, 16) // 16 bytes for AES-128 + if _, err := crypto_rand.Read(iv); err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to generate IV: %w", err) + } + encryptedBody, err := aesCbcEncrypt([]byte(sortedJson), aesKey, iv) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to encrypt body: %w", err) + } + payload := base64.StdEncoding.EncodeToString(append(iv, encryptedBody...)) + + // 4. Make the request + res, err := base.RestyClient.R(). + SetHeaders(headers). + SetBody(payload). + Post(url) + + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: http request failed: %w", err) + } + + if res.StatusCode() != 200 { + return nil, fmt.Errorf("yun139EncryptedRequest: unexpected status code %d: %s", res.StatusCode(), res.String()) + } + + // 5. Decrypt the response + respBody := res.Body() + var decryptedBytes []byte + + if len(respBody) > 0 && respBody[0] == '{' { + log.Warnf("yun139EncryptedRequest: received a plain JSON response, not an encrypted string. Body: %s", string(respBody)) + decryptedBytes = respBody + } else { + decodedResp, err := base64.StdEncoding.DecodeString(string(respBody)) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: response base64 decode failed: %w. Body: '%s'", err, string(respBody)) + } + + if len(decodedResp) < 16 { + return nil, fmt.Errorf("yun139EncryptedRequest: decoded response is too short to be encrypted. Length: %d", len(decodedResp)) + } + + respIv := decodedResp[:16] + respCiphertext := decodedResp[16:] + + decryptedBytes, err = aesCbcDecrypt(respCiphertext, aesKey, respIv) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: response aes decrypt failed: %w", err) + } + } + + log.Debugf("yun139EncryptedRequest: Response Body (decrypted): %s", string(decryptedBytes)) + + // 6. Unmarshal to the final response struct + if resp != nil { + err = utils.Json.Unmarshal(decryptedBytes, resp) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to unmarshal decrypted response: %w", err) + } + } + + return decryptedBytes, nil +} + +func (d *Yun139) step3_third_party_login(dycpwd string) (string, error) { + log.Debugf("\n--- 执行步骤 3: 单点登录 API ---") + ssoLoginURL := "https://user-njs.yun.139.com/user/thirdlogin" + + // 构建原始请求体 + ssoRequestBodyRaw := base.Json{ + "clientkey_decrypt": "l3TryM&Q+X7@dzwk)qP", + "clienttype": "886", + "cpid": "507", + "dycpwd": dycpwd, + "extInfo": base.Json{"ifOpenAccount": "0"}, + "loginMode": "0", + "msisdn": d.Username, + "pintype": "13", + "secinfo": strings.ToUpper(sha1Hash(fmt.Sprintf("fetion.com.cn:%s", dycpwd))), + "version": "20250901", + } + + ssoLoginHeaders := map[string]string{ + "hcy-cool-flag": "1", + "x-huawei-channelSrc": "10246600", + "x-sdk-channelSrc": "", + "x-MM-Source": "0", + "x-UserAgent": "android|23116PN5BC|android15|1.2.6|||1440x3200|10246600", + "x-DeviceInfo": "4|127.0.0.1|5|1.2.6|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|||", + "Content-Type": "text/plain;charset=UTF-8", + "Host": "user-njs.yun.139.com", + "Connection": "Keep-Alive", + "Accept-Encoding": "gzip", + "User-Agent": "okhttp/3.12.2", + } + + // 使用通用加密请求函数 + decryptedLayer1StrBytes, err := d.yun139EncryptedRequest(ssoLoginURL, ssoRequestBodyRaw, ssoLoginHeaders, KEY_HEX_1, nil) + if err != nil { + return "", fmt.Errorf("step3 encrypted request failed: %w", err) + } + + hexInner := jsoniter.Get(decryptedLayer1StrBytes, "data").ToString() + if hexInner == "" { + return "", errors.New("missing data field in first layer decryption result") + } + log.Debugf("DEBUG: 第一层解密提取到 hex_inner: %s...", hexInner[:min(len(hexInner), 50)]) + + // 第二层解密 + key2, err := hex.DecodeString(KEY_HEX_2) + if err != nil { + return "", fmt.Errorf("failed to decode KEY_HEX_2: %w", err) + } + hexInnerBytes, err := hex.DecodeString(hexInner) + if err != nil { + return "", fmt.Errorf("failed to decode hex_inner: %w", err) + } + finalJsonStrBytes, err := aes_ecb_decrypt(hexInnerBytes, key2) + if err != nil { + return "", fmt.Errorf("step3 response layer2 aes ecb decrypt failed: %w", err) + } + log.Debugf("DEBUG: 最终解密结果: %s", string(finalJsonStrBytes)) + + // 提取 authToken + authToken := jsoniter.Get(finalJsonStrBytes, "authToken").ToString() + if authToken == "" { + return "", errors.New("failed to extract authToken from final decryption result") + } + log.Debugf("DEBUG: 提取到 authToken: %s", authToken) + + // 提取 account 和 userDomainId + account := jsoniter.Get(finalJsonStrBytes, "account").ToString() + userDomainId := jsoniter.Get(finalJsonStrBytes, "userDomainId").ToString() + + if account == "" || userDomainId == "" { + return "", errors.New("failed to extract account or userDomainId from final decryption result") + } + + d.UserDomainID = userDomainId + newAuthorization := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("pc:%s:%s", account, authToken))) + return newAuthorization, nil +} + +func (d *Yun139) loginWithPassword() (string, error) { + if d.Username == "" || d.Password == "" || d.MailCookies == "" { + return "", errors.New("username, password or mail_cookies is empty") + } + + passId, err := d.step1_password_login() + if err != nil { + return "", err + } + log.Infof("Step 1 success, passId: %s", passId) + + token, err := d.step2_get_single_token(passId) + if err != nil { + return "", err + } + log.Infof("Step 2 success, token: %s", token) + + newAuth, err := d.step3_third_party_login(token) + if err != nil { + return "", err + } + log.Infof("Step 3 success, new authorization generated.") + + d.Authorization = newAuth // Ensure Authorization is also updated before saving + op.MustSaveDriverStorage(d) + return newAuth, nil +} + +func (d *Yun139) andAlbumRequest(pathname string, body interface{}, resp interface{}) ([]byte, error) { + url := "https://group.yun.139.com/hcy/family/adapter/andAlbum/openApi" + pathname + + headers := map[string]string{ + "Host": "group.yun.139.com", + "authorization": "Basic " + d.getAuthorization(), + "x-svctype": "2", + "hcy-cool-flag": "1", + "api-version": "v2", + "x-huawei-channelsrc": "10246600", + "x-sdk-channelsrc": "", + "x-mm-source": "0", + "x-deviceinfo": "1|127.0.0.1|1|12.3.2|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|zh||||032|0|", //重要参数 + "content-type": "application/json; charset=utf-8", + "user-agent": "okhttp/4.11.0", + "accept-encoding": "gzip", + } + + return d.yun139EncryptedRequest(url, body, headers, KEY_HEX_1, resp) +} + +func (d *Yun139) handleMetaGroupCopy(ctx context.Context, srcObj, dstDir model.Obj) error { + pathname := "/copyContentCatalog" + var sourceContentIDs []string + var sourceCatalogIDs []string + if srcObj.IsDir() { + sourceCatalogIDs = append(sourceCatalogIDs, path.Join("root:/", srcObj.GetPath(), srcObj.GetID())) + } else { + sourceContentIDs = append(sourceContentIDs, path.Join("root:/", srcObj.GetPath(), srcObj.GetID())) + } + + destCatalogID := path.Join("root:/", dstDir.GetPath(), dstDir.GetID()) + log.Debugf("[139Yun Group Copy] srcObj ID: %s, srcObj Path: %s, dstDir ID: %s, dstDir Path: %s, destCatalogID: %s", srcObj.GetID(), srcObj.GetPath(), dstDir.GetID(), dstDir.GetPath(), destCatalogID) + + body := base.Json{ + "commonAccountInfo": base.Json{ + "accountType": "1", + "accountUserId": d.UserDomainID, + }, + "destCatalogID": destCatalogID, + "destCloudID": d.CloudID, + "sourceCatalogIDs": sourceCatalogIDs, + "sourceCloudID": d.CloudID, + "sourceContentIDs": sourceContentIDs, + } + + var resp base.Json + _, err := d.andAlbumRequest(pathname, body, &resp) + return err +} + +// getGroupRootByCloudID 查询 group 上层信息，优先返回 parentCatalogID，回退到 catalogList[0].path +func (d *Yun139) getGroupRootByCloudID(cloudID string) (string, error) { + pathname := "/orchestration/group-rebuild/catalog/v1.0/queryGroupContentList" + body := base.Json{ + "groupID": cloudID, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + "pageInfo": base.Json{ + "pageNum": 1, + "pageSize": 1, + }, + } + var resp base.Json + _, err := d.post(pathname, body, &resp) + if err != nil { + return "", err + } + dataObj, _ := resp["data"].(map[string]interface{}) + if dataObj == nil { + return "", fmt.Errorf("invalid group response data") + } + if gcr, ok := dataObj["getGroupContentResult"].(map[string]interface{}); ok { + if pid, ok := gcr["parentCatalogID"].(string); ok && pid != "" { + return pid, nil + } + if cl, ok := gcr["catalogList"].([]interface{}); ok && len(cl) > 0 { + if first, ok := cl[0].(map[string]interface{}); ok { + if p, ok := first["path"].(string); ok && p != "" { + return p, nil + } + } + } + } + return "", fmt.Errorf("no root found in group response") +} + +// getFamilyRootPath 查询 family 的上层 path（data.path） +// 返回值已去除前缀 "root:/"（或 "root:"），直接返回纯 ID 或 path 部分，便于持久化为 RootFolderID。 +func (d *Yun139) getFamilyRootPath(cloudID string) (string, error) { + // 使用 v1.2 接口（代码日志中已有该请求），pageSize 取 1 足够获取 path 字段 + pathname := "/orchestration/familyCloud-rebuild/content/v1.2/queryContentList" + body := base.Json{ + "catalogID": "", + "catalogType": 3, + "cloudID": cloudID, + "cloudType": 1, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + "contentSortType": 0, + "pageInfo": base.Json{ + "pageNum": 1, + "pageSize": 1, + }, + "sortDirection": 1, + } + var resp base.Json + _, err := d.post(pathname, body, &resp) + if err != nil { + return "", err + } + dataObj, _ := resp["data"].(map[string]interface{}) + if dataObj == nil { + return "", fmt.Errorf("invalid family response data") + } + // helper to strip "root:/" or "root:" prefix + stripRoot := func(s string) string { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "root:/") + s = strings.TrimPrefix(s, "root:") + return s + } + if p, ok := dataObj["path"].(string); ok && p != "" { + return stripRoot(p), nil + } + // 回退：有时 path 在 cloudCatalogList.catalogList 中 + if cl, ok := dataObj["cloudCatalogList"].([]interface{}); ok && len(cl) > 0 { + if first, ok := cl[0].(map[string]interface{}); ok { + if p, ok := first["path"].(string); ok && p != "" { + return stripRoot(p), nil + } + } + } + return "", fmt.Errorf("no path found in family response") +} diff --git a/drivers/189/driver.go b/drivers/189/driver.go index c12c010c3..ffd5c82f6 100644 --- a/drivers/189/driver.go +++ b/drivers/189/driver.go @@ -202,7 +202,7 @@ func (d *Cloud189) GetDetails(ctx context.Context) (*model.StorageDetails, error return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: capacityInfo.CloudCapacityInfo.TotalSize, - FreeSpace: capacityInfo.CloudCapacityInfo.FreeSize, + UsedSpace: capacityInfo.CloudCapacityInfo.UsedSize, }, }, nil } diff --git a/drivers/189/types.go b/drivers/189/types.go index 4aee16e1d..178a3a5c6 100644 --- a/drivers/189/types.go +++ b/drivers/189/types.go @@ -72,15 +72,15 @@ type CapacityResp struct { ResMessage string `json:"res_message"` Account string `json:"account"` CloudCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - MailUsedSize uint64 `json:"mail189UsedSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + MailUsedSize int64 `json:"mail189UsedSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"cloudCapacityInfo"` FamilyCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } diff --git a/drivers/189/util.go b/drivers/189/util.go index 6e0682ea7..bb9a6adb4 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -107,7 +107,7 @@ import ( // res, err = d.client.R(). // SetHeaders(map[string]string{ // "lt": lt, -// "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", +// "User-Agent": base.UserAgentNT, // "Referer": "https://open.e.189.cn/", // "accept": "application/json;charset=UTF-8", // }).SetFormData(map[string]string{ diff --git a/drivers/189_tv/driver.go b/drivers/189_tv/driver.go index 6ee69c7e3..3b4aa51f6 100644 --- a/drivers/189_tv/driver.go +++ b/drivers/189_tv/driver.go @@ -284,18 +284,18 @@ func (y *Cloud189TV) GetDetails(ctx context.Context) (*model.StorageDetails, err if err != nil { return nil, err } - var total, free uint64 + var total, used int64 if y.isFamily() { total = capacityInfo.FamilyCapacityInfo.TotalSize - free = capacityInfo.FamilyCapacityInfo.FreeSize + used = capacityInfo.FamilyCapacityInfo.UsedSize } else { total = capacityInfo.CloudCapacityInfo.TotalSize - free = capacityInfo.CloudCapacityInfo.FreeSize + used = capacityInfo.CloudCapacityInfo.UsedSize } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: used, }, }, nil } diff --git a/drivers/189_tv/types.go b/drivers/189_tv/types.go index 686a090e8..b8243974b 100644 --- a/drivers/189_tv/types.go +++ b/drivers/189_tv/types.go @@ -322,15 +322,15 @@ type CapacityResp struct { ResMessage string `json:"res_message"` Account string `json:"account"` CloudCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - MailUsedSize uint64 `json:"mail189UsedSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + MailUsedSize int64 `json:"mail189UsedSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"cloudCapacityInfo"` FamilyCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } diff --git a/drivers/189_tv/utils.go b/drivers/189_tv/utils.go index 8678f826e..cea9411ec 100644 --- a/drivers/189_tv/utils.go +++ b/drivers/189_tv/utils.go @@ -70,6 +70,9 @@ func (y *Cloud189TV) request(url, method string, callback base.ReqCallback, para } func (y *Cloud189TV) requestWithRetry(url, method string, callback base.ReqCallback, params map[string]string, resp interface{}, retryCount int, isFamily ...bool) ([]byte, error) { + if y.tokenInfo == nil { + return nil, fmt.Errorf("login failed") + } req := y.client.R().SetQueryParams(clientSuffix()) if params != nil { diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 6861f7c23..6aa06c068 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -416,18 +416,18 @@ func (y *Cloud189PC) GetDetails(ctx context.Context) (*model.StorageDetails, err if err != nil { return nil, err } - var total, free uint64 + var total, used int64 if y.isFamily() { total = capacityInfo.FamilyCapacityInfo.TotalSize - free = capacityInfo.FamilyCapacityInfo.FreeSize + used = capacityInfo.FamilyCapacityInfo.UsedSize } else { total = capacityInfo.CloudCapacityInfo.TotalSize - free = capacityInfo.CloudCapacityInfo.FreeSize + used = capacityInfo.CloudCapacityInfo.UsedSize } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: used, }, }, nil } diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index fe219d8c3..32996debd 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -415,15 +415,15 @@ type CapacityResp struct { ResMessage string `json:"res_message"` Account string `json:"account"` CloudCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - MailUsedSize uint64 `json:"mail189UsedSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + MailUsedSize int64 `json:"mail189UsedSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"cloudCapacityInfo"` FamilyCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 346b8df2d..08ee658ca 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -90,6 +90,9 @@ func (y *Cloud189PC) EncryptParams(params Params, isFamily bool) string { } func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, params Params, resp interface{}, isFamily ...bool) ([]byte, error) { + if y.getTokenInfo() == nil { + return nil, fmt.Errorf("login failed") + } req := y.getClient().R().SetQueryParams(clientSuffix()) // 设置params @@ -753,30 +756,24 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo } partInfo := "" var reader io.ReadSeeker - var rateLimitedRd io.Reader threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { - if reader == nil { - var err error - reader, err = ss.GetSectionReader(offset, partSize) - if err != nil { - return err - } - silceMd5.Reset() - w, err := utils.CopyWithBuffer(writers, reader) - if w != partSize { - return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", partSize, w, err) - } - // 计算块md5并进行hex和base64编码 - md5Bytes := silceMd5.Sum(nil) - silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes))) - partInfo = fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) - - rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader) + Before: func(ctx context.Context) (err error) { + reader, err = ss.GetSectionReader(offset, partSize) + if err != nil { + return err } + silceMd5.Reset() + w, err := utils.CopyWithBuffer(writers, reader) + if w != partSize { + return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", partSize, w, err) + } + // 计算块md5并进行hex和base64编码 + md5Bytes := silceMd5.Sum(nil) + silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes))) + partInfo = fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) return nil }, - Do: func(ctx context.Context) error { + Do: func(ctx context.Context) (err error) { reader.Seek(0, io.SeekStart) uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, initMultiUpload.Data.UploadFileID, partInfo) if err != nil { @@ -785,11 +782,11 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo // step.4 上传切片 uploadUrl := uploadUrls[0] - _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, rateLimitedRd, isFamily) + _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, driver.NewLimitedUploadStream(ctx, reader), isFamily) if err != nil { return err } - up(float64(threadG.Success()) * 100 / float64(count)) + up(float64(threadG.Success()+1) * 100 / float64(count+1)) return nil }, After: func(err error) { @@ -801,6 +798,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo if err = threadG.Wait(); err != nil { return nil, err } + defer up(100) if fileMd5 != nil { fileMd5Hex = strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) @@ -992,7 +990,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode return err } - up(float64(threadG.Success()) * 100 / float64(len(uploadUrls))) + up(float64(threadG.Success()+1) * 100 / float64(len(uploadUrls)+1)) uploadProgress.UploadParts[i] = "" return nil }) @@ -1004,6 +1002,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } return nil, err } + defer up(100) } // step.5 提交 diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 7f20637fd..e1ba41eb6 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "math/rand" "net/url" stdpath "path" "strings" @@ -16,6 +17,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ) @@ -23,10 +25,9 @@ import ( type Alias struct { model.Storage Addition - rootOrder []string - pathMap map[string][]string - autoFlatten bool - oneKey string + rootOrder []string + pathMap map[string][]string + root model.Obj } func (d *Alias) Config() driver.Config { @@ -38,9 +39,6 @@ func (d *Alias) GetAddition() driver.Additional { } func (d *Alias) Init(ctx context.Context) error { - if d.Paths == "" { - return errors.New("paths is required") - } paths := strings.Split(d.Paths, "\n") d.rootOrder = make([]string, 0, len(paths)) d.pathMap = make(map[string][]string) @@ -50,19 +48,50 @@ func (d *Alias) Init(ctx context.Context) error { continue } k, v := getPair(path) - if _, ok := d.pathMap[k]; !ok { + temp, ok := d.pathMap[k] + if !ok { d.rootOrder = append(d.rootOrder, k) } - d.pathMap[k] = append(d.pathMap[k], v) + d.pathMap[k] = append(temp, v) } - if len(d.pathMap) == 1 { - for k := range d.pathMap { - d.oneKey = k + + switch len(d.rootOrder) { + case 0: + return errors.New("paths is required") + case 1: + paths := d.pathMap[d.rootOrder[0]] + roots := make(BalancedObjs, 0, len(paths)) + roots = append(roots, &model.Object{ + Name: "root", + Path: paths[0], + IsFolder: true, + Modified: d.Modified, + Mask: model.Locked, + }) + for _, path := range paths[1:] { + roots = append(roots, &model.Object{ + Path: path, + }) + } + d.root = roots + default: + d.root = &model.Object{ + Name: "root", + Path: "/", + IsFolder: true, + Modified: d.Modified, + Mask: model.ReadOnly, } - d.autoFlatten = true - } else { - d.oneKey = "" - d.autoFlatten = false + } + + if !utils.SliceContains(ValidReadConflictPolicy, d.ReadConflictPolicy) { + d.ReadConflictPolicy = FirstRWP + } + if !utils.SliceContains(ValidWriteConflictPolicy, d.WriteConflictPolicy) { + d.WriteConflictPolicy = DisabledWP + } + if !utils.SliceContains(ValidPutConflictPolicy, d.PutConflictPolicy) { + d.PutConflictPolicy = DisabledWP } return nil } @@ -70,308 +99,318 @@ func (d *Alias) Init(ctx context.Context) error { func (d *Alias) Drop(ctx context.Context) error { d.rootOrder = nil d.pathMap = nil + d.root = nil return nil } -func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { - if utils.PathEqual(path, "/") { - return &model.Object{ - Name: "Root", - IsFolder: true, - Path: "/", - }, nil +func (d *Alias) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root == nil { + return nil, errs.StorageNotInit } - root, sub := d.getRootAndPath(path) - dsts, ok := d.pathMap[root] - if !ok { + return d.root, nil +} + +// 通过op.Get调用的话，path一定是子路径(/开头) +func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { + roots, sub := d.getRootsAndPath(path) + if len(roots) == 0 { return nil, errs.ObjectNotFound } - var ret *model.Object - provider := "" - for _, dst := range dsts { - rawPath := stdpath.Join(dst, sub) + for idx, root := range roots { + rawPath := stdpath.Join(root, sub) obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) if err != nil { continue } - storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}) - if ret == nil { - ret = &model.Object{ - Path: path, - Name: obj.GetName(), - Size: obj.GetSize(), - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - HashInfo: obj.GetHash(), - } - if !d.ProviderPassThrough || err != nil { - break + mask := model.GetObjMask(obj) &^ model.Temp + if sub == "" { + // 根目录 + mask |= model.Locked | model.Virtual + } + ret := model.Object{ + Path: rawPath, + Name: obj.GetName(), + Size: obj.GetSize(), + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + Mask: mask, + } + obj = &ret + if d.ProviderPassThrough && !obj.IsDir() { + if storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}); err == nil { + obj = &model.ObjectProvider{ + Object: ret, + Provider: model.Provider{ + Provider: storage.Config().Name, + }, + } } - provider = storage.Config().Name - } else if err != nil || provider != storage.GetStorage().Driver { - provider = "" - break } + + roots = roots[idx+1:] + var objs BalancedObjs + if idx > 0 { + objs = make(BalancedObjs, 0, len(roots)+2) + } else { + objs = make(BalancedObjs, 0, len(roots)+1) + } + objs = append(objs, obj) + if idx > 0 { + objs = append(objs, nil) + } + for _, d := range roots { + objs = append(objs, &tempObj{model.Object{ + Path: stdpath.Join(d, sub), + }}) + } + return objs, nil } - if ret == nil { - return nil, errs.ObjectNotFound - } - if provider != "" { - return &model.ObjectProvider{ - Object: *ret, - Provider: model.Provider{ - Provider: provider, - }, - }, nil - } - return ret, nil + return nil, errs.ObjectNotFound } func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - path := dir.GetPath() - if utils.PathEqual(path, "/") && !d.autoFlatten { - return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough), nil - } - root, sub := d.getRootAndPath(path) - dsts, ok := d.pathMap[root] + dirs, ok := dir.(BalancedObjs) if !ok { - return nil, errs.ObjectNotFound + return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil } - var objs []model.Obj - for _, dst := range dsts { - tmp, err := fs.List(ctx, stdpath.Join(dst, sub), &fs.ListArgs{ + + // 因为alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 + // 所以这里对象不会传回到alias，也就不需要返回BalancedObjs了 + objMap := make(map[string]model.Obj) + for _, dir := range dirs { + if dir == nil { + continue + } + dirPath := dir.GetPath() + tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ NoLog: true, Refresh: args.Refresh, WithStorageDetails: args.WithStorageDetails && d.DetailsPassThrough, }) - if err == nil { - tmp, err = utils.SliceConvert(tmp, func(obj model.Obj) (model.Obj, error) { - thumb, ok := model.GetThumb(obj) - objRes := model.Object{ - Name: obj.GetName(), - Size: obj.GetSize(), - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - } - if !ok { - return &objRes, nil - } - return &model.ObjThumb{ + if err != nil { + continue + } + for _, obj := range tmp { + name := obj.GetName() + if _, exists := objMap[name]; exists { + continue + } + mask := model.GetObjMask(obj) &^ model.Temp + objRes := model.Object{ + Name: name, + Path: stdpath.Join(dirPath, name), + Size: obj.GetSize(), + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + Mask: mask, + } + var objRet model.Obj + if thumb, ok := model.GetThumb(obj); ok { + objRet = &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, - }, nil - }) + } + } else { + objRet = &objRes + } + if details, ok := model.GetStorageDetails(obj); ok { + objRet = &model.ObjStorageDetails{ + Obj: objRet, + StorageDetails: details, + } + } + objMap[name] = objRet } - if err == nil { - objs = append(objs, tmp...) + } + objs := make([]model.Obj, 0, len(objMap)) + for _, obj := range objMap { + objs = append(objs, obj) + } + if d.OrderBy == "" { + sort := getAllSort(dirs) + if sort.OrderBy != "" { + model.SortFiles(objs, sort.OrderBy, sort.OrderDirection) + } + if d.ExtractFolder == "" && sort.ExtractFolder != "" { + model.ExtractFolder(objs, sort.ExtractFolder) } } return objs, nil } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - root, sub := d.getRootAndPath(file.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound - } - // proxy || ftp,s3 - if common.GetApiUrl(ctx) == "" { - args.Redirect = false - } - for _, dst := range dsts { - reqPath := stdpath.Join(dst, sub) - link, fi, err := d.link(ctx, reqPath, args) + if d.ReadConflictPolicy == AllRWP && !args.Redirect { + files, err := d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) if err != nil { - continue + return nil, err } - if link == nil { - // 重定向且需要通过代理 - return &model.Link{ - URL: fmt.Sprintf("%s/p%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)), - }, nil + linkClosers := make([]io.Closer, 0, len(files)) + rrf := make([]model.RangeReaderIF, 0, len(files)) + for _, f := range files { + link, fi, err := d.link(ctx, f.GetPath(), args) + if err != nil { + continue + } + if fi.GetSize() != files.GetSize() { + _ = link.Close() + continue + } + l := *link // 复制一份，避免修改到原始link + if l.ContentLength == 0 { + l.ContentLength = fi.GetSize() + } + if d.DownloadConcurrency > 0 { + l.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + l.PartSize = d.DownloadPartSize * utils.KB + } + rr, err := stream.GetRangeReaderFromLink(l.ContentLength, &l) + if err != nil { + _ = link.Close() + continue + } + linkClosers = append(linkClosers, link) + rrf = append(rrf, rr) } - - resultLink := *link - resultLink.SyncClosers = utils.NewSyncClosers(link) - if args.Redirect { - return &resultLink, nil + rr := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return rrf[rand.Intn(len(rrf))].RangeRead(ctx, httpRange) } + return &model.Link{ + RangeReader: stream.RangeReaderFunc(rr), + SyncClosers: utils.NewSyncClosers(linkClosers...), + }, nil + } - if resultLink.ContentLength == 0 { - resultLink.ContentLength = fi.GetSize() - } - if resultLink.MFile != nil { - return &resultLink, nil - } - if d.DownloadConcurrency > 0 { - resultLink.Concurrency = d.DownloadConcurrency + var link *model.Link + var fi model.Obj + var err error + files := file.(BalancedObjs) + if d.ReadConflictPolicy == RandomBalancedRP || d.ReadConflictPolicy == AllRWP { + rand.Shuffle(len(files), func(i, j int) { + files[i], files[j] = files[j], files[i] + }) + } + for _, f := range files { + if f == nil { + continue } - if d.DownloadPartSize > 0 { - resultLink.PartSize = d.DownloadPartSize * utils.KB + link, fi, err = d.link(ctx, f.GetPath(), args) + if err == nil { + if link == nil { + // 重定向且需要通过代理 + return &model.Link{ + URL: fmt.Sprintf("%s/p%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(f.GetPath(), true), + sign.Sign(f.GetPath())), + }, nil + } + break } + } + if err != nil { + return nil, err + } + resultLink := *link // 复制一份，避免修改到原始link + resultLink.Expiration = nil + resultLink.SyncClosers = utils.NewSyncClosers(link) + if args.Redirect { return &resultLink, nil } - return nil, errs.ObjectNotFound + if resultLink.ContentLength == 0 { + resultLink.ContentLength = fi.GetSize() + } + if d.DownloadConcurrency > 0 { + resultLink.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + resultLink.PartSize = d.DownloadPartSize * utils.KB + } + return &resultLink, nil } func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { - root, sub := d.getRootAndPath(args.Obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound - } - for _, dst := range dsts { - rawPath := stdpath.Join(dst, sub) - storage, actualPath, err := op.GetStorageAndActualPath(rawPath) - if err != nil { - continue - } - other, ok := storage.(driver.Other) - if !ok { - continue - } - obj, err := op.GetUnwrap(ctx, storage, actualPath) - if err != nil { - continue - } - return other.Other(ctx, model.OtherArgs{ - Obj: obj, - Method: args.Method, - Data: args.Data, - }) + // Other 不应负载均衡，这是因为前端是否调用 /fs/other 的判断条件是返回的 provider 的值 + // 而 ProviderPassThrough 开启时，返回的 provider 固定为第一个 obj 的后端驱动 + storage, actualPath, err := op.GetStorageAndActualPath(args.Obj.GetPath()) + if err != nil { + return nil, err } - return nil, errs.NotImplement + return op.Other(ctx, storage, model.FsOtherArgs{ + Path: actualPath, + Method: args.Method, + Data: args.Data, + }) } func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, parentDir, true) + objs, err := d.getWriteObjs(ctx, parentDir) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(*path, dirName))) + for _, obj := range objs { + err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(obj.GetPath(), dirName))) } - return err - } - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot make sub-dir") } return err } func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - if !d.Writable { - return errs.PermissionDenied - } - srcPath, err := d.getReqPath(ctx, srcObj, false) - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be moved") - } - if err != nil { - return err - } - dstPath, err := d.getReqPath(ctx, dstDir, true) - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be moved to") - } - if err != nil { - return err - } - if len(srcPath) == len(dstPath) { - for i := range srcPath { - _, e := fs.Move(ctx, *srcPath[i], *dstPath[i]) + srcs, dsts, err := d.getMoveObjs(ctx, srcObj, dstDir) + if err == nil { + for i, dst := range dsts { + src := srcs[i] + _, e := fs.Move(ctx, src.GetPath(), dst.GetPath()) + err = errors.Join(err, e) + } + srcs = srcs[len(dsts):] + for _, src := range srcs { + e := fs.Remove(ctx, src.GetPath()) err = errors.Join(err, e) } - return err - } else { - return errors.New("parallel paths mismatch") } + return err } func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, srcObj, false) + objs, err := d.getWriteObjs(ctx, srcObj) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.Rename(ctx, *path, newName)) + for _, obj := range objs { + err = errors.Join(err, fs.Rename(ctx, obj.GetPath(), newName)) } - return err - } - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be Rename") } return err } func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - if !d.Writable { - return errs.PermissionDenied - } - srcPath, err := d.getReqPath(ctx, srcObj, false) - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be copied") - } - if err != nil { - return err - } - dstPath, err := d.getReqPath(ctx, dstDir, true) - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be copied to") - } - if err != nil { - return err - } - if len(srcPath) == len(dstPath) { - for i := range srcPath { - _, e := fs.Copy(ctx, *srcPath[i], *dstPath[i]) - err = errors.Join(err, e) - } - return err - } else if len(srcPath) == 1 || !d.ProtectSameName { - for _, path := range dstPath { - _, e := fs.Copy(ctx, *srcPath[0], *path) + srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) + if err == nil { + for i, src := range srcs { + dst := dsts[i] + _, e := fs.Copy(ctx, src.GetPath(), dst.GetPath()) err = errors.Join(err, e) } - return err - } else { - return errors.New("parallel paths mismatch") } + return err } func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, obj, false) + objs, err := d.getWriteObjs(ctx, obj) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.Remove(ctx, *path)) + for _, obj := range objs { + err = errors.Join(err, fs.Remove(ctx, obj.GetPath())) } - return err - } - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be Delete") } return err } func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, dstDir, true) + objs, err := d.getPutObjs(ctx, dstDir) if err == nil { - if len(reqPath) == 1 { - storage, reqActualPath, err := op.GetStorageAndActualPath(*reqPath[0]) + if len(objs) == 1 { + storage, reqActualPath, err := op.GetStorageAndActualPath(objs.GetPath()) if err != nil { return err } @@ -385,10 +424,10 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, if err != nil { return err } - count := float64(len(reqPath) + 1) + count := float64(len(objs) + 1) up(100 / count) - for i, path := range reqPath { - err = errors.Join(err, fs.PutDirectly(ctx, *path, &stream.FileStream{ + for i, obj := range objs { + err = errors.Join(err, fs.PutDirectly(ctx, obj.GetPath(), &stream.FileStream{ Obj: s, Mimetype: s.GetMimetype(), Reader: file, @@ -402,55 +441,40 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, return err } } - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be Put") - } return err } func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, dstDir, true) + objs, err := d.getPutObjs(ctx, dstDir) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.PutURL(ctx, *path, name, url)) + for _, obj := range objs { + err = errors.Join(err, fs.PutURL(ctx, obj.GetPath(), name, url)) } return err } - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot offline download") - } return err } func (d *Alias) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { - root, sub := d.getRootAndPath(obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := d.getBalancedPath(ctx, obj) + if reqPath == "" { + return nil, errs.NotFile } - for _, dst := range dsts { - meta, err := d.getArchiveMeta(ctx, dst, sub, args) - if err == nil { - return meta, nil - } + meta, err := d.getArchiveMeta(ctx, reqPath, args) + if err == nil { + return meta, nil } return nil, errs.NotImplement } func (d *Alias) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { - root, sub := d.getRootAndPath(obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := d.getBalancedPath(ctx, obj) + if reqPath == "" { + return nil, errs.NotFile } - for _, dst := range dsts { - l, err := d.listArchive(ctx, dst, sub, args) - if err == nil { - return l, nil - } + l, err := d.listArchive(ctx, reqPath, args) + if err == nil { + return l, nil } return nil, errs.NotImplement } @@ -459,67 +483,88 @@ func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveIn // alias的两个驱动，一个支持驱动提取，一个不支持，如何兼容？ // 如果访问的是不支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回errs.NotImplement，提取URL前缀就会是/ae，Extract就不会被调用 // 如果访问的是支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回有效值，提取URL前缀就会是/ad，Extract就会被调用 - root, sub := d.getRootAndPath(obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := d.getBalancedPath(ctx, obj) + if reqPath == "" { + return nil, errs.NotFile } - for _, dst := range dsts { - reqPath := stdpath.Join(dst, sub) - link, err := d.extract(ctx, reqPath, args) - if err != nil { - continue - } - if link == nil { - return &model.Link{ - URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - utils.EncodePath(args.InnerPath, true), - url.QueryEscape(args.Password), - sign.SignArchive(reqPath)), - }, nil - } - resultLink := *link - resultLink.SyncClosers = utils.NewSyncClosers(link) - return &resultLink, nil + link, err := d.extract(ctx, reqPath, args) + if err != nil { + return nil, errs.NotImplement + } + if link == nil { + return &model.Link{ + URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(reqPath, true), + utils.EncodePath(args.InnerPath, true), + url.QueryEscape(args.Password), + sign.SignArchive(reqPath)), + }, nil } - return nil, errs.NotImplement + resultLink := *link + resultLink.SyncClosers = utils.NewSyncClosers(link) + return &resultLink, nil } func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { - if !d.Writable { - return errs.PermissionDenied + srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) + if err == nil { + for i, src := range srcs { + dst := dsts[i] + _, e := fs.ArchiveDecompress(ctx, src.GetPath(), dst.GetPath(), args) + err = errors.Join(err, e) + } } - srcPath, err := d.getReqPath(ctx, srcObj, false) - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be decompressed") + return err +} + +func (d *Alias) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + if !d.DetailsPassThrough { + return nil, errs.NotImplement } - if err != nil { - return err + if len(d.rootOrder) != 1 { + return nil, errs.NotImplement } - dstPath, err := d.getReqPath(ctx, dstDir, true) - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be decompressed to") + backends := d.pathMap[d.rootOrder[0]] + var storage driver.Driver + for _, backend := range backends { + s, err := fs.GetStorage(backend, &fs.GetStoragesArgs{}) + if err != nil { + return nil, errs.NotImplement + } + if storage == nil { + storage = s + } else if storage.GetStorage().MountPath != s.GetStorage().MountPath { + return nil, errs.NotImplement + } } - if err != nil { - return err + if storage == nil { // should never access + return nil, errs.NotImplement } - if len(srcPath) == len(dstPath) { - for i := range srcPath { - _, e := fs.ArchiveDecompress(ctx, *srcPath[i], *dstPath[i], args) - err = errors.Join(err, e) + return op.GetStorageDetails(ctx, storage) +} + +func (d *Alias) ResolveLinkCacheMode(path string) driver.LinkCacheMode { + roots, sub := d.getRootsAndPath(path) + if len(roots) == 0 { + return 0 + } + for _, root := range roots { + storage, actualPath, err := op.GetStorageAndActualPath(stdpath.Join(root, sub)) + if err != nil { + continue } - return err - } else if len(srcPath) == 1 || !d.ProtectSameName { - for _, path := range dstPath { - _, e := fs.ArchiveDecompress(ctx, *srcPath[0], *path, args) - err = errors.Join(err, e) + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + continue + } + mode := storage.Config().LinkCacheMode + if mode == -1 { + return storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(actualPath) + } else { + return mode } - return err - } else { - return errors.New("parallel paths mismatch") } + return 0 } var _ driver.Driver = (*Alias)(nil) diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index bdab53748..72eb3c877 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -6,17 +6,15 @@ import ( ) type Addition struct { - // Usually one of two - // driver.RootPath - // define other - Paths string `json:"paths" required:"true" type:"text"` - ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"` - ParallelWrite bool `json:"parallel_write" type:"bool" default:"false"` - DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` - DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` - Writable bool `json:"writable" type:"bool" default:"false"` - ProviderPassThrough bool `json:"provider_pass_through" type:"bool" default:"false"` - DetailsPassThrough bool `json:"details_pass_through" type:"bool" default:"false"` + Paths string `json:"paths" required:"true" type:"text"` + ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random,all" default:"first"` + WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled" help:"How the driver handles identical backend paths when renaming, removing, or making directories."` + PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled" help:"How the driver handles identical backend paths when uploading, copying, moving, or decompressing."` + FileConsistencyCheck bool `json:"file_consistency_check" type:"bool" default:"false"` + DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` + DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` + ProviderPassThrough bool `json:"provider_pass_through" type:"bool" default:"false"` + DetailsPassThrough bool `json:"details_pass_through" type:"bool" default:"false"` } var config = driver.Config{ @@ -26,14 +24,11 @@ var config = driver.Config{ NoUpload: false, DefaultRoot: "/", ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheAuto, } func init() { op.RegisterDriver(func() driver.Driver { - return &Alias{ - Addition: Addition{ - ProtectSameName: true, - }, - } + return &Alias{} }) } diff --git a/drivers/alias/types.go b/drivers/alias/types.go index e560393da..9fade0c5b 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -1 +1,78 @@ package alias + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/pkg/errors" +) + +const ( + DisabledWP = "disabled" + FirstRWP = "first" + DeterministicWP = "deterministic" + DeterministicOrAllWP = "deterministic_or_all" + AllRWP = "all" + AllStrictWP = "all_strict" + RandomBalancedRP = "random" + BalancedByQuotaP = "quota" + BalancedByQuotaStrictP = "quota_strict" +) + +var ( + ValidReadConflictPolicy = []string{FirstRWP, RandomBalancedRP, AllRWP} + ValidWriteConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllRWP, + AllStrictWP} + ValidPutConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllRWP, + AllStrictWP, RandomBalancedRP, BalancedByQuotaP, BalancedByQuotaStrictP} +) + +var ( + ErrPathConflict = errors.New("path conflict") + ErrSamePathLeak = errors.New("leak some of same-name dirs") + ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") + ErrNotEnoughSrcObjs = errors.New("cannot move fewer objs to more paths, please try copying") +) + +type BalancedObjs []model.Obj + +func (b BalancedObjs) GetSize() int64 { + return b[0].GetSize() +} + +func (b BalancedObjs) ModTime() time.Time { + return b[0].ModTime() +} + +func (b BalancedObjs) CreateTime() time.Time { + return b[0].CreateTime() +} + +func (b BalancedObjs) IsDir() bool { + return b[0].IsDir() +} + +func (b BalancedObjs) GetHash() utils.HashInfo { + return b[0].GetHash() +} + +func (b BalancedObjs) GetName() string { + return b[0].GetName() +} + +func (b BalancedObjs) GetPath() string { + return b[0].GetPath() +} + +func (b BalancedObjs) GetID() string { + return b[0].GetID() +} + +func (b BalancedObjs) Unwrap() model.Obj { + return b[0] +} + +var _ model.Obj = (BalancedObjs)(nil) + +type tempObj struct{ model.Object } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 33960c944..8e5eb8a84 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -2,10 +2,10 @@ package alias import ( "context" - "errors" + "math/rand" stdpath "path" "strings" - "sync" + "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -13,20 +13,29 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) -func (d *Alias) listRoot(ctx context.Context, withDetails bool) []model.Obj { +type detailWithIndex struct { + idx int + val *model.StorageDetails +} + +func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model.Obj { var objs []model.Obj - var wg sync.WaitGroup + detailsChan := make(chan detailWithIndex, len(d.pathMap)) + workerCount := 0 for _, k := range d.rootOrder { - obj := model.Object{ + obj := &model.Object{ Name: k, + Path: "/" + k, IsFolder: true, Modified: d.Modified, + Mask: model.Locked | model.Virtual, } idx := len(objs) - objs = append(objs, &obj) + objs = append(objs, obj) v := d.pathMap[k] if !withDetails || len(v) != 1 { continue @@ -35,56 +44,56 @@ func (d *Alias) listRoot(ctx context.Context, withDetails bool) []model.Obj { if err != nil { continue } + obj.Modified = remoteDriver.GetStorage().Modified _, ok := remoteDriver.(driver.WithDetails) if !ok { continue } objs[idx] = &model.ObjStorageDetails{ - Obj: objs[idx], - StorageDetailsWithName: model.StorageDetailsWithName{ - StorageDetails: nil, - DriverName: remoteDriver.Config().Name, - }, - } - wg.Add(1) - go func() { - defer wg.Done() - details, e := op.GetStorageDetails(ctx, remoteDriver) + Obj: objs[idx], + StorageDetails: nil, + } + workerCount++ + go func(dri driver.Driver, i int) { + details, e := op.GetStorageDetails(ctx, dri, refresh) if e != nil { - if !errors.Is(e, errs.NotImplement) { - log.Errorf("failed get %s storage details: %+v", remoteDriver.GetStorage().MountPath, e) + if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { + log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e) } - return } - objs[idx].(*model.ObjStorageDetails).StorageDetails = details - }() + detailsChan <- detailWithIndex{idx: i, val: details} + }(remoteDriver, idx) + } + for workerCount > 0 { + select { + case r := <-detailsChan: + objs[r.idx].(*model.ObjStorageDetails).StorageDetails = r.val + workerCount-- + case <-time.After(time.Second): + workerCount = 0 + } } - wg.Wait() return objs } // do others that not defined in Driver interface func getPair(path string) (string, string) { - // path = strings.TrimSpace(path) - if strings.Contains(path, ":") { - pair := strings.SplitN(path, ":", 2) - if !strings.Contains(pair[0], "/") { - return pair[0], pair[1] - } + if name, path, ok := strings.Cut(path, ":"); ok && !strings.Contains(name, "/") { + return name, path } return stdpath.Base(path), path } -func (d *Alias) getRootAndPath(path string) (string, string) { - if d.autoFlatten { - return d.oneKey, path +func (d *Alias) getRootsAndPath(path string) (roots []string, sub string) { + if len(d.rootOrder) == 1 { + return d.pathMap[d.rootOrder[0]], path } path = strings.TrimPrefix(path, "/") - parts := strings.SplitN(path, "/", 2) - if len(parts) == 1 { - return parts[0], "" + before, after, ok := strings.Cut(path, "/") + if !ok { + return d.pathMap[path], "" } - return parts[0], parts[1] + return d.pathMap[before], after } func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { @@ -92,59 +101,350 @@ func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) ( if err != nil { return nil, nil, err } - if !args.Redirect { - return op.Link(ctx, storage, reqActualPath, args) + if args.Redirect && common.ShouldProxy(storage, stdpath.Base(reqPath)) { + return nil, nil, nil } - obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) - if err != nil { - return nil, nil, err + return op.Link(ctx, storage, reqActualPath, args) +} + +func isConsistent(a, b model.Obj) bool { + if a.GetSize() != b.GetSize() { + return false } - if common.ShouldProxy(storage, stdpath.Base(reqPath)) { - return nil, obj, nil + for ht, v := range a.GetHash().All() { + ah := b.GetHash().GetHash(ht) + if ah != "" && ah != v { + return false + } } - return op.Link(ctx, storage, reqActualPath, args) + return true } -func (d *Alias) getReqPath(ctx context.Context, obj model.Obj, isParent bool) ([]*string, error) { - root, sub := d.getRootAndPath(obj.GetPath()) - if sub == "" && !isParent { - return nil, errs.NotSupport +func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { + objs := bObj.(BalancedObjs) + length := 0 + for _, o := range objs { + var err error + var obj model.Obj + temp, isTemp := o.(*tempObj) + if isTemp { + obj, err = fs.Get(ctx, o.GetPath(), &fs.GetArgs{NoLog: true}) + if err == nil { + if !bObj.IsDir() { + if obj.IsDir() { + err = errs.NotFile + } else if d.FileConsistencyCheck && !isConsistent(bObj, obj) { + err = errs.ObjectNotFound + } + } else if !obj.IsDir() { + err = errs.NotFolder + } + } + } else if o == nil { + err = errs.ObjectNotFound + } + + cont, err := ifContinue(err) + if err != nil { + if cont { + continue + } + return nil, err + } + if isTemp { + objRes := temp.Object + // objRes.Name = obj.GetName() + // objRes.Size = obj.GetSize() + // objRes.Modified = obj.ModTime() + // objRes.HashInfo = obj.GetHash() + objs[length] = &objRes + } else { + objs[length] = o + } + length++ + if !cont { + break + } } - dsts, ok := d.pathMap[root] - all := true - if !ok { + if length == 0 { return nil, errs.ObjectNotFound } - var reqPath []*string - for _, dst := range dsts { - path := stdpath.Join(dst, sub) - _, err := fs.Get(ctx, path, &fs.GetArgs{NoLog: true}) + return objs[:length], nil +} + +func (d *Alias) getBalancedPath(ctx context.Context, file model.Obj) string { + if d.ReadConflictPolicy == FirstRWP { + return file.GetPath() + } + files := file.(BalancedObjs) + if rand.Intn(len(files)) == 0 { + return file.GetPath() + } + files, _ = d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) + return files[rand.Intn(len(files))].GetPath() +} + +func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { + if policy == AllRWP { + return func(err error) (bool, error) { + return true, err + } + } + all := true + l := 0 + return func(err error) (bool, error) { if err != nil { + switch policy { + case AllStrictWP: + return false, ErrSamePathLeak + case DeterministicOrAllWP: + if l >= 2 { + return false, ErrSamePathLeak + } + } all = false - if d.ProtectSameName && d.ParallelWrite && len(reqPath) >= 2 { - return nil, errs.NotImplement + } else { + switch policy { + case FirstRWP: + return false, nil + case DeterministicWP: + if l > 0 { + return false, ErrPathConflict + } + case DeterministicOrAllWP: + if l > 0 && !all { + return false, ErrSamePathLeak + } } + l += 1 + } + return true, err + } +} + +func (d *Alias) getWriteObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { + if d.WriteConflictPolicy == DisabledWP { + return nil, errs.PermissionDenied + } + return d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) +} + +func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { + if d.PutConflictPolicy == DisabledWP { + return nil, errs.PermissionDenied + } + objs, err := d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + if err != nil { + return nil, err + } + strict := false + switch d.PutConflictPolicy { + case RandomBalancedRP: + ri := rand.Intn(len(objs)) + return objs[ri : ri+1], nil + case BalancedByQuotaStrictP: + strict = true + fallthrough + case BalancedByQuotaP: + objs, ok := getRandomObjByQuotaBalanced(ctx, objs, strict, obj.GetSize()) + if !ok { + return nil, ErrNoEnoughSpace + } + return objs, nil + default: + return objs, nil + } +} + +func getRandomObjByQuotaBalanced(ctx context.Context, reqPath BalancedObjs, strict bool, objSize int64) (BalancedObjs, bool) { + // Get all space + details := make([]*model.StorageDetails, len(reqPath)) + detailsChan := make(chan detailWithIndex, len(reqPath)) + workerCount := 0 + for i, p := range reqPath { + s, err := fs.GetStorage(p.GetPath(), &fs.GetStoragesArgs{}) + if err != nil { continue } - if !d.ProtectSameName && !d.ParallelWrite { - return []*string{&path}, nil + if _, ok := s.(driver.WithDetails); !ok { + continue } - reqPath = append(reqPath, &path) - if d.ProtectSameName && !d.ParallelWrite && len(reqPath) >= 2 { - return nil, errs.NotImplement + workerCount++ + go func(dri driver.Driver, i int) { + d, e := op.GetStorageDetails(ctx, dri) + if e != nil { + if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { + log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e) + } + } + detailsChan <- detailWithIndex{idx: i, val: d} + }(s, i) + } + for workerCount > 0 { + select { + case r := <-detailsChan: + details[r.idx] = r.val + workerCount-- + case <-time.After(time.Second): + workerCount = 0 } - if d.ProtectSameName && d.ParallelWrite && len(reqPath) >= 2 && !all { - return nil, errs.NotImplement + } + + // Try select one that has space info + selected, ok := selectRandom(details, func(d *model.StorageDetails) uint64 { + if d == nil || d.FreeSpace() < objSize { + return 0 + } + return uint64(d.FreeSpace()) + }) + if !ok { + if strict { + return nil, false + } else { + // No strict mode, return any of non-details ones + noDetails := make([]int, 0, len(details)) + for i, d := range details { + if d == nil { + noDetails = append(noDetails, i) + } + } + if len(noDetails) == 0 { + return nil, false + } + selected = noDetails[rand.Intn(len(noDetails))] } } - if len(reqPath) == 0 { - return nil, errs.ObjectNotFound + return reqPath[selected : selected+1], true +} + +func selectRandom[Item any](arr []Item, getWeight func(Item) uint64) (int, bool) { + var totalWeight uint64 = 0 + for _, i := range arr { + totalWeight += getWeight(i) + } + if totalWeight == 0 { + return 0, false + } + r := rand.Uint64() % totalWeight + for i, item := range arr { + w := getWeight(item) + if r < w { + return i, true + } + r -= w + } + return 0, false +} + +func (d *Alias) getCopyObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { + if d.PutConflictPolicy == DisabledWP { + return nil, nil, errs.PermissionDenied + } + dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + if err != nil { + return nil, nil, err + } + dstStorageMap := make(map[string][]model.Obj) + allocatingDst := make(map[model.Obj]struct{}) + for _, o := range dstObjs { + storage, e := fs.GetStorage(o.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + return nil, nil, errors.WithMessagef(e, "cannot copy to virtual path [%s]", o.GetPath()) + } + mp := storage.GetStorage().MountPath + dstStorageMap[mp] = append(dstStorageMap[mp], o) + allocatingDst[o] = struct{}{} + } + tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllRWP)) + if err != nil { + return nil, nil, err + } + srcObjs := make(BalancedObjs, 0, len(dstObjs)) + for _, src := range tmpSrcObjs { + storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + continue + } + mp := storage.GetStorage().MountPath + if tmp, ok := dstStorageMap[mp]; ok { + for _, dst := range tmp { + dstObjs[len(srcObjs)] = dst + srcObjs = append(srcObjs, src) + delete(allocatingDst, dst) + } + delete(dstStorageMap, mp) + } + } + dstObjs = dstObjs[:len(srcObjs)] + for dst := range allocatingDst { + src := tmpSrcObjs[0] + if d.ReadConflictPolicy == RandomBalancedRP || d.ReadConflictPolicy == AllRWP { + src = tmpSrcObjs[rand.Intn(len(tmpSrcObjs))] + } + srcObjs = append(srcObjs, src) + dstObjs = append(dstObjs, dst) + } + return srcObjs, dstObjs, nil +} + +func (d *Alias) getMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { + if d.PutConflictPolicy == DisabledWP { + return nil, nil, errs.PermissionDenied + } + dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + if err != nil { + return nil, nil, err + } + tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllRWP)) + if err != nil { + return nil, nil, err + } + if len(tmpSrcObjs) < len(dstObjs) { + return nil, nil, ErrNotEnoughSrcObjs + } + dstStorageMap := make(map[string][]model.Obj) + allocatingDst := make(map[model.Obj]struct{}) + for _, o := range dstObjs { + storage, e := fs.GetStorage(o.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + return nil, nil, errors.WithMessagef(e, "cannot move to virtual path [%s]", o.GetPath()) + } + mp := storage.GetStorage().MountPath + dstStorageMap[mp] = append(dstStorageMap[mp], o) + allocatingDst[o] = struct{}{} + } + srcObjs := make(BalancedObjs, 0, len(tmpSrcObjs)) + restSrcObjs := make(BalancedObjs, 0, len(tmpSrcObjs)-len(dstObjs)) + for _, src := range tmpSrcObjs { + storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + continue + } + mp := storage.GetStorage().MountPath + if tmp, ok := dstStorageMap[mp]; ok { + dst := tmp[0] + if len(tmp) == 1 { + delete(dstStorageMap, mp) + } else { + dstStorageMap[mp] = tmp[1:] + } + dstObjs[len(srcObjs)] = dst + srcObjs = append(srcObjs, src) + delete(allocatingDst, dst) + } else { + restSrcObjs = append(restSrcObjs, src) + } } - return reqPath, nil + dstObjs = dstObjs[:len(srcObjs)] + // len(restSrcObjs) >= len(allocatingDst) + srcObjs = append(srcObjs, restSrcObjs...) + for dst := range allocatingDst { + dstObjs = append(dstObjs, dst) + } + return srcObjs, dstObjs, nil } -func (d *Alias) getArchiveMeta(ctx context.Context, dst, sub string, args model.ArchiveArgs) (model.ArchiveMeta, error) { - reqPath := stdpath.Join(dst, sub) +func (d *Alias) getArchiveMeta(ctx context.Context, reqPath string, args model.ArchiveArgs) (model.ArchiveMeta, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err @@ -158,8 +458,7 @@ func (d *Alias) getArchiveMeta(ctx context.Context, dst, sub string, args model. return nil, errs.NotImplement } -func (d *Alias) listArchive(ctx context.Context, dst, sub string, args model.ArchiveInnerArgs) ([]model.Obj, error) { - reqPath := stdpath.Join(dst, sub) +func (d *Alias) listArchive(ctx context.Context, reqPath string, args model.ArchiveInnerArgs) ([]model.Obj, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err @@ -191,3 +490,43 @@ func (d *Alias) extract(ctx context.Context, reqPath string, args model.ArchiveI link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) return link, err } + +func getAllSort(dirs []model.Obj) model.Sort { + ret := model.Sort{} + noSort := false + noExtractFolder := false + for _, dir := range dirs { + if dir == nil { + continue + } + storage, err := fs.GetStorage(dir.GetPath(), &fs.GetStoragesArgs{}) + if err != nil { + continue + } + if !noSort && storage.GetStorage().OrderBy != "" { + if ret.OrderBy == "" { + ret.OrderBy = storage.GetStorage().OrderBy + ret.OrderDirection = storage.GetStorage().OrderDirection + if ret.OrderDirection == "" { + ret.OrderDirection = "asc" + } + } else if ret.OrderBy != storage.GetStorage().OrderBy || ret.OrderDirection != storage.GetStorage().OrderDirection { + ret.OrderBy = "" + ret.OrderDirection = "" + noSort = true + } + } + if !noExtractFolder && storage.GetStorage().ExtractFolder != "" { + if ret.ExtractFolder == "" { + ret.ExtractFolder = storage.GetStorage().ExtractFolder + } else if ret.ExtractFolder != storage.GetStorage().ExtractFolder { + ret.ExtractFolder = "" + noExtractFolder = true + } + } + if noSort && noExtractFolder { + break + } + } + return ret +} diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go new file mode 100644 index 000000000..a9fc44fcb --- /dev/null +++ b/drivers/alist_v3/driver.go @@ -0,0 +1,379 @@ +package alist_v3 + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +type AListV3 struct { + model.Storage + Addition +} + +func (d *AListV3) Config() driver.Config { + return config +} + +func (d *AListV3) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *AListV3) Init(ctx context.Context) error { + d.Addition.Address = strings.TrimSuffix(d.Addition.Address, "/") + var resp common.Resp[MeResp] + _, _, err := d.request("/me", http.MethodGet, func(req *resty.Request) { + req.SetResult(&resp) + }) + if err != nil { + return err + } + // if the username is not empty and the username is not the same as the current username, then login again + if d.Username != resp.Data.Username { + err = d.login() + if err != nil { + return err + } + } + // re-get the user info + _, _, err = d.request("/me", http.MethodGet, func(req *resty.Request) { + req.SetResult(&resp) + }) + if err != nil { + return err + } + if utils.SliceContains(resp.Data.Role, model.GUEST) { + u := d.Address + "/api/public/settings" + res, err := base.RestyClient.R().Get(u) + if err != nil { + return err + } + allowMounted := utils.Json.Get(res.Body(), "data", conf.AllowMounted).ToString() == "true" + if !allowMounted { + return fmt.Errorf("the site does not allow mounted") + } + } + return err +} + +func (d *AListV3) Drop(ctx context.Context) error { + return nil +} + +func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var resp common.Resp[FsListResp] + _, _, err := d.request("/fs/list", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ListReq{ + PageReq: model.PageReq{ + Page: 1, + PerPage: 0, + }, + Path: dir.GetPath(), + Password: d.MetaPassword, + Refresh: false, + }) + }) + if err != nil { + return nil, err + } + var files []model.Obj + for _, f := range resp.Data.Content { + file := model.ObjThumb{ + Object: model.Object{ + Name: f.Name, + Modified: f.Modified, + Ctime: f.Created, + Size: f.Size, + IsFolder: f.IsDir, + HashInfo: utils.FromString(f.HashInfo), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, + } + files = append(files, &file) + } + return files, nil +} + +func (d *AListV3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var resp common.Resp[FsGetResp] + headers := map[string]string{ + "User-Agent": base.UserAgent, + } + // if PassUAToUpsteam is true, then pass the user-agent to the upstream + if d.PassUAToUpsteam { + userAgent := args.Header.Get("user-agent") + if userAgent != "" { + headers["User-Agent"] = userAgent + } + } + // if PassIPToUpsteam is true, then pass the ip address to the upstream + if d.PassIPToUpsteam { + ip := args.IP + if ip != "" { + headers["X-Forwarded-For"] = ip + headers["X-Real-Ip"] = ip + } + } + _, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(FsGetReq{ + Path: file.GetPath(), + Password: d.MetaPassword, + }).SetHeaders(headers) + }) + if err != nil { + return nil, err + } + return &model.Link{ + URL: resp.Data.RawURL, + }, nil +} + +func (d *AListV3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + _, _, err := d.request("/fs/mkdir", http.MethodPost, func(req *resty.Request) { + req.SetBody(MkdirOrLinkReq{ + Path: path.Join(parentDir.GetPath(), dirName), + }) + }) + return err +} + +func (d *AListV3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + _, _, err := d.request("/fs/move", http.MethodPost, func(req *resty.Request) { + req.SetBody(MoveCopyReq{ + SrcDir: path.Dir(srcObj.GetPath()), + DstDir: dstDir.GetPath(), + Names: []string{srcObj.GetName()}, + }) + }) + return err +} + +func (d *AListV3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + _, _, err := d.request("/fs/rename", http.MethodPost, func(req *resty.Request) { + req.SetBody(RenameReq{ + Path: srcObj.GetPath(), + Name: newName, + }) + }) + return err +} + +func (d *AListV3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + _, _, err := d.request("/fs/copy", http.MethodPost, func(req *resty.Request) { + req.SetBody(MoveCopyReq{ + SrcDir: path.Dir(srcObj.GetPath()), + DstDir: dstDir.GetPath(), + Names: []string{srcObj.GetName()}, + }) + }) + return err +} + +func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { + _, _, err := d.request("/fs/remove", http.MethodPost, func(req *resty.Request) { + req.SetBody(RemoveReq{ + Dir: path.Dir(obj.GetPath()), + Names: []string{obj.GetName()}, + }) + }) + return err +} + +func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", reader) + if err != nil { + return err + } + req.Header.Set("Authorization", d.Token) + req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) + req.Header.Set("Password", d.MetaPassword) + if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { + req.Header.Set("X-File-Md5", md5) + } + if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { + req.Header.Set("X-File-Sha1", sha1) + } + if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { + req.Header.Set("X-File-Sha256", sha256) + } + + req.ContentLength = s.GetSize() + // client := base.NewHttpClient() + // client.Timeout = time.Hour * 6 + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + + bytes, err := io.ReadAll(res.Body) + if err != nil { + return err + } + log.Debugf("[openlist] response body: %s", string(bytes)) + if res.StatusCode >= 400 { + return fmt.Errorf("request failed, status: %s", res.Status) + } + code := utils.Json.Get(bytes, "code").ToInt() + if code != 200 { + if code == 401 || code == 403 { + err = d.login() + if err != nil { + return err + } + } + return fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(bytes, "message").ToString()) + } + return nil +} + +func (d *AListV3) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotImplement + } + var resp common.Resp[ArchiveMetaResp] + _, code, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }) + }) + if code == 202 { + return nil, errs.WrongArchivePassword + } + if err != nil { + return nil, err + } + var tree []model.ObjTree + if resp.Data.Content != nil { + tree = make([]model.ObjTree, 0, len(resp.Data.Content)) + for _, content := range resp.Data.Content { + tree = append(tree, &content) + } + } + return &model.ArchiveMetaInfo{ + Comment: resp.Data.Comment, + Encrypted: resp.Data.Encrypted, + Tree: tree, + }, nil +} + +func (d *AListV3) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotImplement + } + var resp common.Resp[ArchiveListResp] + _, code, err := d.request("/fs/archive/list", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveListReq{ + ArchiveMetaReq: ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }, + PageReq: model.PageReq{ + Page: 1, + PerPage: 0, + }, + InnerPath: args.InnerPath, + }) + }) + if code == 202 { + return nil, errs.WrongArchivePassword + } + if err != nil { + return nil, err + } + var files []model.Obj + for _, f := range resp.Data.Content { + file := model.ObjThumb{ + Object: model.Object{ + Name: f.Name, + Modified: f.Modified, + Ctime: f.Created, + Size: f.Size, + IsFolder: f.IsDir, + HashInfo: utils.FromString(f.HashInfo), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, + } + files = append(files, &file) + } + return files, nil +} + +func (d *AListV3) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotSupport + } + var resp common.Resp[ArchiveMetaResp] + _, _, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }) + }) + if err != nil { + return nil, err + } + return &model.Link{ + URL: fmt.Sprintf("%s?inner=%s&pass=%s&sign=%s", + resp.Data.RawURL, + utils.EncodePath(args.InnerPath, true), + url.QueryEscape(args.Password), + resp.Data.Sign), + }, nil +} + +func (d *AListV3) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { + if !d.ForwardArchiveReq { + return errs.NotImplement + } + dir, name := path.Split(srcObj.GetPath()) + _, _, err := d.request("/fs/archive/decompress", http.MethodPost, func(req *resty.Request) { + req.SetBody(DecompressReq{ + ArchivePass: args.Password, + CacheFull: args.CacheFull, + DstDir: dstDir.GetPath(), + InnerPath: args.InnerPath, + Name: []string{name}, + PutIntoNewDir: args.PutIntoNewDir, + SrcDir: dir, + }) + }) + return err +} + +func (d *AListV3) ResolveLinkCacheMode(_ string) driver.LinkCacheMode { + var mode driver.LinkCacheMode + if d.PassIPToUpsteam { + mode |= driver.LinkCacheIP + } + if d.PassUAToUpsteam { + mode |= driver.LinkCacheUA + } + return mode +} + +var _ driver.Driver = (*AListV3)(nil) diff --git a/drivers/alist_v3/meta.go b/drivers/alist_v3/meta.go new file mode 100644 index 000000000..2e5b83f8e --- /dev/null +++ b/drivers/alist_v3/meta.go @@ -0,0 +1,32 @@ +package alist_v3 + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + Address string `json:"url" required:"true"` + MetaPassword string `json:"meta_password"` + Username string `json:"username"` + Password string `json:"password"` + Token string `json:"token"` + PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` + PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` + ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` +} + +var config = driver.Config{ + Name: "AList V3", + LocalSort: true, + DefaultRoot: "/", + ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheAuto, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &AListV3{} + }) +} diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go new file mode 100644 index 000000000..527de7799 --- /dev/null +++ b/drivers/alist_v3/types.go @@ -0,0 +1,182 @@ +package alist_v3 + +import ( + "encoding/json" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type ListReq struct { + model.PageReq + Path string `json:"path" form:"path"` + Password string `json:"password" form:"password"` + Refresh bool `json:"refresh"` +} + +type ObjResp struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfo string `json:"hashinfo"` +} + +type FsListResp struct { + Content []ObjResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Write bool `json:"write"` + Provider string `json:"provider"` +} + +type FsGetReq struct { + Path string `json:"path" form:"path"` + Password string `json:"password" form:"password"` +} + +type FsGetResp struct { + ObjResp + RawURL string `json:"raw_url"` + Readme string `json:"readme"` + Provider string `json:"provider"` + Related []ObjResp `json:"related"` +} + +type MkdirOrLinkReq struct { + Path string `json:"path" form:"path"` +} + +type MoveCopyReq struct { + SrcDir string `json:"src_dir"` + DstDir string `json:"dst_dir"` + Names []string `json:"names"` +} + +type RenameReq struct { + Path string `json:"path"` + Name string `json:"name"` +} + +type RemoveReq struct { + Dir string `json:"dir"` + Names []string `json:"names"` +} + +type LoginResp struct { + Token string `json:"token"` +} + +type MeResp struct { + Id int `json:"id"` + Username string `json:"username"` + Password string `json:"password"` + BasePath string `json:"base_path"` + Role IntSlice `json:"role"` + Disabled bool `json:"disabled"` + Permission int `json:"permission"` + SsoId string `json:"sso_id"` + Otp bool `json:"otp"` +} + +type IntSlice []int + +func (s *IntSlice) UnmarshalJSON(b []byte) error { + var i int + if json.Unmarshal(b, &i) == nil { + *s = []int{i} + return nil + } + return json.Unmarshal(b, (*[]int)(s)) +} + +type ArchiveMetaReq struct { + ArchivePass string `json:"archive_pass"` + Password string `json:"password"` + Path string `json:"path"` + Refresh bool `json:"refresh"` +} + +type TreeResp struct { + ObjResp + Children []TreeResp `json:"children"` + hashCache *utils.HashInfo +} + +func (t *TreeResp) GetSize() int64 { + return t.Size +} + +func (t *TreeResp) GetName() string { + return t.Name +} + +func (t *TreeResp) ModTime() time.Time { + return t.Modified +} + +func (t *TreeResp) CreateTime() time.Time { + return t.Created +} + +func (t *TreeResp) IsDir() bool { + return t.ObjResp.IsDir +} + +func (t *TreeResp) GetHash() utils.HashInfo { + return utils.FromString(t.HashInfo) +} + +func (t *TreeResp) GetID() string { + return "" +} + +func (t *TreeResp) GetPath() string { + return "" +} + +func (t *TreeResp) GetChildren() []model.ObjTree { + ret := make([]model.ObjTree, 0, len(t.Children)) + for _, child := range t.Children { + ret = append(ret, &child) + } + return ret +} + +func (t *TreeResp) Thumb() string { + return t.ObjResp.Thumb +} + +type ArchiveMetaResp struct { + Comment string `json:"comment"` + Encrypted bool `json:"encrypted"` + Content []TreeResp `json:"content"` + RawURL string `json:"raw_url"` + Sign string `json:"sign"` +} + +type ArchiveListReq struct { + model.PageReq + ArchiveMetaReq + InnerPath string `json:"inner_path"` +} + +type ArchiveListResp struct { + Content []ObjResp `json:"content"` + Total int64 `json:"total"` +} + +type DecompressReq struct { + ArchivePass string `json:"archive_pass"` + CacheFull bool `json:"cache_full"` + DstDir string `json:"dst_dir"` + InnerPath string `json:"inner_path"` + Name []string `json:"name"` + PutIntoNewDir bool `json:"put_into_new_dir"` + SrcDir string `json:"src_dir"` +} diff --git a/drivers/alist_v3/util.go b/drivers/alist_v3/util.go new file mode 100644 index 000000000..821fe5465 --- /dev/null +++ b/drivers/alist_v3/util.go @@ -0,0 +1,65 @@ +package alist_v3 + +import ( + "fmt" + "net/http" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +func (d *AListV3) login() error { + if d.Username == "" { + return nil + } + var resp common.Resp[LoginResp] + _, _, err := d.request("/auth/login", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(base.Json{ + "username": d.Username, + "password": d.Password, + }) + }) + if err != nil { + return err + } + d.Token = resp.Data.Token + op.MustSaveDriverStorage(d) + return nil +} + +func (d *AListV3) request(api, method string, callback base.ReqCallback, retry ...bool) ([]byte, int, error) { + url := d.Address + "/api" + api + req := base.RestyClient.R() + req.SetHeader("Authorization", d.Token) + if callback != nil { + callback(req) + } + res, err := req.Execute(method, url) + if err != nil { + code := 0 + if res != nil { + code = res.StatusCode() + } + return nil, code, err + } + log.Debugf("[openlist] response body: %s", res.String()) + if res.StatusCode() >= 400 { + return nil, res.StatusCode(), fmt.Errorf("request failed, status: %s", res.Status()) + } + code := utils.Json.Get(res.Body(), "code").ToInt() + if code != 200 { + if (code == 401 || code == 403) && !utils.IsBool(retry...) { + err = d.login() + if err != nil { + return nil, code, err + } + return d.request(api, method, callback, true) + } + return nil, code, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) + } + return res.Body(), 200, nil +} diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index 6fd9bff26..3e71e8c6e 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -334,12 +334,12 @@ func (d *AliDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error if err != nil { return nil, err } - used := utils.Json.Get(res, "drive_used_size").ToUint64() - total := utils.Json.Get(res, "drive_total_size").ToUint64() + used := utils.Json.Get(res, "drive_used_size").ToInt64() + total := utils.Json.Get(res, "drive_total_size").ToInt64() return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index 20dd92dca..ee93b3303 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -77,7 +77,6 @@ func (d *AliyundriveOpen) GetRoot(ctx context.Context) (model.Obj, error) { ID: d.RootFolderID, Path: "/", Name: "root", - Size: 0, Modified: d.Modified, IsFolder: true, }, nil @@ -296,12 +295,12 @@ func (d *AliyundriveOpen) GetDetails(ctx context.Context) (*model.StorageDetails if err != nil { return nil, err } - total := utils.Json.Get(res, "personal_space_info", "total_size").ToUint64() - used := utils.Json.Get(res, "personal_space_info", "used_size").ToUint64() + total := utils.Json.Get(res, "personal_space_info", "total_size").ToInt64() + used := utils.Json.Get(res, "personal_space_info", "used_size").ToInt64() return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 9114997c2..a4a6c1de1 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -242,11 +242,11 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m if err != nil { return nil, err } - rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd) err = retry.Do(func() error { rd.Seek(0, io.SeekStart) - return d.uploadPart(ctx, rateLimitedRd, createResp.PartInfoList[i]) + return d.uploadPart(ctx, driver.NewLimitedUploadStream(ctx, rd), createResp.PartInfoList[i]) }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) diff --git a/drivers/aliyundrive_open/util.go b/drivers/aliyundrive_open/util.go index d171a7909..22112332a 100644 --- a/drivers/aliyundrive_open/util.go +++ b/drivers/aliyundrive_open/util.go @@ -38,7 +38,6 @@ func (d *AliyundriveOpen) _refreshToken(ctx context.Context) (string, string, er return "", "", err } _, err = base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, diff --git a/drivers/all.go b/drivers/all.go index 8184f9871..de5dfa41b 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -13,9 +13,11 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/189_tv" _ "github.com/OpenListTeam/OpenList/v4/drivers/189pc" _ "github.com/OpenListTeam/OpenList/v4/drivers/alias" + _ "github.com/OpenListTeam/OpenList/v4/drivers/alist_v3" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_share" + _ "github.com/OpenListTeam/OpenList/v4/drivers/autoindex" _ "github.com/OpenListTeam/OpenList/v4/drivers/azure_blob" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" @@ -36,6 +38,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/google_drive" _ "github.com/OpenListTeam/OpenList/v4/drivers/google_photo" _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud" + _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/ilanzou" _ "github.com/OpenListTeam/OpenList/v4/drivers/ipfs_api" _ "github.com/OpenListTeam/OpenList/v4/drivers/kodbox" @@ -55,9 +58,11 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/openlist_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" _ "github.com/OpenListTeam/OpenList/v4/drivers/pikpak_share" + _ "github.com/OpenListTeam/OpenList/v4/drivers/proton_drive" _ "github.com/OpenListTeam/OpenList/v4/drivers/quark_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/quark_uc" _ "github.com/OpenListTeam/OpenList/v4/drivers/quark_uc_tv" + _ "github.com/OpenListTeam/OpenList/v4/drivers/qihoo360" _ "github.com/OpenListTeam/OpenList/v4/drivers/s3" _ "github.com/OpenListTeam/OpenList/v4/drivers/seafile" _ "github.com/OpenListTeam/OpenList/v4/drivers/sftp" @@ -75,11 +80,11 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/webdav" _ "github.com/OpenListTeam/OpenList/v4/drivers/weiyun" _ "github.com/OpenListTeam/OpenList/v4/drivers/wopan" + _ "github.com/OpenListTeam/OpenList/v4/drivers/wps" _ "github.com/OpenListTeam/OpenList/v4/drivers/yandex_disk" ) // All do nothing,just for import // same as _ import func All() { - } diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go new file mode 100644 index 000000000..5758ec6cd --- /dev/null +++ b/drivers/autoindex/driver.go @@ -0,0 +1,169 @@ +package autoindex + +import ( + "context" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/antchfx/htmlquery" + "github.com/antchfx/xpath" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type AutoIndex struct { + model.Storage + Addition + itemXPath *xpath.Expr + nameXPath *xpath.Expr + modifiedXPath *xpath.Expr + sizeXPath *xpath.Expr + ignores map[string]any +} + +func (d *AutoIndex) Config() driver.Config { + return config +} + +func (d *AutoIndex) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *AutoIndex) Init(ctx context.Context) error { + var err error + d.itemXPath, err = xpath.Compile(d.ItemXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Item XPath") + } + d.nameXPath, err = xpath.Compile(d.NameXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Name XPath") + } + if len(d.ModifiedXPath) > 0 { + d.modifiedXPath, err = xpath.Compile(d.ModifiedXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Modified XPath") + } + } + if len(d.SizeXPath) > 0 { + d.sizeXPath, err = xpath.Compile(d.SizeXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Size XPath") + } + } + ignores := strings.Split(d.IgnoreFileNames, "\n") + d.ignores = make(map[string]any, len(ignores)) + for _, i := range ignores { + i = strings.TrimSpace(i) + if len(i) == 0 { + continue + } + d.ignores[i] = struct{}{} + } + hasScheme := strings.Contains(d.URL, "://") + hasSuffix := strings.HasSuffix(d.URL, "/") + if !hasScheme || !hasSuffix { + if !hasSuffix { + d.URL = d.URL + "/" + } + if !hasScheme { + d.URL = "https://" + d.URL + } + op.MustSaveDriverStorage(d) + } + return nil +} + +func (d *AutoIndex) Drop(ctx context.Context) error { + return nil +} + +func (d *AutoIndex) GetRoot(ctx context.Context) (model.Obj, error) { + return &model.Object{ + Name: op.RootName, + Path: d.URL, + Modified: d.Modified, + Mask: model.Locked, + IsFolder: true, + }, nil +} + +func (d *AutoIndex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + res, err := base.RestyClient.R(). + SetContext(ctx). + SetDoNotParseResponse(true). + Get(dir.GetPath()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to get url [%s]", dir.GetPath()) + } + defer res.RawResponse.Body.Close() + doc, err := htmlquery.Parse(res.RawBody()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to parse [%s]", dir.GetPath()) + } + itemsIter := d.itemXPath.Select(htmlquery.CreateXPathNavigator(doc)) + var objs []model.Obj + for itemsIter.MoveNext() { + nameFull, err := parseString(d.nameXPath.Evaluate(itemsIter.Current().Copy())) + if err != nil { + log.Warnf("skip invalid name evaluating result: %v", err) + continue + } + nameFull = strings.TrimSpace(nameFull) + name, isDir := strings.CutSuffix(nameFull, "/") + if _, ok := d.ignores[name]; ok { + continue + } + var size int64 = 0 + exact := false + modified := time.Now() + if d.sizeXPath != nil { + size, exact, err = parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy())) + if err != nil { + log.Errorf("failed to parse size of %s: %v", name, err) + } + } + if d.modifiedXPath != nil { + modified, err = parseTime(d.modifiedXPath.Evaluate(itemsIter.Current().Copy()), d.ModifiedTimeFormat) + if err != nil { + log.Errorf("failed to parse modified time of %s: %v", name, err) + } + } + var o model.Obj = &model.Object{ + Name: name, + IsFolder: isDir, + Path: dir.GetPath() + nameFull, + Modified: modified, + Size: size, + } + if exact { + o = &exactSizeObj{Obj: o} + } + objs = append(objs, o) + } + return objs, nil +} + +func (d *AutoIndex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if _, ok := file.(*exactSizeObj); ok || args.Redirect { + return &model.Link{URL: file.GetPath()}, nil + } + res, err := base.RestyClient.R(). + SetContext(ctx). + SetDoNotParseResponse(true). + Head(file.GetPath()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to head [%s]", file.GetPath()) + } + _ = res.RawResponse.Body.Close() + return &model.Link{ + URL: file.GetPath(), + ContentLength: res.RawResponse.ContentLength, + }, nil +} + +var _ driver.Driver = (*AutoIndex)(nil) diff --git a/drivers/autoindex/meta.go b/drivers/autoindex/meta.go new file mode 100644 index 000000000..8ebae5924 --- /dev/null +++ b/drivers/autoindex/meta.go @@ -0,0 +1,29 @@ +package autoindex + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + URL string `json:"url" required:"true"` + ItemXPath string `json:"item_xpath" required:"true"` + NameXPath string `json:"name_xpath" required:"true"` + ModifiedXPath string `json:"modified_xpath"` + SizeXPath string `json:"size_xpath"` + IgnoreFileNames string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory\nUp"` + ModifiedTimeFormat string `json:"modified_time_format" default:"02-Jan-2006 15:04" help:"Must be based on the time point Mon Jan 2 15:04:05 -0700 MST 2006"` +} + +var config = driver.Config{ + Name: "AutoIndex", + LocalSort: true, + CheckStatus: true, + NoUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &AutoIndex{} + }) +} diff --git a/drivers/autoindex/types.go b/drivers/autoindex/types.go new file mode 100644 index 000000000..48ec396f5 --- /dev/null +++ b/drivers/autoindex/types.go @@ -0,0 +1,13 @@ +package autoindex + +import ( + "fmt" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +var ( + errEmptyEvaluateResult = fmt.Errorf("empty result") +) + +type exactSizeObj struct{ model.Obj } diff --git a/drivers/autoindex/util.go b/drivers/autoindex/util.go new file mode 100644 index 000000000..f5f36a736 --- /dev/null +++ b/drivers/autoindex/util.go @@ -0,0 +1,116 @@ +package autoindex + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/antchfx/xpath" + "github.com/pkg/errors" +) + +var units = map[string]int64{ + "": 1, + "b": 1, + "byte": 1, + "bytes": 1, + "k": 1 << 10, + "kb": 1 << 10, + "kib": 1 << 10, + "m": 1 << 20, + "mb": 1 << 20, + "mib": 1 << 20, + "g": 1 << 30, + "gb": 1 << 30, + "gib": 1 << 30, + "t": 1 << 40, + "tb": 1 << 40, + "tib": 1 << 40, + "p": 1 << 50, + "pb": 1 << 50, + "pib": 1 << 50, +} + +func splitUnit(s string) (string, string) { + for i := len(s) - 1; i >= 0; i-- { + if s[i] >= '0' && s[i] <= '9' { + return strings.TrimSpace(s[:i+1]), strings.TrimSpace(s[i+1:]) + } + } + return "", s +} + +func parseSize(a any) (int64, bool, error) { + // 第二个返回值exact表示大小是否精确 + if f, ok := a.(float64); ok { + return int64(f), false, nil + } + s, err := parseString(a) + if errors.Is(err, errEmptyEvaluateResult) { + // 可能是错误，也可能确实大小为0 + // 如果确实大小为0，大概率不会下载，exact返回false也不会有什么性能损失 + // 如果是错误，exact返回true会导致本地代理出错，综合来看返回false更好 + return 0, false, nil + } + if err != nil { + return 0, false, err + } + s = strings.TrimSpace(s) + if s == "-" { + return 0, false, nil + } + nbs, unit := splitUnit(s) + mul, ok := units[strings.ToLower(unit)] + exact := mul == 1 + if !ok { + mul = 1 + // 推测无单位，exact应为false + } + nb, err := strconv.ParseInt(nbs, 10, 64) + if err != nil { + fnb, err := strconv.ParseFloat(nbs, 64) + if err != nil { + return 0, false, fmt.Errorf("failed to convert %s to number", nbs) + } + nb = int64(fnb * float64(mul)) + exact = false + } else { + nb = nb * mul + } + return nb, exact, nil +} + +func parseString(res any) (string, error) { + if r, ok := res.(string); ok { + if len(r) == 0 { + return "", errEmptyEvaluateResult + } + return r, nil + } + n, ok := res.(*xpath.NodeIterator) + if !ok { + return "", fmt.Errorf("unsupported evaluating result") + } + if !n.MoveNext() { + return "", fmt.Errorf("no matched nodes") + } + ns := n.Current().Value() + if len(ns) == 0 { + return "", errEmptyEvaluateResult + } + return ns, nil +} + +func parseTime(res any, format string) (time.Time, error) { + s, err := parseString(res) + if err != nil { + return time.Now(), err + } + s = strings.TrimSpace(s) + t, err := time.Parse(format, s) + if err != nil { + return time.Now(), errors.WithMessagef(err, "failed to convert %s to time", s) + } + return t, nil +} diff --git a/drivers/autoindex/util_test.go b/drivers/autoindex/util_test.go new file mode 100644 index 000000000..ba743b943 --- /dev/null +++ b/drivers/autoindex/util_test.go @@ -0,0 +1,49 @@ +package autoindex + +import ( + "testing" +) + +type wantType struct { + v int64 + exact bool + error bool +} + +func TestParseSize(t *testing.T) { + tests := []struct { + input string + want wantType + }{ + {"100", wantType{100, true, false}}, + {"1k", wantType{1024, false, false}}, + {"1kb", wantType{1024, false, false}}, + {"1K", wantType{1024, false, false}}, // case insensitive + {"1.5m", wantType{1572864, false, false}}, // 1.5 * 1024^2 + {"500 bytes", wantType{500, true, false}}, + {"-", wantType{0, false, false}}, + {"", wantType{0, false, false}}, + {"abc", wantType{0, false, true}}, + {"1.5GB", wantType{1610612736, false, false}}, // 1.5 * 1024^3 + {"2t", wantType{2199023255552, false, false}}, // 2 * 1024^4 + {"1p", wantType{1125899906842624, false, false}}, // 1 * 1024^5 + {"0", wantType{0, true, false}}, + {" 100 ", wantType{100, true, false}}, // trimmed + {"100b", wantType{100, true, false}}, + {"1gib", wantType{1073741824, false, false}}, // 1024^3 + {"1z", wantType{1, false, false}}, // invalid unit, mul=1 + {"1.5", wantType{1, false, false}}, // float without unit, truncated + {"2.7k", wantType{2764, false, false}}, // 2.7 * 1024 truncated + {"1.0g", wantType{1073741824, false, false}}, // 1.0 * 1024^3 + {"invalid", wantType{0, false, true}}, + {"123xyz", wantType{123, false, false}}, // unit not found, mul=1 + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got, exact, err := parseSize(tt.input) + if got != tt.want.v || exact != tt.want.exact || (err != nil) != tt.want.error { + t.Errorf("ParseSize(%q) = (%d, %t, %t), want (%d, %t, %t)", tt.input, got, exact, err != nil, tt.want.v, tt.want.exact, tt.want.error) + } + }) + } +} diff --git a/drivers/azure_blob/driver.go b/drivers/azure_blob/driver.go index 888637dc9..ddfe3ff68 100644 --- a/drivers/azure_blob/driver.go +++ b/drivers/azure_blob/driver.go @@ -25,12 +25,11 @@ type AzureBlob struct { Addition client *azblob.Client containerClient *container.Client - config driver.Config } // Config returns the driver configuration. func (d *AzureBlob) Config() driver.Config { - return d.config + return config } // GetAddition returns additional settings specific to Azure Blob Storage. diff --git a/drivers/azure_blob/meta.go b/drivers/azure_blob/meta.go index 3aae5044f..d675d4083 100644 --- a/drivers/azure_blob/meta.go +++ b/drivers/azure_blob/meta.go @@ -6,17 +6,13 @@ import ( ) type Addition struct { + driver.RootPath Endpoint string `json:"endpoint" required:"true" default:"https://.blob.core.windows.net/" help:"e.g. https://accountname.blob.core.windows.net/. The full endpoint URL for Azure Storage, including the unique storage account name (3 ~ 24 numbers and lowercase letters only)."` AccessKey string `json:"access_key" required:"true" help:"The access key for Azure Storage, used for authentication. https://learn.microsoft.com/azure/storage/common/storage-account-keys-manage"` ContainerName string `json:"container_name" required:"true" help:"The name of the container in Azure Storage (created in the Azure portal). https://learn.microsoft.com/azure/storage/blobs/blob-containers-portal"` SignURLExpire int `json:"sign_url_expire" type:"number" default:"4" help:"The expiration time for SAS URLs, in hours."` } -// implement GetRootId interface -func (r Addition) GetRootId() string { - return r.ContainerName -} - var config = driver.Config{ Name: "Azure Blob Storage", LocalSort: true, @@ -25,8 +21,6 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { - return &AzureBlob{ - config: config, - } + return &AzureBlob{} }) } diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 4f8e3446f..fe77aca38 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -1,15 +1,19 @@ package baidu_netdisk import ( + "bytes" "context" "crypto/md5" "encoding/hex" "errors" "io" + "mime/multipart" + "net/http" "net/url" "os" stdpath "path" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -17,6 +21,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" @@ -31,6 +36,8 @@ type BaiduNetdisk struct { vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) } +var ErrUploadIDExpired = errors.New("uploadid expired") + func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -41,18 +48,20 @@ func (d *BaiduNetdisk) GetAddition() driver.Additional { func (d *BaiduNetdisk) Init(ctx context.Context) error { d.uploadThread, _ = strconv.Atoi(d.UploadThread) - if d.uploadThread < 1 || d.uploadThread > 32 { - d.uploadThread, d.UploadThread = 3, "3" + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" } if _, err := url.Parse(d.UploadAPI); d.UploadAPI == "" || err != nil { - d.UploadAPI = "https://d.pcs.baidu.com" + d.UploadAPI = UPLOAD_FALLBACK_API } res, err := d.get("/xpan/nas", map[string]string{ "method": "uinfo", }, nil) - log.Debugf("[baidu] get uinfo: %s", string(res)) + log.Debugf("[baidu_netdisk] get uinfo: %s", string(res)) if err != nil { return err } @@ -75,9 +84,10 @@ func (d *BaiduNetdisk) List(ctx context.Context, dir model.Obj, args model.ListA } func (d *BaiduNetdisk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - if d.DownloadAPI == "crack" { + switch d.DownloadAPI { + case "crack": return d.linkCrack(file, args) - } else if d.DownloadAPI == "crack_video" { + case "crack_video": return d.linkCrackVideo(file, args) } return d.linkOfficial(file, args) @@ -179,6 +189,11 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo // **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间，而不是文件时间。 // 而实际上云盘存储的时间是文件时间，所以此处需要覆盖时间，保证缓存与云盘的数据一致 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // 百度网盘不允许上传空文件 + if stream.GetSize() < 1 { + return nil, ErrBaiduEmptyFilesNotAllowed + } + // rapid upload if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { return newObj, nil @@ -189,7 +204,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F tmpF *os.File err error ) - if _, ok := cache.(io.ReaderAt); !ok { + if cache == nil { tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err @@ -214,7 +229,6 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // cal md5 for first 256k data const SliceSize int64 = 256 * utils.KB - // cal md5 blockList := make([]string, 0, count) byteSize := sliceSize fileMd5H := md5.New() @@ -244,7 +258,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } if tmpF != nil { if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) + return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) } _, err = tmpF.Seek(0, io.SeekStart) if err != nil { @@ -258,31 +272,14 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 预上传 - // 尝试获取之前的进度 + // step.1 尝试读取已保存进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - params := map[string]string{ - "method": "precreate", - } - form := map[string]string{ - "path": path, - "size": strconv.FormatInt(streamSize, 10), - "isdir": "0", - "autoinit": "1", - "rtype": "3", - "block_list": blockListStr, - "content-md5": contentMd5, - "slice-md5": sliceMd5, - } - joinTime(form, ctime, mtime) - - log.Debugf("[baidu_netdisk] precreate data: %s", form) - _, err = d.postForm("/xpan/file", params, form, &precreateResp) + // 没有进度，走预上传 + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) if err != nil { return nil, err } - log.Debugf("%+v", precreateResp) if precreateResp.ReturnType == 2 { // rapid upload, since got md5 match from baidu server // 修复时间，具体原因见 Put 方法注释的 **注意** @@ -291,48 +288,95 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return fileToObj(precreateResp.File), nil } } + ensureUploadURL := func() { + if precreateResp.UploadURL != "" { + return + } + precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) + } + // step.2 上传分片 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(1), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) - - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break +uploadLoop: + for range 2 { + // 获取上传域名 + ensureUploadURL() + // 并发上传 + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(UPLOAD_RETRY_COUNT), + retry.Delay(UPLOAD_RETRY_WAIT_TIME), + retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), + retry.DelayType(retry.BackOffDelay), + retry.RetryIf(func(err error) bool { + return !errors.Is(err, ErrUploadIDExpired) + }), + retry.LastErrorOnly(true)) + + totalParts := len(precreateResp.BlockList) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) { + break + } + if partseq < 0 { + continue + } + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize + } + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": precreateResp.Uploadid, + "partseq": strconv.Itoa(partseq), + } + section := io.NewSectionReader(cache, offset, size) + err := d.uploadSlice(ctx, precreateResp.UploadURL, params, stream.GetName(), section) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + progress := float64(threadG.Success()+1) * 100 / float64(totalParts+1) + up(progress) + return nil + }) } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - byteSize = lastBlockSize + err = threadG.Wait() + if err == nil { + break uploadLoop } - threadG.Go(func(ctx context.Context) error { - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), + + // 保存进度（所有错误都会保存） + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + + if errors.Is(err, context.Canceled) { + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch") + // 重新 precreate（所有分片都要重传） + newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime) + if err2 != nil { + return nil, err2 } - err := d.uploadSlice(ctx, params, stream.GetName(), - driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) - if err != nil { - return err + if newPre.ReturnType == 2 { + return fileToObj(newPre.File), nil } - up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) - precreateResp.BlockList[i] = -1 - return nil - }) - } - if err = threadG.Wait(); err != nil { - // 如果属于用户主动取消，则保存上传进度 - if errors.Is(err, context.Canceled) { - precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + precreateResp = newPre + precreateResp.UploadURL = "" + // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + continue uploadLoop } return nil, err } + defer up(100) // step.3 创建文件 var newFile File @@ -343,23 +387,104 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = ctime newFile.Mtime = mtime + // 上传成功清理进度 + base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5) return fileToObj(newFile), nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { - res, err := base.RestyClient.R(). - SetContext(ctx). - SetQueryParams(params). - SetFileReader("file", fileName, file). - Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") +// precreate 执行预上传操作，支持首次上传和 uploadid 过期重试 +func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + params := map[string]string{"method": "precreate"} + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + } + + // 只有在首次上传时才包含 content-md5 和 slice-md5 + if contentMd5 != "" && sliceMd5 != "" { + form["content-md5"] = contentMd5 + form["slice-md5"] = sliceMd5 + } + + joinTime(form, ctime, mtime) + + var precreateResp PrecreateResp + _, err := d.postForm("/xpan/file", params, form, &precreateResp) + if err != nil { + return nil, err + } + + // 修复时间，具体原因见 Put 方法注释的 **注意** + if precreateResp.ReturnType == 2 { + precreateResp.File.Ctime = ctime + precreateResp.File.Mtime = mtime + } + + return &precreateResp, nil +} + +func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file *io.SectionReader) error { + b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) + mw := multipart.NewWriter(b) + _, err := mw.CreateFormFile("file", fileName) + if err != nil { + return err + } + headSize := b.Len() + err = mw.Close() if err != nil { return err } - log.Debugln(res.RawResponse.Status + res.String()) - errCode := utils.Json.Get(res.Body(), "error_code").ToInt() - errNo := utils.Json.Get(res.Body(), "errno").ToInt() + head := bytes.NewReader(b.Bytes()[:headSize]) + tail := bytes.NewReader(b.Bytes()[headSize:]) + rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) + if err != nil { + return err + } + query := req.URL.Query() + for k, v := range params { + query.Set(k, v) + } + req.URL.RawQuery = query.Encode() + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(b.Len()) + file.Size() + + client := net.NewHttpClient() + if d.UploadSliceTimeout > 0 { + client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) + } else { + client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT + } + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + b.Reset() + _, err = b.ReadFrom(resp.Body) + if err != nil { + return err + } + body := b.Bytes() + respStr := string(body) + log.Debugln(respStr) + lower := strings.ToLower(respStr) + // 合并 uploadid 过期检测逻辑 + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error in uploading to baidu, will retry. response=%s", res.String()) + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) } return nil } @@ -369,7 +494,7 @@ func (d *BaiduNetdisk) GetDetails(ctx context.Context) (*model.StorageDetails, e if err != nil { return nil, err } - return &model.StorageDetails{DiskUsage: *du}, nil + return &model.StorageDetails{DiskUsage: du}, nil } var _ driver.Driver = (*BaiduNetdisk)(nil) diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 642025911..3f3bed022 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -1,6 +1,8 @@ package baidu_netdisk import ( + "time" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) @@ -18,15 +20,27 @@ type Addition struct { AccessToken string RefreshToken string `json:"refresh_token" required:"true"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + UploadSliceTimeout int `json:"upload_timeout" type:"number" default:"60" help:"per-slice upload timeout in seconds"` UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + UseDynamicUploadAPI bool `json:"use_dynamic_upload_api" default:"true" help:"dynamically get upload api domain, when enabled, the 'Upload API' setting will be used as a fallback if failed to get"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"` OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` } +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 + UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) + DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 60 // 上传分片请求默认超时时间 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second * 1 + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 +) + var config = driver.Config{ Name: "BaiduNetdisk", DefaultRoot: "/", + PreferProxy: true, } func init() { diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index 0e9ee4432..2ea037b3f 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -1,12 +1,16 @@ package baidu_netdisk import ( + "errors" "path" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +var ( + ErrBaiduEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu netdisk") ) type TokenErrResp struct { @@ -71,9 +75,7 @@ func fileToObj(f File) *model.ObjThumb { Modified: time.Unix(f.ServerMtime, 0), Ctime: time.Unix(f.ServerCtime, 0), IsFolder: f.Isdir == 1, - - // 直接获取的MD5是错误的 - HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), + // 百度API返回的MD5不可信，不使用HashInfo }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } @@ -188,13 +190,39 @@ type PrecreateResp struct { // return_type=2 File File `json:"info"` + + UploadURL string `json:"-"` // 保存断点续传对应的上传域名 +} + +type UploadServerResp struct { + BakServer []any `json:"bak_server"` + BakServers []struct { + Server string `json:"server"` + } `json:"bak_servers"` + ClientIP string `json:"client_ip"` + ErrorCode int `json:"error_code"` + ErrorMsg string `json:"error_msg"` + Expire int `json:"expire"` + Host string `json:"host"` + Newno string `json:"newno"` + QuicServer []any `json:"quic_server"` + QuicServers []struct { + Server string `json:"server"` + } `json:"quic_servers"` + RequestID int64 `json:"request_id"` + Server []any `json:"server"` + ServerTime int `json:"server_time"` + Servers []struct { + Server string `json:"server"` + } `json:"servers"` + Sl int `json:"sl"` } type QuotaResp struct { - Errno int `json:"errno"` - RequestId int64 `json:"request_id"` - Total uint64 `json:"total"` - Used uint64 `json:"used"` - //Free uint64 `json:"free"` + Errno int `json:"errno"` + RequestId int64 `json:"request_id"` + Total int64 `json:"total"` + Used int64 `json:"used"` + //FreeSpace uint64 `json:"free"` //Expire bool `json:"expire"` } diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 6ae911e95..0e27fb305 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -41,7 +41,6 @@ func (d *BaiduNetdisk) _refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, @@ -114,14 +113,14 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall errno := utils.Json.Get(res.Body(), "errno").ToInt() if errno != 0 { if utils.SliceContains([]int{111, -6}, errno) { - log.Info("refreshing baidu_netdisk token.") + log.Info("[baidu_netdisk] refreshing baidu_netdisk token.") err2 := d.refreshToken() if err2 != nil { return retry.Unrecoverable(err2) } } - if 31023 == errno && d.DownloadAPI == "crack_video" { + if errno == 31023 && d.DownloadAPI == "crack_video" { result = res.Body() return nil } @@ -153,7 +152,7 @@ func (d *BaiduNetdisk) postForm(pathname string, params map[string]string, form func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { start := 0 - limit := 200 + limit := 1000 params := map[string]string{ "method": "list", "dir": dir, @@ -169,7 +168,6 @@ func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { for { params["start"] = strconv.Itoa(start) params["limit"] = strconv.Itoa(limit) - start += limit var resp ListResp _, err := d.get("/xpan/file", params, &resp) if err != nil { @@ -188,6 +186,11 @@ func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { } else { res = append(res, resp.List...) } + + if len(resp.List) < limit { + break + } + start += limit } return res, nil } @@ -246,7 +249,7 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, _ model.LinkArgs) (*model.Link, func (d *BaiduNetdisk) linkCrackVideo(file model.Obj, _ model.LinkArgs) (*model.Link, error) { param := map[string]string{ "type": "VideoURL", - "path": fmt.Sprintf("%s", file.GetPath()), + "path": file.GetPath(), "fs_id": file.GetID(), "devuid": "0%1", "clienttype": "1", @@ -325,10 +328,10 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { // 非会员固定为 4MB if d.vipType == 0 { if d.CustomUploadPartSize != 0 { - log.Warnf("CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") + log.Warnf("[baidu_netdisk] CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") } if filesize > MaxSliceNum*DefaultSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return DefaultSliceSize @@ -336,17 +339,17 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { if d.CustomUploadPartSize != 0 { if d.CustomUploadPartSize < DefaultSliceSize { - log.Warnf("CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) return DefaultSliceSize } if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) return VipSliceSize } if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) return SVipSliceSize } @@ -376,24 +379,67 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { } if filesize > MaxSliceNum*maxSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return maxSliceSize } -func (d *BaiduNetdisk) quota(ctx context.Context) (*model.DiskUsage, error) { +func (d *BaiduNetdisk) quota(ctx context.Context) (model.DiskUsage, error) { var resp QuotaResp _, err := d.request("https://pan.baidu.com/api/quota", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, &resp) if err != nil { - return nil, err + return model.DiskUsage{}, err } - return &model.DiskUsage{ - TotalSpace: resp.Total, - FreeSpace: resp.Total - resp.Used, - }, nil + return model.DiskUsage{TotalSpace: resp.Total, UsedSpace: resp.Used}, nil +} + +// getUploadUrl 从开放平台获取上传域名/地址，并发请求会被合并，结果会在 uploadid 生命周期内复用。 +// 如果获取失败，则返回 Upload API设置项。 +func (d *BaiduNetdisk) getUploadUrl(path, uploadId string) string { + if !d.UseDynamicUploadAPI || uploadId == "" { + return d.UploadAPI + } + + uploadUrl, err := d.requestForUploadUrl(path, uploadId) + if err != nil { + return d.UploadAPI + } + return uploadUrl +} + +// requestForUploadUrl 请求获取上传地址。 +// 实测此接口不需要认证，传method和upload_version就行，不过还是按文档规范调用。 +// https://pan.baidu.com/union/doc/Mlvw5hfnr +func (d *BaiduNetdisk) requestForUploadUrl(path, uploadId string) (string, error) { + params := map[string]string{ + "method": "locateupload", + "appid": "250528", + "path": path, + "uploadid": uploadId, + "upload_version": "2.0", + } + apiUrl := "https://d.pcs.baidu.com/rest/2.0/pcs/file" + var resp UploadServerResp + _, err := d.request(apiUrl, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(params) + }, &resp) + if err != nil { + return "", err + } + // 应该是https开头的一个地址 + var uploadUrl string + if len(resp.Servers) > 0 { + uploadUrl = resp.Servers[0].Server + } else if len(resp.BakServers) > 0 { + uploadUrl = resp.BakServers[0].Server + } + if uploadUrl == "" { + return "", errors.New("upload URL is empty") + } + return uploadUrl, nil } // func encodeURIComponent(str string) string { diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index 00e36ee65..1d5d5105d 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -371,7 +371,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if err != nil { return err } - up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) + up(float64(threadG.Success()+1) * 100 / float64(len(precreateResp.BlockList)+1)) precreateResp.BlockList[i] = -1 return nil }) @@ -383,6 +383,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil } return nil, err } + defer up(100) fallthrough case 2: //step.4 创建文件 params["uploadid"] = precreateResp.UploadID diff --git a/drivers/baidu_photo/meta.go b/drivers/baidu_photo/meta.go index 91997c9d7..6c6ca5c3a 100644 --- a/drivers/baidu_photo/meta.go +++ b/drivers/baidu_photo/meta.go @@ -18,8 +18,9 @@ type Addition struct { } var config = driver.Config{ - Name: "BaiduPhoto", - LocalSort: true, + Name: "BaiduPhoto", + LocalSort: true, + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/base/client.go b/drivers/base/client.go index a91d62541..196b078da 100644 --- a/drivers/base/client.go +++ b/drivers/base/client.go @@ -15,9 +15,12 @@ var ( RestyClient *resty.Client HttpClient *http.Client ) -var UserAgent = "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0" + var DefaultTimeout = time.Second * 30 +const UserAgent = "Mozilla/5.0 (Macintosh; Apple macOS 26_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/142.0.0.0 OpenList/425.6.30" +const UserAgentNT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/142.0.0.0 OpenList/425.6.30" + func InitClient() { NoRedirectClient = resty.New().SetRedirectPolicy( resty.RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { @@ -25,6 +28,7 @@ func InitClient() { }), ).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) NoRedirectClient.SetHeader("user-agent", UserAgent) + net.SetRestyProxyIfConfigured(NoRedirectClient) RestyClient = NewRestyClient() HttpClient = net.NewHttpClient() @@ -37,5 +41,7 @@ func NewRestyClient() *resty.Client { SetRetryResetReaders(true). SetTimeout(DefaultTimeout). SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) + + net.SetRestyProxyIfConfigured(client) return client } diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index cb12b29f1..dfd25d195 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -10,6 +10,7 @@ import ( "mime/multipart" "net/http" "net/url" + "strconv" "strings" "time" @@ -225,21 +226,18 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStr if resp.Result != 1 { return errors.New("get upload data error") } - body := &bytes.Buffer{} + body := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) writer := multipart.NewWriter(body) - filePart, err := writer.CreateFormFile("file", file.GetName()) - if err != nil { - return err - } - _, err = utils.CopyWithBuffer(filePart, file) + _, err = writer.CreateFormFile("file", file.GetName()) if err != nil { return err } + headSize := body.Len() err = writer.WriteField("_token", resp.Msg.Token) if err != nil { return err } - err = writer.WriteField("puid", fmt.Sprintf("%d", resp.Msg.Puid)) + err = writer.WriteField("puid", strconv.Itoa(resp.Msg.Puid)) if err != nil { fmt.Println("Error writing param2 to request body:", err) return err @@ -248,30 +246,34 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStr if err != nil { return err } + head := bytes.NewReader(body.Bytes()[:headSize]) + tail := bytes.NewReader(body.Bytes()[headSize:]) r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: &driver.SimpleReaderWithSize{ - Reader: body, - Size: int64(body.Len()), + Reader: io.MultiReader(head, file, tail), + Size: int64(body.Len()) + file.GetSize(), }, UpdateProgress: up, }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://pan-yz.chaoxing.com/upload", r) if err != nil { return err } req.Header.Set("Content-Type", writer.FormDataContentType()) - req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len())) + req.ContentLength = int64(body.Len()) + file.GetSize() resps, err := http.DefaultClient.Do(req) if err != nil { return err } defer resps.Body.Close() - bodys, err := io.ReadAll(resps.Body) + body.Reset() + _, err = body.ReadFrom(resps.Body) if err != nil { return err } var fileRsp UploadFileDataRsp - err = json.Unmarshal(bodys, &fileRsp) + err = json.Unmarshal(body.Bytes(), &fileRsp) if err != nil { return err } diff --git a/drivers/chaoxing/types.go b/drivers/chaoxing/types.go index 9e6e270d2..ca171c4d5 100644 --- a/drivers/chaoxing/types.go +++ b/drivers/chaoxing/types.go @@ -258,7 +258,7 @@ type UploadDoneParam struct { func fileToObj(f File) *model.Object { if len(f.Content.FolderName) > 0 { return &model.Object{ - ID: fmt.Sprintf("%d", f.ID), + ID: strconv.Itoa(f.ID), Name: f.Content.FolderName, Size: 0, Modified: time.UnixMilli(f.Inserttime), diff --git a/drivers/chaoxing/util.go b/drivers/chaoxing/util.go index 715c248a8..7e3067ce9 100644 --- a/drivers/chaoxing/util.go +++ b/drivers/chaoxing/util.go @@ -9,6 +9,7 @@ import ( "fmt" "mime/multipart" "net/http" + "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -172,7 +173,7 @@ func (d *ChaoXing) Login() (string, error) { return "", err } req.Header.Set("Content-Type", writer.FormDataContentType()) - req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len())) + req.Header.Set("Content-Length", strconv.Itoa(body.Len())) resp, err := http.DefaultClient.Do(req) if err != nil { return "", err diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 572f9936b..b544391dd 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" @@ -52,14 +53,11 @@ func (d *Chunk) Drop(ctx context.Context) error { return nil } +func (Addition) GetRootPath() string { + return "/" +} + func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { - if utils.PathEqual(path, "/") { - return &model.Object{ - Name: "Root", - IsFolder: true, - Path: "/", - }, nil - } remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return nil, err @@ -272,17 +270,13 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( // 检查0号块不等于-1 以支持空文件 // 如果块数量大于1 最后一块不可能为0 // 只检查中间块是否有0 - for i, l := 0, len(chunkFile.chunkSizes)-2; ; i++ { - if i == 0 { - if chunkFile.chunkSizes[i] == -1 { - return nil, fmt.Errorf("chunk part[%d] are missing", i) - } - } else if chunkFile.chunkSizes[i] == 0 { + if chunkFile.chunkSizes[0] == -1 { + return nil, fmt.Errorf("chunk part[%d] are missing", 0) + } + for i, l := 1, len(chunkFile.chunkSizes)-1; i < l; i++ { + if chunkFile.chunkSizes[i] == 0 { return nil, fmt.Errorf("chunk part[%d] are missing", i) } - if i >= l { - break - } } fileSize := chunkFile.GetSize() mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { @@ -429,9 +423,10 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream UpdateProgress: up, } dst := stdpath.Join(remoteActualPath, dstDir.GetPath(), d.ChunkPrefix+file.GetName()) + skipHookCtx := context.WithValue(ctx, conf.SkipHookKey, struct{}{}) if d.StoreHash { for ht, value := range file.GetHash().All() { - _ = op.Put(ctx, remoteStorage, dst, &stream.FileStream{ + _ = op.Put(skipHookCtx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: fmt.Sprintf("hash_%s_%s%s", ht.Name, value, d.CustomExt), Size: 1, @@ -439,7 +434,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, Mimetype: "application/octet-stream", Reader: bytes.NewReader([]byte{0}), // 兼容不支持空文件的驱动 - }, nil, true) + }, nil) } } fullPartCount := int(file.GetSize() / d.PartSize) @@ -450,7 +445,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream } partIndex := 0 for partIndex < fullPartCount { - err = op.Put(ctx, remoteStorage, dst, &stream.FileStream{ + err = op.Put(skipHookCtx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(partIndex), Size: d.PartSize, @@ -458,7 +453,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, Mimetype: file.GetMimetype(), Reader: io.LimitReader(upReader, d.PartSize), - }, nil, true) + }, nil) if err != nil { _ = op.Remove(ctx, remoteStorage, dst) return err diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 41444c023..55462be7e 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -73,6 +73,7 @@ func (d *Cloudreve) List(ctx context.Context, dir model.Obj, args model.ListArgs } src.Size = dprop.Size } + src.Path = path.Join(dir.GetPath(), src.Name) return objectToObj(src, thumb), nil }) } @@ -203,6 +204,17 @@ func (d *Cloudreve) create(ctx context.Context, dir model.Obj, file model.Obj) e }, nil) } +func (d *Cloudreve) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var r StorageDetails + d.request(http.MethodGet, "/user/storage", nil, &r) + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: r.Total, + UsedSpace: r.Used, + }, + }, nil +} + //func (d *Cloudreve) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index c9f89b2b3..db545c7a9 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -59,6 +59,7 @@ func objectToObj(f Object, t model.Thumbnail) *model.ObjThumb { Size: int64(f.Size), Modified: f.Date, IsFolder: f.Type == "dir", + Path: f.Path, }, Thumbnail: t, } @@ -68,3 +69,9 @@ type Config struct { LoginCaptcha bool `json:"loginCaptcha"` CaptchaType string `json:"captcha_type"` } + +type StorageDetails struct { + Used int64 `json:"used"` + Free int64 `json:"free"` + Total int64 `json:"total"` +} diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index c9894b3a1..6d5e5aae1 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -291,6 +291,7 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U } return nil }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), @@ -351,7 +352,9 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u default: return nil } - }, retry.Attempts(3), + }, + retry.Context(ctx), + retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) @@ -414,7 +417,9 @@ func (d *Cloudreve) upS3(ctx context.Context, stream model.FileStreamer, u Uploa etags = append(etags, etag) return nil } - }, retry.Attempts(3), + }, + retry.Context(ctx), + retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) diff --git a/drivers/cloudreve_v4/driver.go b/drivers/cloudreve_v4/driver.go index 180ca1bdd..cd5cf1b3b 100644 --- a/drivers/cloudreve_v4/driver.go +++ b/drivers/cloudreve_v4/driver.go @@ -129,20 +129,23 @@ func (d *CloudreveV4) List(ctx context.Context, dir model.Obj, args model.ListAr } } return &model.ObjThumb{ - Object: model.Object{ - ID: src.ID, - Path: src.Path, - Name: src.Name, - Size: src.Size, - Modified: src.UpdatedAt, - Ctime: src.CreatedAt, - IsFolder: src.Type == 1, - }, + Object: *fileToObject(&src), Thumbnail: thumb, }, nil }) } +func (d *CloudreveV4) Get(ctx context.Context, path string) (model.Obj, error) { + var info File + err := d.request(http.MethodGet, "/file/info", func(req *resty.Request) { + req.SetQueryParam("uri", d.RootFolderPath+path) + }, &info) + if err != nil { + return nil, err + } + return fileToObject(&info), nil +} + func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var url FileUrlResp err := d.request(http.MethodPost, "/file/url", func(req *resty.Request) { @@ -299,7 +302,9 @@ func (d *CloudreveV4) Put(ctx context.Context, dstDir model.Obj, file model.File case "onedrive": err = d.upOneDrive(ctx, file, u, up) case "s3": - err = d.upS3(ctx, file, u, up) + err = d.upS3(ctx, file, u, up, "s3") + case "ks3": + err = d.upS3(ctx, file, u, up, "ks3") default: return errs.NotImplement } @@ -351,7 +356,7 @@ func (d *CloudreveV4) GetDetails(ctx context.Context) (*model.StorageDetails, er return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: r.Total, - FreeSpace: r.Total - r.Used, + UsedSpace: r.Used, }, }, nil } diff --git a/drivers/cloudreve_v4/types.go b/drivers/cloudreve_v4/types.go index a10f9fe1c..b67cfc862 100644 --- a/drivers/cloudreve_v4/types.go +++ b/drivers/cloudreve_v4/types.go @@ -122,6 +122,18 @@ type File struct { PrimaryEntity string `json:"primary_entity"` } +func fileToObject(f *File) *model.Object { + return &model.Object{ + ID: f.ID, + Path: f.Path, + Name: f.Name, + Size: f.Size, + Modified: f.UpdatedAt, + Ctime: f.CreatedAt, + IsFolder: f.Type == 1, + } +} + type StoragePolicy struct { ID string `json:"id"` Name string `json:"name"` @@ -206,7 +218,7 @@ type FolderSummaryResp struct { } type CapacityResp struct { - Total uint64 `json:"total"` - Used uint64 `json:"used"` + Total int64 `json:"total"` + Used int64 `json:"used"` // StoragePackTotal uint64 `json:"storage_pack_total"` } diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go index 200dfbb69..f8fe5f269 100644 --- a/drivers/cloudreve_v4/util.go +++ b/drivers/cloudreve_v4/util.go @@ -16,6 +16,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -30,6 +31,7 @@ import ( const ( CodeLoginRequired = http.StatusUnauthorized + CodePathNotExist = 40016 // Path not exist CodeCredentialInvalid = 40020 // Failed to issue token ) @@ -101,6 +103,9 @@ func (d *CloudreveV4) _request(method string, path string, callback base.ReqCall if r.Code == CodeCredentialInvalid { return ErrorIssueToken } + if r.Code == CodePathNotExist { + return errs.ObjectNotFound + } return fmt.Errorf("%d: %s", r.Code, r.Msg) } @@ -447,7 +452,9 @@ func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u F return errors.New(up.Msg) } return nil - }, retry.Attempts(3), + }, + retry.Context(ctx), + retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) @@ -508,7 +515,9 @@ func (d *CloudreveV4) upOneDrive(ctx context.Context, file model.FileStreamer, u default: return nil } - }, retry.Attempts(3), + }, + retry.Context(ctx), + retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) @@ -525,7 +534,7 @@ func (d *CloudreveV4) upOneDrive(ctx context.Context, file model.FileStreamer, u }, nil) } -func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { +func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress, s3Type string) error { DEFAULT := int64(u.ChunkSize) ss, err := stream.NewStreamSectionReader(file, int(DEFAULT), &up) if err != nil { @@ -556,6 +565,9 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU } req.ContentLength = byteSize req.Header.Set("User-Agent", d.getUA()) + if s3Type == "ks3" { + req.Header.Set("Content-Type", "application/octet-stream") + } res, err := base.HttpClient.Do(req) if err != nil { return err @@ -572,6 +584,7 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU return nil } }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), @@ -604,7 +617,11 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU if err != nil { return err } - req.Header.Set("Content-Type", "application/xml") + if s3Type == "ks3" { + req.Header.Set("Content-Type", "application/octet-stream") + } else { + req.Header.Set("Content-Type", "application/xml") + } req.Header.Set("User-Agent", d.getUA()) res, err := base.HttpClient.Do(req) if err != nil { @@ -617,7 +634,5 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU } // 上传成功发送回调请求 - return d.request(http.MethodGet, "/callback/s3/"+u.SessionID+"/"+u.CallbackSecret, func(req *resty.Request) { - req.SetBody("{}") - }, nil) + return d.request(http.MethodGet, "/callback/"+s3Type+"/"+u.SessionID+"/"+u.CallbackSecret, nil, nil) } diff --git a/drivers/cnb_releases/driver.go b/drivers/cnb_releases/driver.go index d80e69584..09ec153c8 100644 --- a/drivers/cnb_releases/driver.go +++ b/drivers/cnb_releases/driver.go @@ -50,7 +50,8 @@ func (d *CnbReleases) Drop(ctx context.Context) error { } func (d *CnbReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - if dir.GetPath() == "/" { + dirID := dir.GetID() + if dirID == "" { // get all releases for root dir var resp ReleaseList @@ -75,36 +76,32 @@ func (d *CnbReleases) List(ctx context.Context, dir model.Obj, args model.ListAr IsFolder: true, }, nil }) - } else { - // get release info by release id - releaseID := dir.GetID() - if releaseID == "" { - return nil, errs.ObjectNotFound - } - var resp Release - err := d.Request(http.MethodGet, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { - req.SetPathParam("repo", d.Repo) - req.SetPathParam("release_id", releaseID) - }, &resp) - if err != nil { - return nil, err - } + } - return utils.SliceConvert(resp.Assets, func(src ReleaseAsset) (model.Obj, error) { - return &Object{ - Object: model.Object{ - ID: src.ID, - Path: src.Path, - Name: src.Name, - Size: src.Size, - Ctime: src.CreatedAt, - Modified: src.UpdatedAt, - IsFolder: false, - }, - ParentID: dir.GetID(), - }, nil - }) + var resp Release + err := d.Request(http.MethodGet, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { + req.SetPathParam("repo", d.Repo) + req.SetPathParam("release_id", dirID) + }, &resp) + if err != nil { + return nil, err } + + return utils.SliceConvert(resp.Assets, func(src ReleaseAsset) (model.Obj, error) { + return &Object{ + Object: model.Object{ + ID: src.ID, + Path: src.Path, + Name: src.Name, + Size: src.Size, + Ctime: src.CreatedAt, + Modified: src.UpdatedAt, + IsFolder: false, + }, + ParentID: dirID, + }, nil + }) + } func (d *CnbReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -200,15 +197,20 @@ func (d *CnbReleases) Put(ctx context.Context, dstDir model.Obj, file model.File if err != nil { return err } - head := bytes.NewReader(b.Bytes()[:headSize]) tail := bytes.NewReader(b.Bytes()[headSize:]) - rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: io.MultiReader(head, file, tail), + Size: int64(b.Len()) + file.GetSize(), + }, + UpdateProgress: up, + }) // use net/http to upload file ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(resp.ExpiresInSec+1)*time.Second) defer cancel() - req, err := http.NewRequestWithContext(ctxWithTimeout, http.MethodPost, resp.UploadURL, rateLimitedRd) + req, err := http.NewRequestWithContext(ctxWithTimeout, http.MethodPost, resp.UploadURL, r) if err != nil { return err } diff --git a/drivers/cnb_releases/meta.go b/drivers/cnb_releases/meta.go index 2894d8a2b..7301861d3 100644 --- a/drivers/cnb_releases/meta.go +++ b/drivers/cnb_releases/meta.go @@ -6,7 +6,7 @@ import ( ) type Addition struct { - driver.RootPath + driver.RootID Repo string `json:"repo" type:"string" required:"true"` Token string `json:"token" type:"string" required:"true"` UseTagName bool `json:"use_tag_name" type:"bool" default:"false" help:"Use tag name instead of release name"` @@ -14,9 +14,8 @@ type Addition struct { } var config = driver.Config{ - Name: "CNB Releases", - LocalSort: true, - DefaultRoot: "/", + Name: "CNB Releases", + LocalSort: true, } func init() { diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index b00a2ea05..9e8b5c5c7 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -3,6 +3,7 @@ package crypt import ( "bytes" "context" + "errors" "fmt" "io" stdpath "path" @@ -29,8 +30,7 @@ import ( type Crypt struct { model.Storage Addition - cipher *rcCrypt.Cipher - remoteStorage driver.Driver + cipher *rcCrypt.Cipher } const obfuscatedPrefix = "___Obfuscated___" @@ -60,15 +60,7 @@ func (d *Crypt) Init(ctx context.Context) error { } d.FileNameEncoding = utils.GetNoneEmpty(d.FileNameEncoding, "base64") d.EncryptedSuffix = utils.GetNoneEmpty(d.EncryptedSuffix, ".bin") - - op.MustSaveDriverStorage(d) - - // need remote storage exist - storage, err := fs.GetStorage(d.RemotePath, &fs.GetStoragesArgs{}) - if err != nil { - return fmt.Errorf("can't find remote storage: %w", err) - } - d.remoteStorage = storage + d.RemotePath = utils.FixAndCleanPath(d.RemotePath) p, _ := strings.CutPrefix(d.Password, obfuscatedPrefix) p2, _ := strings.CutPrefix(d.Salt, obfuscatedPrefix) @@ -108,150 +100,146 @@ func (d *Crypt) Drop(ctx context.Context) error { } func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - path := dir.GetPath() - // return d.list(ctx, d.RemotePath, path) - // remoteFull - - objs, err := fs.List(ctx, d.getPathForRemote(path, true), &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) + remoteFullPath := dir.GetPath() + objs, err := fs.List(ctx, remoteFullPath, &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) // the obj must implement the model.SetPath interface // return objs, err if err != nil { return nil, err } - var result []model.Obj + result := make([]model.Obj, 0, len(objs)) for _, obj := range objs { - if obj.IsDir() { - name, err := d.cipher.DecryptDirName(obj.GetName()) - if err != nil { - // filter illegal files - continue - } - if !d.ShowHidden && strings.HasPrefix(name, ".") { - continue - } - objRes := model.Object{ - Name: name, - Size: 0, - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - Ctime: obj.CreateTime(), - // discarding hash as it's encrypted - } - result = append(result, &objRes) - } else { - thumb, ok := model.GetThumb(obj) - size, err := d.cipher.DecryptedSize(obj.GetSize()) - if err != nil { - // filter illegal files - continue - } - name, err := d.cipher.DecryptFileName(obj.GetName()) - if err != nil { - // filter illegal files - continue - } - if !d.ShowHidden && strings.HasPrefix(name, ".") { - continue - } - objRes := model.Object{ - Name: name, - Size: size, - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - Ctime: obj.CreateTime(), - // discarding hash as it's encrypted - } - if d.Thumbnail && thumb == "" { - thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") - thumb = fmt.Sprintf("%s/d%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(thumbPath, true), - sign.Sign(thumbPath)) - } - if !ok && !d.Thumbnail { - result = append(result, &objRes) + size := obj.GetSize() + mask := model.GetObjMask(obj) + name := obj.GetName() + if mask&model.Virtual == 0 { + if obj.IsDir() { + name, err = d.cipher.DecryptDirName(model.UnwrapObjName(obj).GetName()) + if err != nil { + // filter illegal files + continue + } } else { - objWithThumb := model.ObjThumb{ - Object: objRes, - Thumbnail: model.Thumbnail{ - Thumbnail: thumb, - }, + size, err = d.cipher.DecryptedSize(size) + if err != nil { + // filter illegal files + continue + } + name, err = d.cipher.DecryptFileName(model.UnwrapObjName(obj).GetName()) + if err != nil { + // filter illegal files + continue } - result = append(result, &objWithThumb) } } + if !d.ShowHidden && strings.HasPrefix(name, ".") { + continue + } + objRes := &model.Object{ + Path: stdpath.Join(remoteFullPath, obj.GetName()), + Name: name, + Size: size, + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + Ctime: obj.CreateTime(), + Mask: mask &^ model.Temp, + // discarding hash as it's encrypted + } + if !d.Thumbnail || !strings.HasPrefix(args.ReqPath, "/") { + result = append(result, objRes) + continue + } + thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") + thumb := fmt.Sprintf("%s/d%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(thumbPath, true), + sign.Sign(thumbPath)) + result = append(result, &model.ObjThumb{ + Object: *objRes, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + }) } return result, nil } +func (a Addition) GetRootPath() string { + return a.RemotePath +} + func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) { - if utils.PathEqual(path, "/") { - return &model.Object{ - Name: "Root", - IsFolder: true, - Path: "/", - }, nil - } - remoteFullPath := "" - var remoteObj model.Obj - var err, err2 error firstTryIsFolder, secondTry := guessPath(path) - remoteFullPath = d.getPathForRemote(path, firstTryIsFolder) - remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) + remoteFullPath := stdpath.Join(d.RemotePath, d.encryptPath(path, firstTryIsFolder)) + remoteObj, err := fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) if err != nil { - if errs.IsObjectNotFound(err) && secondTry { + if errors.Is(err, errs.StorageNotFound) { + remoteFullPath = stdpath.Join(d.RemotePath, path) + remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) + if err != nil { + // 可能是 虚拟路径+开启文件夹加密：返回NotSupport让op.Get去尝试op.List查找 + return nil, errs.NotSupport + } + } else if secondTry && errs.IsObjectNotFound(err) { // try the opposite - remoteFullPath = d.getPathForRemote(path, !firstTryIsFolder) - remoteObj, err2 = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) - if err2 != nil { - return nil, err2 + remoteFullPath = stdpath.Join(d.RemotePath, d.encryptPath(path, !firstTryIsFolder)) + remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, err } } else { return nil, err } } - var size int64 = 0 - name := "" - if !remoteObj.IsDir() { - size, err = d.cipher.DecryptedSize(remoteObj.GetSize()) - if err != nil { - log.Warnf("DecryptedSize failed for %s ,will use original size, err:%s", path, err) - size = remoteObj.GetSize() - } - name, err = d.cipher.DecryptFileName(remoteObj.GetName()) - if err != nil { - log.Warnf("DecryptFileName failed for %s ,will use original name, err:%s", path, err) - name = remoteObj.GetName() - } - } else { - name, err = d.cipher.DecryptDirName(remoteObj.GetName()) - if err != nil { - log.Warnf("DecryptDirName failed for %s ,will use original name, err:%s", path, err) - name = remoteObj.GetName() + + size := remoteObj.GetSize() + name := remoteObj.GetName() + mask := model.GetObjMask(remoteObj) &^ model.Temp + if mask&model.Virtual == 0 { + if !remoteObj.IsDir() { + decryptedSize, err := d.cipher.DecryptedSize(size) + if err != nil { + log.Warnf("DecryptedSize failed for %s ,will use original size, err:%s", path, err) + } else { + size = decryptedSize + } + decryptedName, err := d.cipher.DecryptFileName(model.UnwrapObjName(remoteObj).GetName()) + if err != nil { + log.Warnf("DecryptFileName failed for %s ,will use original name, err:%s", path, err) + } else { + name = decryptedName + } + } else { + decryptedName, err := d.cipher.DecryptDirName(model.UnwrapObjName(remoteObj).GetName()) + if err != nil { + log.Warnf("DecryptDirName failed for %s ,will use original name, err:%s", path, err) + } else { + name = decryptedName + } } } - obj := &model.Object{ - Path: path, + return &model.Object{ + Path: remoteFullPath, Name: name, Size: size, Modified: remoteObj.ModTime(), IsFolder: remoteObj.IsDir(), - } - return obj, nil - // return nil, errs.ObjectNotFound + Ctime: remoteObj.CreateTime(), + Mask: mask, + }, nil } // https://github.com/rclone/rclone/blob/v1.67.0/backend/crypt/cipher.go#L37 const fileHeaderSize = 32 -func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - dstDirActualPath, err := d.getActualPathForRemote(file.GetPath(), false) +func (d *Crypt) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(file.GetPath()) if err != nil { - return nil, fmt.Errorf("failed to convert path to remote path: %w", err) + return nil, err } - remoteLink, remoteFile, err := op.Link(ctx, d.remoteStorage, dstDirActualPath, args) + remoteLink, remoteFile, err := op.Link(ctx, remoteStorage, remoteActualPath, model.LinkArgs{}) if err != nil { return nil, err } @@ -317,35 +305,29 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } return readSeeker, nil }), - SyncClosers: utils.NewSyncClosers(remoteLink), + SyncClosers: utils.NewSyncClosers(remoteLink), + RequireReference: remoteLink.RequireReference, }, nil } func (d *Crypt) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - dstDirActualPath, err := d.getActualPathForRemote(parentDir.GetPath(), true) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(parentDir.GetPath()) if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) + return err } - dir := d.cipher.EncryptDirName(dirName) - return op.MakeDir(ctx, d.remoteStorage, stdpath.Join(dstDirActualPath, dir)) + encryptedName := d.cipher.EncryptDirName(dirName) + return op.MakeDir(ctx, remoteStorage, stdpath.Join(remoteActualPath, encryptedName)) } func (d *Crypt) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath(), srcObj.IsDir()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath(), dstDir.IsDir()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - return op.Move(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + _, err := fs.Move(ctx, srcObj.GetPath(), dstDir.GetPath()) + return err } func (d *Crypt) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - remoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath(), srcObj.IsDir()) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(srcObj.GetPath()) if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) + return err } var newEncryptedName string if srcObj.IsDir() { @@ -353,33 +335,26 @@ func (d *Crypt) Rename(ctx context.Context, srcObj model.Obj, newName string) er } else { newEncryptedName = d.cipher.EncryptFileName(newName) } - return op.Rename(ctx, d.remoteStorage, remoteActualPath, newEncryptedName) + return op.Rename(ctx, remoteStorage, remoteActualPath, newEncryptedName) } func (d *Crypt) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath(), srcObj.IsDir()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath(), dstDir.IsDir()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - return op.Copy(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + _, err := fs.Copy(ctx, srcObj.GetPath(), dstDir.GetPath()) + return err } func (d *Crypt) Remove(ctx context.Context, obj model.Obj) error { - remoteActualPath, err := d.getActualPathForRemote(obj.GetPath(), obj.IsDir()) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(obj.GetPath()) if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) + return err } - return op.Remove(ctx, d.remoteStorage, remoteActualPath) + return op.Remove(ctx, remoteStorage, remoteActualPath) } func (d *Crypt) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { - dstDirActualPath, err := d.getActualPathForRemote(dstDir.GetPath(), true) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(dstDir.GetPath()) if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) + return err } // Encrypt the data into wrappedIn @@ -403,15 +378,15 @@ func (d *Crypt) Put(ctx context.Context, dstDir model.Obj, streamer model.FileSt ForceStreamUpload: true, Exist: streamer.GetExist(), } - err = op.Put(ctx, d.remoteStorage, dstDirActualPath, streamOut, up, false) - if err != nil { - return err - } - return nil + return op.Put(ctx, remoteStorage, remoteActualPath, streamOut, up) } func (d *Crypt) GetDetails(ctx context.Context) (*model.StorageDetails, error) { - remoteDetails, err := op.GetStorageDetails(ctx, d.remoteStorage) + remoteStorage, _, err := op.GetStorageAndActualPath(d.RemotePath) + if err != nil { + return nil, errs.NotImplement + } + remoteDetails, err := op.GetStorageDetails(ctx, remoteStorage) if err != nil { return nil, err } @@ -420,8 +395,4 @@ func (d *Crypt) GetDetails(ctx context.Context) (*model.StorageDetails, error) { }, nil } -//func (d *Safe) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { -// return nil, errs.NotSupport -//} - var _ driver.Driver = (*Crypt)(nil) diff --git a/drivers/crypt/meta.go b/drivers/crypt/meta.go index b2d009ba3..6a4659910 100644 --- a/drivers/crypt/meta.go +++ b/drivers/crypt/meta.go @@ -6,11 +6,6 @@ import ( ) type Addition struct { - // Usually one of two - //driver.RootPath - //driver.RootID - // define other - FileNameEnc string `json:"filename_encryption" type:"select" required:"true" options:"off,standard,obfuscate" default:"off"` DirNameEnc string `json:"directory_name_encryption" type:"select" required:"true" options:"false,true" default:"false"` RemotePath string `json:"remote_path" required:"true" help:"This is where the encrypted data stores"` @@ -32,6 +27,7 @@ var config = driver.Config{ NoCache: true, DefaultRoot: "/", NoLinkURL: true, + CheckStatus: true, } func init() { diff --git a/drivers/crypt/util.go b/drivers/crypt/util.go index 417059d38..e0f2bef74 100644 --- a/drivers/crypt/util.go +++ b/drivers/crypt/util.go @@ -4,8 +4,6 @@ import ( stdpath "path" "path/filepath" "strings" - - "github.com/OpenListTeam/OpenList/v4/internal/op" ) // will give the best guessing based on the path @@ -15,30 +13,17 @@ func guessPath(path string) (isFolder, secondTry bool) { return true, false } lastSlash := strings.LastIndex(path, "/") - if strings.Index(path[lastSlash:], ".") < 0 { + if !strings.Contains(path[lastSlash:], ".") { //no dot, try folder then try file return true, true } return false, true } -func (d *Crypt) getPathForRemote(path string, isFolder bool) (remoteFullPath string) { - if isFolder && !strings.HasSuffix(path, "/") { - path = path + "/" +func (d *Crypt) encryptPath(path string, isFolder bool) string { + if isFolder { + return d.cipher.EncryptDirName(path) } dir, fileName := filepath.Split(path) - - remoteDir := d.cipher.EncryptDirName(dir) - remoteFileName := "" - if len(strings.TrimSpace(fileName)) > 0 { - remoteFileName = d.cipher.EncryptFileName(fileName) - } - return stdpath.Join(d.RemotePath, remoteDir, remoteFileName) - -} - -// actual path is used for internal only. any link for user should come from remoteFullPath -func (d *Crypt) getActualPathForRemote(path string, isFolder bool) (string, error) { - _, remoteActualPath, err := op.GetStorageAndActualPath(d.getPathForRemote(path, isFolder)) - return remoteActualPath, err + return stdpath.Join(d.cipher.EncryptDirName(dir), d.cipher.EncryptFileName(fileName)) } diff --git a/drivers/degoo/driver.go b/drivers/degoo/driver.go index 648a13679..4cbc8be63 100644 --- a/drivers/degoo/driver.go +++ b/drivers/degoo/driver.go @@ -201,3 +201,24 @@ func (d *Degoo) Put(ctx context.Context, dstDir model.Obj, file model.FileStream } return nil } + +func (d *Degoo) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.getUserInfo(ctx) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(quota.GetUserInfo3.UsedQuota, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse used quota: %v", err) + } + total, err := strconv.ParseInt(quota.GetUserInfo3.TotalQuota, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse total quota: %v", err) + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} diff --git a/drivers/degoo/types.go b/drivers/degoo/types.go index 9d793b764..b0420c953 100644 --- a/drivers/degoo/types.go +++ b/drivers/degoo/types.go @@ -108,3 +108,26 @@ type DegooGetBucketWriteAuth4Data struct { type DegooSetUploadFile3Data struct { SetUploadFile3 bool `json:"setUploadFile3"` } + +type DegooGetUserInfo3Data struct { + GetUserInfo3 struct { + // ID string + // FirstName string + // LastName string + // Email string + // AvatarURL string + // CountryCode string = CN + // LanguageCode string = zh-cn + // Phone string + // AccountType int + UsedQuota string `json:"UsedQuota"` + TotalQuota string `json:"TotalQuota"` + // OAuth2Provider + // GPMigrationStatus int + // FeatureNoAds bool + // FeatureTopSecret bool + // FeatureDownsampling bool + // FeatureAutomaticVideoUploads bool + // FileSizeLimit string + } `json:"getUserInfo3"` +} diff --git a/drivers/degoo/util.go b/drivers/degoo/util.go index a146e3d47..26901269c 100644 --- a/drivers/degoo/util.go +++ b/drivers/degoo/util.go @@ -460,3 +460,19 @@ func (d *Degoo) getOverlay4(ctx context.Context, id string) (DegooFileItem, erro } return resp.GetOverlay4, nil } + +func (d *Degoo) getUserInfo(ctx context.Context) (DegooGetUserInfo3Data, error) { + const query = "query GetUserInfo3($Token: String!) { getUserInfo3(Token: $Token) { UsedQuota TotalQuota } }" + variables := map[string]interface{}{ + "Token": d.AccessToken, + } + data, err := d.apiCall(ctx, "GetUserInfo3", query, variables) + var resp DegooGetUserInfo3Data + if err != nil { + return resp, err + } + if err = json.Unmarshal(data, &resp); err != nil { + return resp, fmt.Errorf("failed to parse user info: %w", err) + } + return resp, nil +} diff --git a/drivers/doubao/driver.go b/drivers/doubao/driver.go index d2ba04ea3..6d4a4fc6e 100644 --- a/drivers/doubao/driver.go +++ b/drivers/doubao/driver.go @@ -15,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" + "golang.org/x/time/rate" ) type Doubao struct { @@ -23,6 +24,7 @@ type Doubao struct { *UploadToken UserId string uploadThread int + limiter *rate.Limiter } func (d *Doubao) Config() driver.Config { @@ -61,6 +63,17 @@ func (d *Doubao) Init(ctx context.Context) error { d.UploadToken = uploadToken } + if d.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) + } + + return nil +} + +func (d *Doubao) WaitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } return nil } @@ -69,6 +82,10 @@ func (d *Doubao) Drop(ctx context.Context) error { } func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + var files []model.Obj fileList, err := d.getFiles(dir.GetID(), "") if err != nil { @@ -95,6 +112,10 @@ func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + var downloadUrl string if u, ok := file.(*Object); ok { @@ -160,6 +181,10 @@ func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var r UploadNodeResp _, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ @@ -177,6 +202,10 @@ func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var r UploadNodeResp _, err := d.request("/samantha/aispace/move_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ @@ -191,6 +220,10 @@ func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var r BaseResp _, err := d.request("/samantha/aispace/rename_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ @@ -207,6 +240,10 @@ func (d *Doubao) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, } func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var r BaseResp _, err := d.request("/samantha/aispace/delete_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}}) @@ -215,6 +252,10 @@ func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error { } func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + // 根据MIME类型确定数据类型 mimetype := file.GetMimetype() dataType := FileDataType diff --git a/drivers/doubao/meta.go b/drivers/doubao/meta.go index d2e350e3c..83dade919 100644 --- a/drivers/doubao/meta.go +++ b/drivers/doubao/meta.go @@ -10,9 +10,10 @@ type Addition struct { // driver.RootPath driver.RootID // define other - Cookie string `json:"cookie" type:"text"` - UploadThread string `json:"upload_thread" default:"3"` - DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"` + Cookie string `json:"cookie" type:"text"` + UploadThread string `json:"upload_thread" default:"3"` + DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"` + LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"` } var config = driver.Config{ @@ -23,6 +24,10 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { - return &Doubao{} + return &Doubao{ + Addition: Addition{ + LimitRate: 2, + }, + } }) } diff --git a/drivers/doubao/util.go b/drivers/doubao/util.go index f45c276f6..b1b38ee92 100644 --- a/drivers/doubao/util.go +++ b/drivers/doubao/util.go @@ -10,7 +10,6 @@ import ( "fmt" "hash/crc32" "io" - "math" "math/rand" "net/http" "net/url" @@ -18,7 +17,6 @@ import ( "sort" "strconv" "strings" - "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -62,7 +60,7 @@ const ( VideoDataType = "video" DefaultChunkSize = int64(5 * 1024 * 1024) // 5MB MaxRetryAttempts = 3 // 最大重试次数 - UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" + UserAgent = base.UserAgentNT Region = "cn-north-1" UploadTimeout = 3 * time.Minute ) @@ -486,7 +484,7 @@ func (d *Doubao) Upload(ctx context.Context, config *UploadConfig, dstDir model. "Authorization": {storeInfo.Auth}, "Content-Type": {"application/octet-stream"}, "Content-Crc32": {crc32Value}, - "Content-Length": {fmt.Sprintf("%d", file.GetSize())}, + "Content-Length": {strconv.FormatInt(file.GetSize(), 10)}, "Content-Disposition": {fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI))}, } res, err := base.HttpClient.Do(req) @@ -562,9 +560,7 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi retry.MaxJitter(200*time.Millisecond), ) - var partsMutex sync.Mutex // 并行上传所有分片 - hash := crc32.NewIEEE() for partIndex := range totalParts { if utils.IsCanceled(uploadCtx) { break @@ -578,32 +574,38 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi size = fileSize - offset } var reader io.ReadSeeker - var rateLimitedRd io.Reader crc32Value := "" threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { - if reader == nil { - var err error - reader, err = ss.GetSectionReader(offset, size) - if err != nil { - return err - } - hash.Reset() - w, err := utils.CopyWithBuffer(hash, reader) + Before: func(ctx context.Context) (err error) { + reader, err = ss.GetSectionReader(offset, size) + return + }, + Do: func(ctx context.Context) (err error) { + reader.Seek(0, io.SeekStart) + if crc32Value == "" { + // 把耗时的计算放在这里，避免阻塞其他协程 + crc32Hash := crc32.NewIEEE() + w, err := utils.CopyWithBuffer(crc32Hash, reader) if w != size { return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", size, w, err) } - crc32Value = hex.EncodeToString(hash.Sum(nil)) - rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader) + crc32Value = hex.EncodeToString(crc32Hash.Sum(nil)) + reader.Seek(0, io.SeekStart) } - return nil - }, - Do: func(ctx context.Context) error { - reader.Seek(0, io.SeekStart) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s?uploadid=%s&part_number=%d&phase=transfer", uploadUrl, uploadID, partNumber), rateLimitedRd) + req, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + uploadUrl, + driver.NewLimitedUploadStream(ctx, reader), + ) if err != nil { return err } + query := req.URL.Query() + query.Add("uploadid", uploadID) + query.Add("part_number", strconv.FormatInt(partNumber, 10)) + query.Add("phase", "transfer") + req.URL.RawQuery = query.Encode() req.Header = map[string][]string{ "Referer": {BaseURL + "/"}, "Origin": {BaseURL}, @@ -612,7 +614,7 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi "Authorization": {storeInfo.Auth}, "Content-Type": {"application/octet-stream"}, "Content-Crc32": {crc32Value}, - "Content-Length": {fmt.Sprintf("%d", size)}, + "Content-Length": {strconv.FormatInt(size, 10)}, "Content-Disposition": {fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI))}, } res, err := base.HttpClient.Do(req) @@ -629,16 +631,14 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi return fmt.Errorf("upload part failed: crc32 mismatch, expected %s, got %s", crc32Value, uploadResp.Data.Crc32) } // 记录成功上传的分片 - partsMutex.Lock() parts[partIndex] = UploadPart{ PartNumber: strconv.FormatInt(partNumber, 10), Etag: uploadResp.Data.Etag, Crc32: crc32Value, } - partsMutex.Unlock() // 更新进度 - progress := 10.0 + 90.0*float64(threadG.Success()+1)/float64(totalParts) - up(math.Min(progress, 95.0)) + progress := 95 * float64(threadG.Success()+1) / float64(totalParts) + up(progress) return nil }, After: func(err error) { diff --git a/drivers/doubao_share/driver.go b/drivers/doubao_share/driver.go index 89cda2129..4d23d7022 100644 --- a/drivers/doubao_share/driver.go +++ b/drivers/doubao_share/driver.go @@ -40,6 +40,7 @@ func (d *DoubaoShare) Drop(ctx context.Context) error { return nil } +// 潜在bug：配置二级目录时，可能会出问题 func (d *DoubaoShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { // 检查是否为根目录 if dir.GetID() == "" && dir.GetPath() == "/" { @@ -91,18 +92,17 @@ func (d *DoubaoShare) Link(ctx context.Context, file model.Obj, args model.LinkA downloadUrl = r.Data.OriginalMediaInfo.MainURL default: - var r GetFileUrlResp - _, err := d.request("/alice/message/get_file_url", http.MethodPost, func(req *resty.Request) { + var r GetDownloadInfoResp + _, err := d.request("/samantha/aispace/get_download_info", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ - "uris": []string{u.Key}, - "type": FileNodeType[u.NodeType], + "requests": []base.Json{{"node_id": file.GetID()}}, }) }, &r) if err != nil { return nil, err } - downloadUrl = r.Data.FileUrls[0].MainURL + downloadUrl = r.Data.DownloadInfos[0].MainURL } // 生成标准的Content-Disposition diff --git a/drivers/doubao_share/types.go b/drivers/doubao_share/types.go index 4ef12cedb..15e107656 100644 --- a/drivers/doubao_share/types.go +++ b/drivers/doubao_share/types.go @@ -115,14 +115,14 @@ type FilePath []struct { UpdateTime int64 `json:"update_time"` } -type GetFileUrlResp struct { +type GetDownloadInfoResp struct { BaseResp Data struct { - FileUrls []struct { - URI string `json:"uri"` - MainURL string `json:"main_url"` - BackURL string `json:"back_url"` - } `json:"file_urls"` + DownloadInfos []struct { + NodeID string `json:"node_id"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"download_infos"` } `json:"data"` } diff --git a/drivers/doubao_share/util.go b/drivers/doubao_share/util.go index 8e9887152..a52130305 100644 --- a/drivers/doubao_share/util.go +++ b/drivers/doubao_share/util.go @@ -43,7 +43,7 @@ const ( FileDataType = "file" ImgDataType = "image" VideoDataType = "video" - UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" + UserAgent = base.UserAgentNT ) func (d *DoubaoShare) request(path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { diff --git a/drivers/dropbox/util.go b/drivers/dropbox/util.go index d7404a429..ac0bcceeb 100644 --- a/drivers/dropbox/util.go +++ b/drivers/dropbox/util.go @@ -24,7 +24,6 @@ func (d *Dropbox) refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, @@ -176,12 +175,12 @@ func (d *Dropbox) finishUploadSession(ctx context.Context, toPath string, offset req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Authorization", "Bearer "+d.AccessToken) if d.RootNamespaceId != "" { - apiPathRootJson, err := d.buildPathRootHeader() - if err != nil { - return err + apiPathRootJson, err := d.buildPathRootHeader() + if err != nil { + return err + } + req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) } - req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) -} uploadFinishArgs := UploadFinishArgs{ Commit: struct { @@ -227,12 +226,12 @@ func (d *Dropbox) startUploadSession(ctx context.Context) (string, error) { req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Authorization", "Bearer "+d.AccessToken) if d.RootNamespaceId != "" { - apiPathRootJson, err := d.buildPathRootHeader() - if err != nil { - return "", err + apiPathRootJson, err := d.buildPathRootHeader() + if err != nil { + return "", err + } + req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) } - req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) -} req.Header.Set("Dropbox-API-Arg", "{\"close\":false}") res, err := base.HttpClient.Do(req) @@ -249,9 +248,8 @@ func (d *Dropbox) startUploadSession(ctx context.Context) (string, error) { } func (d *Dropbox) buildPathRootHeader() (string, error) { - return utils.Json.MarshalToString(map[string]interface{}{ - ".tag": "root", - "root": d.RootNamespaceId, - }) + return utils.Json.MarshalToString(map[string]interface{}{ + ".tag": "root", + "root": d.RootNamespaceId, + }) } - diff --git a/drivers/febbox/meta.go b/drivers/febbox/meta.go index e449ad002..cb6ff8fb1 100644 --- a/drivers/febbox/meta.go +++ b/drivers/febbox/meta.go @@ -16,9 +16,10 @@ type Addition struct { } var config = driver.Config{ - Name: "FebBox", - NoUpload: true, - DefaultRoot: "0", + Name: "FebBox", + NoUpload: true, + DefaultRoot: "0", + LinkCacheMode: driver.LinkCacheIP, } func init() { diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index f8d0e6ef5..484533786 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -60,11 +60,13 @@ func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m if entry.Name == "." || entry.Name == ".." { continue } + name := decode(entry.Name, d.Encoding) f := model.Object{ - Name: decode(entry.Name, d.Encoding), + Name: name, Size: int64(entry.Size), Modified: entry.Time, IsFolder: entry.Type == ftp.EntryTypeFolder, + Path: stdpath.Join(dir.GetPath(), name), } res = append(res, &f) } @@ -113,9 +115,7 @@ func (d *FTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*m } return &model.Link{ - RangeReader: &model.FileRangeReader{ - RangeReaderIF: stream.RateLimitRangeReaderFunc(resultRangeReader), - }, + RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader), SyncClosers: utils.NewSyncClosers(utils.CloseFunc(conn.Quit)), }, nil } diff --git a/drivers/ftp/meta.go b/drivers/ftp/meta.go index 8f30776c0..0ec0e735e 100644 --- a/drivers/ftp/meta.go +++ b/drivers/ftp/meta.go @@ -31,11 +31,11 @@ type Addition struct { } var config = driver.Config{ - Name: "FTP", - LocalSort: true, - OnlyLinkMFile: false, - DefaultRoot: "/", - NoLinkURL: true, + Name: "FTP", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: "/", + NoLinkURL: true, } func init() { diff --git a/drivers/github/types.go b/drivers/github/types.go index 6f9029d5b..d0d24839c 100644 --- a/drivers/github/types.go +++ b/drivers/github/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Links struct { @@ -36,6 +37,7 @@ func (o *Object) toModelObj() *model.Object { Size: o.Size, Modified: time.Unix(0, 0), IsFolder: o.Type == "dir", + Path: utils.FixAndCleanPath(o.Path), } } @@ -69,6 +71,7 @@ func (o *TreeObjResp) toModelObj() *model.Object { Size: o.Size, Modified: time.Unix(0, 0), IsFolder: o.Type == "tree", + Path: utils.FixAndCleanPath(o.Path), } } diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index c11c9cac2..8a8025c5a 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -51,6 +52,9 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis if d.Addition.ShowReadme { files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) } + if d.Addition.ShowSourceCode { + files = append(files, point.GetSourceCode()...) + } } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 nextDir := GetNextDir(point.Point, path) if nextDir == "" { @@ -67,7 +71,7 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis } if !hasSameDir { files = append(files, File{ - Path: path + "/" + nextDir, + Path: stdpath.Join(path, nextDir), FileName: nextDir, Size: point.GetLatestSize(), UpdateAt: point.Release.PublishedAt, @@ -102,7 +106,7 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis if !hasSameDir { files = append(files, File{ FileName: nextDir, - Path: path + "/" + nextDir, + Path: stdpath.Join(path, nextDir), Size: point.GetAllVersionSize(), UpdateAt: (*point.Releases)[0].PublishedAt, CreateAt: (*point.Releases)[0].CreatedAt, @@ -117,6 +121,10 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis } files = append(files, point.GetReleaseByTagName(tagName)...) + + if d.Addition.ShowSourceCode { + files = append(files, point.GetSourceCodeByTagName(tagName)...) + } } } } diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go index 4f3aae7fc..fc5fad695 100644 --- a/drivers/github_releases/meta.go +++ b/drivers/github_releases/meta.go @@ -6,10 +6,11 @@ import ( ) type Addition struct { - driver.RootID + driver.RootPath RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"OpenListTeam/OpenList" help:"structure:[path:]org/repo"` ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` + ShowSourceCode bool `json:"show_source_code" type:"bool" default:"false" help:"show Source code (zip/tar.gz)"` ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` } diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index 3bbf1e766..663aec77e 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -2,6 +2,7 @@ package github_releases import ( "encoding/json" + "path" "strings" "time" @@ -45,10 +46,10 @@ func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, erro // 获取最新版本 func (m *MountPoint) GetLatestRelease() []File { - files := make([]File, 0) + files := make([]File, 0, len(m.Release.Assets)) for _, asset := range m.Release.Assets { files = append(files, File{ - Path: m.Point + "/" + asset.Name, + Path: path.Join(m.Point, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", @@ -74,7 +75,7 @@ func (m *MountPoint) GetAllVersion() []File { files := make([]File, 0) for _, release := range *m.Releases { file := File{ - Path: m.Point + "/" + release.TagName, + Path: path.Join(m.Point, release.TagName), FileName: release.TagName, Size: m.GetSizeByTagName(release.TagName), Type: "dir", @@ -97,7 +98,7 @@ func (m *MountPoint) GetReleaseByTagName(tagName string) []File { files := make([]File, 0) for _, asset := range item.Assets { files = append(files, File{ - Path: m.Point + "/" + tagName + "/" + asset.Name, + Path: path.Join(m.Point, tagName, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", @@ -143,6 +144,60 @@ func (m *MountPoint) GetAllVersionSize() int64 { return size } +func (m *MountPoint) GetSourceCode() []File { + files := make([]File, 0) + + // 无法获取文件大小，此处设为 1 + files = append(files, File{ + Path: path.Join(m.Point, "Source code (zip)"), + FileName: "Source code (zip)", + Size: 1, + Type: "file", + UpdateAt: m.Release.CreatedAt, + CreateAt: m.Release.CreatedAt, + Url: m.Release.ZipballUrl, + }) + files = append(files, File{ + Path: path.Join(m.Point, "Source code (tar.gz)"), + FileName: "Source code (tar.gz)", + Size: 1, + Type: "file", + UpdateAt: m.Release.CreatedAt, + CreateAt: m.Release.CreatedAt, + Url: m.Release.TarballUrl, + }) + + return files +} + +func (m *MountPoint) GetSourceCodeByTagName(tagName string) []File { + for _, item := range *m.Releases { + if item.TagName == tagName { + files := make([]File, 0) + files = append(files, File{ + Path: path.Join(m.Point, "Source code (zip)"), + FileName: "Source code (zip)", + Size: 1, + Type: "file", + UpdateAt: item.CreatedAt, + CreateAt: item.CreatedAt, + Url: item.ZipballUrl, + }) + files = append(files, File{ + Path: path.Join(m.Point, "Source code (tar.gz)"), + FileName: "Source code (tar.gz)", + Size: 1, + Type: "file", + UpdateAt: item.CreatedAt, + CreateAt: item.CreatedAt, + Url: item.TarballUrl, + }) + return files + } + } + return nil +} + func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) []File { if m.OtherFile == nil || refresh { resp, _ := get("https://api.github.com/repos/" + m.Repo + "/contents") @@ -155,7 +210,7 @@ func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), for _, file := range *m.OtherFile { if strings.HasSuffix(file.Name, ".md") || strings.HasPrefix(file.Name, "LICENSE") { files = append(files, File{ - Path: m.Point + "/" + file.Name, + Path: path.Join(m.Point, file.Name), FileName: file.Name, Size: file.Size, Type: "file", diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 36b9ff01f..94ef854f2 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -175,23 +175,23 @@ func (d *GoogleDrive) GetDetails(ctx context.Context) (*model.StorageDetails, er if err != nil { return nil, err } - var total, used uint64 + var total, used int64 if about.StorageQuota.Limit == nil { total = 0 } else { - total, err = strconv.ParseUint(*about.StorageQuota.Limit, 10, 64) + total, err = strconv.ParseInt(*about.StorageQuota.Limit, 10, 64) if err != nil { return nil, err } } - used, err = strconv.ParseUint(about.StorageQuota.Usage, 10, 64) + used, err = strconv.ParseInt(about.StorageQuota.Usage, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 372b62540..042abafa4 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -27,6 +27,14 @@ import ( // do others that not defined in Driver interface +// Google Drive API field constants +const ( + // File list query fields + FilesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken" + // Single file query fields + FileInfoFields = "id,name,mimeType,size,md5Checksum,sha1Checksum,sha256Checksum" +) + type googleDriveServiceAccount struct { // Type string `json:"type"` // ProjectID string `json:"project_id"` @@ -50,7 +58,6 @@ func (d *GoogleDrive) refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, @@ -235,7 +242,7 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { } query := map[string]string{ "orderBy": orderBy, - "fields": "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken", + "fields": FilesListFields, "pageSize": "1000", "q": fmt.Sprintf("'%s' in parents and trashed = false", id), //"includeItemsFromAllDrives": "true", @@ -249,11 +256,82 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { return nil, err } pageToken = resp.NextPageToken + + // Batch process shortcuts, API calls only for file shortcuts + shortcutTargetIds := make([]string, 0) + shortcutIndices := make([]int, 0) + + // Collect target IDs of all file shortcuts (skip folder shortcuts) + for i := range resp.Files { + if resp.Files[i].MimeType == "application/vnd.google-apps.shortcut" && + resp.Files[i].ShortcutDetails.TargetId != "" && + resp.Files[i].ShortcutDetails.TargetMimeType != "application/vnd.google-apps.folder" { + shortcutTargetIds = append(shortcutTargetIds, resp.Files[i].ShortcutDetails.TargetId) + shortcutIndices = append(shortcutIndices, i) + } + } + + // Batch get target file info (only for file shortcuts) + if len(shortcutTargetIds) > 0 { + targetFiles := d.batchGetTargetFilesInfo(shortcutTargetIds) + // Update shortcut file info + for j, targetId := range shortcutTargetIds { + if targetFile, exists := targetFiles[targetId]; exists { + fileIndex := shortcutIndices[j] + if targetFile.Size != "" { + resp.Files[fileIndex].Size = targetFile.Size + } + if targetFile.MD5Checksum != "" { + resp.Files[fileIndex].MD5Checksum = targetFile.MD5Checksum + } + if targetFile.SHA1Checksum != "" { + resp.Files[fileIndex].SHA1Checksum = targetFile.SHA1Checksum + } + if targetFile.SHA256Checksum != "" { + resp.Files[fileIndex].SHA256Checksum = targetFile.SHA256Checksum + } + } + } + } + res = append(res, resp.Files...) } return res, nil } +// getTargetFileInfo gets target file details for shortcuts +func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) { + var targetFile File + url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s", targetId) + query := map[string]string{ + "fields": FileInfoFields, + } + _, err := d.request(url, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &targetFile) + if err != nil { + return File{}, err + } + return targetFile, nil +} + +// batchGetTargetFilesInfo batch gets target file info, sequential processing to avoid concurrency complexity +func (d *GoogleDrive) batchGetTargetFilesInfo(targetIds []string) map[string]File { + if len(targetIds) == 0 { + return make(map[string]File) + } + + result := make(map[string]File) + // Sequential processing to avoid concurrency complexity + for _, targetId := range targetIds { + file, err := d.getTargetFileInfo(targetId) + if err == nil { + result[targetId] = file + } + } + return result +} + func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer, url string, up driver.UpdateProgress) error { defaultChunkSize := d.ChunkSize * 1024 * 1024 ss, err := stream.NewStreamSectionReader(file, int(defaultChunkSize), &up) @@ -304,6 +382,7 @@ func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer, up(float64(offset+chunkSize) / float64(file.GetSize()) * 100) return nil }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) diff --git a/drivers/halalcloud_open/common.go b/drivers/halalcloud_open/common.go new file mode 100644 index 000000000..575ccff3a --- /dev/null +++ b/drivers/halalcloud_open/common.go @@ -0,0 +1,111 @@ +package halalcloudopen + +import ( + "sync" + "time" + + sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user" +) + +var ( + slicePostErrorRetryInterval = time.Second * 120 + retryTimes = 5 +) + +type halalCommon struct { + // *AuthService // 登录信息 + UserInfo *sdkUser.User // 用户信息 + refreshTokenFunc func(token string) error + // serv *AuthService + configs sync.Map +} + +func (m *halalCommon) GetAccessToken() (string, error) { + value, exists := m.configs.Load("access_token") + if !exists { + return "", nil // 如果不存在，返回空字符串 + } + return value.(string), nil // 返回配置项的值 +} + +// GetRefreshToken implements ConfigStore. +func (m *halalCommon) GetRefreshToken() (string, error) { + value, exists := m.configs.Load("refresh_token") + if !exists { + return "", nil // 如果不存在，返回空字符串 + } + return value.(string), nil // 返回配置项的值 +} + +// SetAccessToken implements ConfigStore. +func (m *halalCommon) SetAccessToken(token string) error { + m.configs.Store("access_token", token) + return nil +} + +// SetRefreshToken implements ConfigStore. +func (m *halalCommon) SetRefreshToken(token string) error { + m.configs.Store("refresh_token", token) + if m.refreshTokenFunc != nil { + return m.refreshTokenFunc(token) + } + return nil +} + +// SetToken implements ConfigStore. +func (m *halalCommon) SetToken(accessToken string, refreshToken string, expiresIn int64) error { + m.configs.Store("access_token", accessToken) + m.configs.Store("refresh_token", refreshToken) + m.configs.Store("expires_in", expiresIn) + if m.refreshTokenFunc != nil { + return m.refreshTokenFunc(refreshToken) + } + return nil +} + +// ClearConfigs implements ConfigStore. +func (m *halalCommon) ClearConfigs() error { + m.configs = sync.Map{} // 清空map + return nil +} + +// DeleteConfig implements ConfigStore. +func (m *halalCommon) DeleteConfig(key string) error { + _, exists := m.configs.Load(key) + if !exists { + return nil // 如果不存在，直接返回 + } + m.configs.Delete(key) // 删除指定的配置项 + return nil +} + +// GetConfig implements ConfigStore. +func (m *halalCommon) GetConfig(key string) (string, error) { + value, exists := m.configs.Load(key) + if !exists { + return "", nil // 如果不存在，返回空字符串 + } + return value.(string), nil // 返回配置项的值 +} + +// ListConfigs implements ConfigStore. +func (m *halalCommon) ListConfigs() (map[string]string, error) { + configs := make(map[string]string) + m.configs.Range(func(key, value interface{}) bool { + configs[key.(string)] = value.(string) // 将每个配置项添加到map中 + return true // 继续遍历 + }) + return configs, nil // 返回所有配置项 +} + +// SetConfig implements ConfigStore. +func (m *halalCommon) SetConfig(key string, value string) error { + m.configs.Store(key, value) // 使用Store方法设置或更新配置项 + return nil // 成功设置配置项后返回nil +} + +func NewHalalCommon() *halalCommon { + return &halalCommon{ + configs: sync.Map{}, + } +} diff --git a/drivers/halalcloud_open/driver.go b/drivers/halalcloud_open/driver.go new file mode 100644 index 000000000..6a74538d4 --- /dev/null +++ b/drivers/halalcloud_open/driver.go @@ -0,0 +1,29 @@ +package halalcloudopen + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + sdkClient "github.com/halalcloud/golang-sdk-lite/halalcloud/apiclient" + sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" +) + +type HalalCloudOpen struct { + *halalCommon + model.Storage + Addition + sdkClient *sdkClient.Client + sdkUserFileService *sdkUserFile.UserFileService + sdkUserService *sdkUser.UserService + uploadThread int +} + +func (d *HalalCloudOpen) Config() driver.Config { + return config +} + +func (d *HalalCloudOpen) GetAddition() driver.Additional { + return &d.Addition +} + +var _ driver.Driver = (*HalalCloudOpen)(nil) diff --git a/drivers/halalcloud_open/driver_curd_impl.go b/drivers/halalcloud_open/driver_curd_impl.go new file mode 100644 index 000000000..2bd32cf56 --- /dev/null +++ b/drivers/halalcloud_open/driver_curd_impl.go @@ -0,0 +1,127 @@ +package halalcloudopen + +import ( + "context" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + sdkModel "github.com/halalcloud/golang-sdk-lite/halalcloud/model" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" +) + +func (d *HalalCloudOpen) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj, error) { + + files := make([]model.Obj, 0) + limit := int64(100) + token := "" + + for { + result, err := d.sdkUserFileService.List(ctx, &sdkUserFile.FileListRequest{ + Parent: &sdkUserFile.File{Path: dir.GetPath()}, + ListInfo: &sdkModel.ScanListRequest{ + Limit: limit, + Token: token, + }, + }) + if err != nil { + return nil, err + } + + for i := 0; len(result.Files) > i; i++ { + files = append(files, NewObjFile(result.Files[i])) + } + + if result.ListInfo == nil || result.ListInfo.Token == "" { + break + } + token = result.ListInfo.Token + + } + return files, nil +} + +func (d *HalalCloudOpen) makeDir(ctx context.Context, dir model.Obj, name string) (model.Obj, error) { + _, err := d.sdkUserFileService.Create(ctx, &sdkUserFile.File{ + Path: dir.GetPath(), + Name: name, + }) + return nil, err +} + +func (d *HalalCloudOpen) move(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { + oldDir := obj.GetPath() + newDir := dir.GetPath() + _, err := d.sdkUserFileService.Move(ctx, &sdkUserFile.BatchOperationRequest{ + Source: []*sdkUserFile.File{ + { + Path: oldDir, + }, + }, + Dest: &sdkUserFile.File{ + Path: newDir, + }, + }) + return nil, err +} + +func (d *HalalCloudOpen) rename(ctx context.Context, obj model.Obj, name string) (model.Obj, error) { + + _, err := d.sdkUserFileService.Rename(ctx, &sdkUserFile.File{ + Path: obj.GetPath(), + Name: name, + }) + return nil, err +} + +func (d *HalalCloudOpen) copy(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { + id := obj.GetID() + sourcePath := obj.GetPath() + if len(id) > 0 { + sourcePath = "" + } + + destID := dir.GetID() + destPath := dir.GetPath() + if len(destID) > 0 { + destPath = "" + } + dest := &sdkUserFile.File{ + Path: destPath, + Identity: destID, + } + _, err := d.sdkUserFileService.Copy(ctx, &sdkUserFile.BatchOperationRequest{ + Source: []*sdkUserFile.File{ + { + Path: sourcePath, + Identity: id, + }, + }, + Dest: dest, + }) + return nil, err +} + +func (d *HalalCloudOpen) remove(ctx context.Context, obj model.Obj) error { + id := obj.GetID() + _, err := d.sdkUserFileService.Delete(ctx, &sdkUserFile.BatchOperationRequest{ + Source: []*sdkUserFile.File{ + { + Identity: id, + Path: obj.GetPath(), + }, + }, + }) + return err +} + +func (d *HalalCloudOpen) details(ctx context.Context) (*model.StorageDetails, error) { + ret, err := d.sdkUserService.GetStatisticsAndQuota(ctx) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: ret.DiskStatisticsQuota.BytesQuota, + UsedSpace: ret.DiskStatisticsQuota.BytesUsed, + }, + }, nil +} diff --git a/drivers/halalcloud_open/driver_get_link.go b/drivers/halalcloud_open/driver_get_link.go new file mode 100644 index 000000000..cecc46b38 --- /dev/null +++ b/drivers/halalcloud_open/driver_get_link.go @@ -0,0 +1,108 @@ +package halalcloudopen + +import ( + "context" + "crypto/sha1" + "io" + "strconv" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" + "github.com/rclone/rclone/lib/readers" +) + +func (d *HalalCloudOpen) getLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if args.Redirect { + // return nil, model.ErrUnsupported + fid := file.GetID() + fpath := file.GetPath() + if fid != "" { + fpath = "" + } + fi, err := d.sdkUserFileService.GetDirectDownloadAddress(ctx, &sdkUserFile.DirectDownloadRequest{ + Identity: fid, + Path: fpath, + }) + if err != nil { + return nil, err + } + expireAt := fi.ExpireAt + duration := time.Until(time.UnixMilli(expireAt)) + return &model.Link{ + URL: fi.DownloadAddress, + Expiration: &duration, + }, nil + } + result, err := d.sdkUserFileService.ParseFileSlice(ctx, &sdkUserFile.File{ + Identity: file.GetID(), + Path: file.GetPath(), + }) + if err != nil { + return nil, err + } + fileAddrs := []*sdkUserFile.SliceDownloadInfo{} + var addressDuration int64 + + nodesNumber := len(result.RawNodes) + nodesIndex := nodesNumber - 1 + startIndex, endIndex := 0, nodesIndex + for nodesIndex >= 0 { + if nodesIndex >= 200 { + endIndex = 200 + } else { + endIndex = nodesNumber + } + for ; endIndex <= nodesNumber; endIndex += 200 { + if endIndex == 0 { + endIndex = 1 + } + sliceAddress, err := d.sdkUserFileService.GetSliceDownloadAddress(ctx, &sdkUserFile.SliceDownloadAddressRequest{ + Identity: result.RawNodes[startIndex:endIndex], + Version: 1, + }) + if err != nil { + return nil, err + } + addressDuration, _ = strconv.ParseInt(sliceAddress.ExpireAt, 10, 64) + fileAddrs = append(fileAddrs, sliceAddress.Addresses...) + startIndex = endIndex + nodesIndex -= 200 + } + + } + + size, _ := strconv.ParseInt(result.FileSize, 10, 64) + chunks := getChunkSizes(result.Sizes) + resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + length := httpRange.Length + if httpRange.Length < 0 || httpRange.Start+httpRange.Length >= size { + length = size - httpRange.Start + } + oo := &openObject{ + ctx: ctx, + d: fileAddrs, + chunk: []byte{}, + chunks: chunks, + skip: httpRange.Start, + sha: result.Sha1, + shaTemp: sha1.New(), + } + + return readers.NewLimitedReadCloser(oo, length), nil + } + + var duration time.Duration + if addressDuration != 0 { + duration = time.Until(time.UnixMilli(addressDuration)) + } else { + duration = time.Until(time.Now().Add(time.Hour)) + } + + return &model.Link{ + RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader), + Expiration: &duration, + }, nil +} diff --git a/drivers/halalcloud_open/driver_init.go b/drivers/halalcloud_open/driver_init.go new file mode 100644 index 000000000..9f7026380 --- /dev/null +++ b/drivers/halalcloud_open/driver_init.go @@ -0,0 +1,50 @@ +package halalcloudopen + +import ( + "context" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/halalcloud/golang-sdk-lite/halalcloud/apiclient" + sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" +) + +func (d *HalalCloudOpen) Init(ctx context.Context) error { + if d.uploadThread < 1 || d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 3, 3 + } + if d.halalCommon == nil { + d.halalCommon = &halalCommon{ + UserInfo: &sdkUser.User{}, + refreshTokenFunc: func(token string) error { + d.Addition.RefreshToken = token + op.MustSaveDriverStorage(d) + return nil + }, + } + } + if d.Addition.RefreshToken != "" { + d.halalCommon.SetRefreshToken(d.Addition.RefreshToken) + } + timeout := d.Addition.TimeOut + if timeout <= 0 { + timeout = 60 + } + host := d.Addition.Host + if host == "" { + host = "openapi.2dland.cn" + } + + client := apiclient.NewClient(nil, host, d.Addition.ClientID, d.Addition.ClientSecret, d.halalCommon, apiclient.WithTimeout(time.Second*time.Duration(timeout))) + d.sdkClient = client + d.sdkUserFileService = sdkUserFile.NewUserFileService(client) + d.sdkUserService = sdkUser.NewUserService(client) + userInfo, err := d.sdkUserService.Get(ctx, &sdkUser.User{}) + if err != nil { + return err + } + d.halalCommon.UserInfo = userInfo + // 能够获取到用户信息，已经检查了 RefreshToken 的有效性，无需再次检查 + return nil +} diff --git a/drivers/halalcloud_open/driver_interface.go b/drivers/halalcloud_open/driver_interface.go new file mode 100644 index 000000000..0ddf248bb --- /dev/null +++ b/drivers/halalcloud_open/driver_interface.go @@ -0,0 +1,48 @@ +package halalcloudopen + +import ( + "context" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func (d *HalalCloudOpen) Drop(ctx context.Context) error { + return nil +} + +func (d *HalalCloudOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.getFiles(ctx, dir) +} + +func (d *HalalCloudOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return d.getLink(ctx, file, args) +} + +func (d *HalalCloudOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return d.makeDir(ctx, parentDir, dirName) +} + +func (d *HalalCloudOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.move(ctx, srcObj, dstDir) +} + +func (d *HalalCloudOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return d.rename(ctx, srcObj, newName) +} + +func (d *HalalCloudOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.copy(ctx, srcObj, dstDir) +} + +func (d *HalalCloudOpen) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(ctx, obj) +} + +func (d *HalalCloudOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return d.put(ctx, dstDir, stream, up) +} + +func (d *HalalCloudOpen) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + return d.details(ctx) +} diff --git a/drivers/halalcloud_open/halalcloud_upload.go b/drivers/halalcloud_open/halalcloud_upload.go new file mode 100644 index 000000000..16a342fd9 --- /dev/null +++ b/drivers/halalcloud_open/halalcloud_upload.go @@ -0,0 +1,258 @@ +package halalcloudopen + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" + "github.com/ipfs/go-cid" + log "github.com/sirupsen/logrus" +) + +func (d *HalalCloudOpen) put(ctx context.Context, dstDir model.Obj, fileStream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + + newPath := path.Join(dstDir.GetPath(), fileStream.GetName()) + + uploadTask, err := d.sdkUserFileService.CreateUploadTask(ctx, &sdkUserFile.File{ + Path: newPath, + Size: fileStream.GetSize(), + }) + if err != nil { + return nil, err + } + + if uploadTask.Created { + return nil, nil + } + + slicesList := make([]string, 0) + codec := uint64(0x55) + if uploadTask.BlockCodec > 0 { + codec = uint64(uploadTask.BlockCodec) + } + blockHashType := uploadTask.BlockHashType + mhType := uint64(0x12) + if blockHashType > 0 { + mhType = uint64(blockHashType) + } + prefix := cid.Prefix{ + Codec: codec, + MhLength: -1, + MhType: mhType, + Version: 1, + } + blockSize := uploadTask.BlockSize + // + // Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety. + // read file + bufferSize := int(blockSize) + buffer := make([]byte, bufferSize) + offset := 0 + teeReader := io.TeeReader(fileStream, driver.NewProgress(fileStream.GetSize(), up)) + for { + n, err := teeReader.Read(buffer[offset:]) // 这里 len(buf[offset:]) <= 4MB + if n > 0 { + offset += n + if offset == int(blockSize) { + uploadCid, err := postFileSlice(ctx, buffer, uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes) + if err != nil { + return nil, err + } + slicesList = append(slicesList, uploadCid.String()) + offset = 0 + } + } + + if err != nil { + if err == io.EOF { + if offset > 0 { + uploadCid, err := postFileSlice(ctx, buffer[:offset], uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes) + if err != nil { + return nil, err + } + slicesList = append(slicesList, uploadCid.String()) + } + break + } + return nil, err + } + } + newFile, err := makeFile(ctx, slicesList, uploadTask.Task, uploadTask.UploadAddress, retryTimes) + if err != nil { + return nil, err + } + + return NewObjFile(newFile), nil + +} + +func makeFile(ctx context.Context, fileSlice []string, taskID string, uploadAddress string, retry int) (*sdkUserFile.File, error) { + var lastError error = nil + for range retry { + newFile, err := doMakeFile(fileSlice, taskID, uploadAddress) + if err == nil { + return newFile, nil + } + if ctx.Err() != nil { + return nil, err + } + log.Errorf("make file slice failed, retrying... error: %s", err.Error()) + if strings.Contains(err.Error(), "not found") { + return nil, err + } + lastError = err + time.Sleep(slicePostErrorRetryInterval) + } + return nil, fmt.Errorf("mk file slice failed after %d times, error: %s", retry, lastError.Error()) +} + +func doMakeFile(fileSlice []string, taskID string, uploadAddress string) (*sdkUserFile.File, error) { + accessUrl := uploadAddress + "/" + taskID + getTimeOut := time.Minute * 2 + u, err := url.Parse(accessUrl) + if err != nil { + return nil, err + } + n, _ := json.Marshal(fileSlice) + httpRequest := http.Request{ + Method: http.MethodPost, + URL: u, + Header: map[string][]string{ + "Accept": {"application/json"}, + "Content-Type": {"application/json"}, + //"Content-Length": {strconv.Itoa(len(n))}, + }, + Body: io.NopCloser(bytes.NewReader(n)), + } + httpClient := http.Client{ + Timeout: getTimeOut, + } + httpResponse, err := httpClient.Do(&httpRequest) + if err != nil { + return nil, err + } + defer httpResponse.Body.Close() + if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated { + b, _ := io.ReadAll(httpResponse.Body) + message := string(b) + log.Errorf("make file failed, status code: %d, message: %s", httpResponse.StatusCode, message) + + return nil, fmt.Errorf("mk file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message) + } + b, _ := io.ReadAll(httpResponse.Body) + var result *UploadedFile + err = json.Unmarshal(b, &result) + if err != nil { + log.Errorf("make file failed from response, status code: %d, message: %s", httpResponse.StatusCode, string(b)) + return nil, err + } + return &sdkUserFile.File{ + Identity: result.Identity, + Path: result.Path, + Size: result.Size, + ContentIdentity: result.ContentIdentity, + }, nil +} +func postFileSlice(ctx context.Context, fileSlice []byte, taskID string, uploadAddress string, preix cid.Prefix, retry int) (cid.Cid, error) { + var lastError error = nil + for range retry { + newCid, err := doPostFileSlice(fileSlice, taskID, uploadAddress, preix) + if err == nil { + return newCid, nil + } + if ctx.Err() != nil { + return cid.Undef, err + } + time.Sleep(slicePostErrorRetryInterval) + lastError = err + } + return cid.Undef, fmt.Errorf("upload file slice failed after %d times, error: %s", retry, lastError.Error()) +} +func doPostFileSlice(fileSlice []byte, taskID string, uploadAddress string, preix cid.Prefix) (cid.Cid, error) { + // 1. sum file slice + newCid, err := preix.Sum(fileSlice) + if err != nil { + return cid.Undef, err + } + // 2. post file slice + sliceCidString := newCid.String() + // /{taskID}/{sliceID} + accessUrl := uploadAddress + "/" + taskID + "/" + sliceCidString + getTimeOut := time.Second * 30 + // get {accessUrl} in {getTimeOut} + u, err := url.Parse(accessUrl) + if err != nil { + return cid.Undef, err + } + // header: accept: application/json + // header: content-type: application/octet-stream + // header: content-length: {fileSlice.length} + // header: x-content-cid: {sliceCidString} + // header: x-task-id: {taskID} + httpRequest := http.Request{ + Method: http.MethodGet, + URL: u, + Header: map[string][]string{ + "Accept": {"application/json"}, + }, + } + httpClient := http.Client{ + Timeout: getTimeOut, + } + httpResponse, err := httpClient.Do(&httpRequest) + if err != nil { + log.Errorf("access %s failed, method: %s", accessUrl, http.MethodGet) + return cid.Undef, err + } + if httpResponse.StatusCode != http.StatusOK { + log.Errorf("access %s failed, method: %s, status code: %d", accessUrl, http.MethodGet, httpResponse.StatusCode) + return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d", httpResponse.StatusCode) + } + var result bool + b, err := io.ReadAll(httpResponse.Body) + if err != nil { + return cid.Undef, err + } + err = json.Unmarshal(b, &result) + if err != nil { + return cid.Undef, err + } + if result { + return newCid, nil + } + + httpRequest = http.Request{ + Method: http.MethodPost, + URL: u, + Header: map[string][]string{ + "Accept": {"application/json"}, + "Content-Type": {"application/octet-stream"}, + // "Content-Length": {strconv.Itoa(len(fileSlice))}, + }, + Body: io.NopCloser(bytes.NewReader(fileSlice)), + } + httpResponse, err = httpClient.Do(&httpRequest) + if err != nil { + return cid.Undef, err + } + defer httpResponse.Body.Close() + if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated { + b, _ := io.ReadAll(httpResponse.Body) + message := string(b) + log.Errorf("upload file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message) + return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message) + } + // + + return newCid, nil +} diff --git a/drivers/halalcloud_open/meta.go b/drivers/halalcloud_open/meta.go new file mode 100644 index 000000000..62f280b4e --- /dev/null +++ b/drivers/halalcloud_open/meta.go @@ -0,0 +1,40 @@ +package halalcloudopen + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // define other + RefreshToken string `json:"refresh_token" required:"false" help:"If using a personal API approach, the RefreshToken is not required."` + UploadThread int `json:"upload_thread" type:"number" default:"3" help:"1 <= thread <= 32"` + + ClientID string `json:"client_id" required:"true" default:""` + ClientSecret string `json:"client_secret" required:"true" default:""` + Host string `json:"host" required:"false" default:"openapi.2dland.cn"` + TimeOut int `json:"timeout" type:"number" default:"60" help:"timeout in seconds"` +} + +var config = driver.Config{ + Name: "HalalCloudOpen", + OnlyProxy: false, + DefaultRoot: "/", + NoLinkURL: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &HalalCloudOpen{} + }) +} + +type UploadedFile struct { + Identity string `json:"identity"` + UserIdentity string `json:"user_identity"` + Path string `json:"path"` + Size int64 `json:"size"` + ContentIdentity string `json:"content_identity"` +} diff --git a/drivers/halalcloud_open/obj_file.go b/drivers/halalcloud_open/obj_file.go new file mode 100644 index 000000000..839e9f4c9 --- /dev/null +++ b/drivers/halalcloud_open/obj_file.go @@ -0,0 +1,60 @@ +package halalcloudopen + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" +) + +type ObjFile struct { + sdkFile *sdkUserFile.File + fileSize int64 + modTime time.Time + createTime time.Time +} + +func NewObjFile(f *sdkUserFile.File) model.Obj { + ofile := &ObjFile{sdkFile: f} + ofile.fileSize = f.Size + modTimeTs := f.UpdateTs + ofile.modTime = time.UnixMilli(modTimeTs) + createTimeTs := f.CreateTs + ofile.createTime = time.UnixMilli(createTimeTs) + return ofile +} + +func (f *ObjFile) GetSize() int64 { + return f.fileSize +} + +func (f *ObjFile) GetName() string { + return f.sdkFile.Name +} + +func (f *ObjFile) ModTime() time.Time { + return f.modTime +} + +func (f *ObjFile) IsDir() bool { + return f.sdkFile.Dir +} + +func (f *ObjFile) GetHash() utils.HashInfo { + return utils.HashInfo{ + // TODO: support more hash types + } +} + +func (f *ObjFile) GetID() string { + return f.sdkFile.Identity +} + +func (f *ObjFile) GetPath() string { + return f.sdkFile.Path +} + +func (f *ObjFile) CreateTime() time.Time { + return f.createTime +} diff --git a/drivers/halalcloud_open/utils.go b/drivers/halalcloud_open/utils.go new file mode 100644 index 000000000..2cb6706a6 --- /dev/null +++ b/drivers/halalcloud_open/utils.go @@ -0,0 +1,185 @@ +package halalcloudopen + +import ( + "context" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" + "github.com/ipfs/go-cid" +) + +// get the next chunk +func (oo *openObject) getChunk(_ context.Context) (err error) { + if oo.id >= len(oo.chunks) { + return io.EOF + } + var chunk []byte + err = utils.Retry(3, time.Second, func() (err error) { + chunk, err = getRawFiles(oo.d[oo.id]) + return err + }) + if err != nil { + return err + } + oo.id++ + oo.chunk = chunk + return nil +} + +// Read reads up to len(p) bytes into p. +func (oo *openObject) Read(p []byte) (n int, err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return 0, fmt.Errorf("read on closed file") + } + // Skip data at the start if requested + for oo.skip > 0 { + //size := 1024 * 1024 + _, size, err := oo.ChunkLocation(oo.id) + if err != nil { + return 0, err + } + if oo.skip < int64(size) { + break + } + oo.id++ + oo.skip -= int64(size) + } + if len(oo.chunk) == 0 { + err = oo.getChunk(oo.ctx) + if err != nil { + return 0, err + } + if oo.skip > 0 { + oo.chunk = (oo.chunk)[oo.skip:] + oo.skip = 0 + } + } + n = copy(p, oo.chunk) + oo.shaTemp.Write(p[:n]) + oo.chunk = (oo.chunk)[n:] + return n, nil +} + +// Close closed the file - MAC errors are reported here +func (oo *openObject) Close() (err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return nil + } + // 校验Sha1 + if string(oo.shaTemp.Sum(nil)) != oo.sha { + return fmt.Errorf("failed to finish download: SHA mismatch") + } + + oo.closed = true + return nil +} + +func GetMD5Hash(text string) string { + tHash := md5.Sum([]byte(text)) + return hex.EncodeToString(tHash[:]) +} + +type chunkSize struct { + position int64 + size int +} + +type openObject struct { + ctx context.Context + mu sync.Mutex + d []*sdkUserFile.SliceDownloadInfo + id int + skip int64 + chunk []byte + chunks []chunkSize + closed bool + sha string + shaTemp hash.Hash +} + +func getChunkSizes(sliceSize []*sdkUserFile.SliceSize) (chunks []chunkSize) { + chunks = make([]chunkSize, 0) + for _, s := range sliceSize { + // 对最后一个做特殊处理 + endIndex := s.EndIndex + startIndex := s.StartIndex + if endIndex == 0 { + endIndex = startIndex + } + for j := startIndex; j <= endIndex; j++ { + size := s.Size + chunks = append(chunks, chunkSize{position: j, size: int(size)}) + } + } + return chunks +} + +func (oo *openObject) ChunkLocation(id int) (position int64, size int, err error) { + if id < 0 || id >= len(oo.chunks) { + return 0, 0, errors.New("invalid arguments") + } + + return (oo.chunks)[id].position, (oo.chunks)[id].size, nil +} + +func getRawFiles(addr *sdkUserFile.SliceDownloadInfo) ([]byte, error) { + + if addr == nil { + return nil, errors.New("addr is nil") + } + + client := http.Client{ + Timeout: time.Duration(60 * time.Second), // Set timeout to 60 seconds + } + resp, err := client.Get(addr.DownloadAddress) + if err != nil { + + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bad status: %s, body: %s", resp.Status, body) + } + + if addr.Encrypt > 0 { + cd := uint8(addr.Encrypt) + for idx := 0; idx < len(body); idx++ { + body[idx] = body[idx] ^ cd + } + } + storeType := addr.StoreType + if storeType != 10 { + + sourceCid, err := cid.Decode(addr.Identity) + if err != nil { + return nil, err + } + checkCid, err := sourceCid.Prefix().Sum(body) + if err != nil { + return nil, err + } + if !checkCid.Equals(sourceCid) { + return nil, fmt.Errorf("bad cid: %s, body: %s", checkCid.String(), body) + } + } + + return body, nil + +} diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 1c1473602..e58e01bc2 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -152,8 +152,7 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) req := base.NoRedirectClient.R() req.SetHeaders(map[string]string{ - "Referer": d.conf.site + "/", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", + "Referer": d.conf.site + "/", }) if d.Addition.Ip != "" { req.SetHeader("X-Forwarded-For", d.Addition.Ip) @@ -409,12 +408,15 @@ func (d *ILanZou) GetDetails(ctx context.Context) (*model.StorageDetails, error) if err != nil { return nil, err } - total := utils.Json.Get(res, "map", "totalSize").ToUint64() * 1024 - used := utils.Json.Get(res, "map", "usedSize").ToUint64() * 1024 + vipSize := utils.Json.Get(res, "map", "vipSize").ToInt64() * 1024 + totalSize := utils.Json.Get(res, "map", "totalSize").ToInt64() * 1024 + rewardSize := utils.Json.Get(res, "map", "rewardSize").ToInt64() * 1024 + total := totalSize + rewardSize + vipSize + used := utils.Json.Get(res, "map", "usedSize").ToInt64() * 1024 return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index 3d26ebde2..ea03cfd02 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -71,9 +71,8 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr req.SetHeaders(map[string]string{ "Origin": d.conf.site, "Referer": d.conf.site + "/", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", - "Accept-Encoding": "gzip, deflate, br, zstd", - "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,mt;q=0.5", + "Accept-Encoding": "gzip", + "Accept-Language": "zh-CN,zh;q=0.9,en-US,en;q=0.8", }) if d.Addition.Ip != "" { diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index 6fa269930..01d7c1ece 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -31,7 +31,7 @@ func (d *LanZou) GetAddition() driver.Additional { func (d *LanZou) Init(ctx context.Context) (err error) { if d.UserAgent == "" { - d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/89.0.4389.111 Safari/537.39" + d.UserAgent = base.UserAgentNT } switch d.Type { case "account": diff --git a/drivers/lanzou/meta.go b/drivers/lanzou/meta.go index 50d88082e..fca9b88f5 100644 --- a/drivers/lanzou/meta.go +++ b/drivers/lanzou/meta.go @@ -17,7 +17,7 @@ type Addition struct { SharePassword string `json:"share_password"` BaseUrl string `json:"baseUrl" required:"true" default:"https://pc.woozooo.com" help:"basic URL for file operation"` ShareUrl string `json:"shareUrl" required:"true" default:"https://pan.lanzoui.com" help:"used to get the sharing page"` - UserAgent string `json:"user_agent" required:"true" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/89.0.4389.111 Safari/537.39"` + UserAgent string `json:"user_agent" required:"true" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.39"` RepairFileInfo bool `json:"repair_file_info" help:"To use webdav, you need to enable it"` } diff --git a/drivers/lenovonas_share/driver.go b/drivers/lenovonas_share/driver.go index 0e5045340..012e2e63a 100644 --- a/drivers/lenovonas_share/driver.go +++ b/drivers/lenovonas_share/driver.go @@ -2,6 +2,7 @@ package LenovoNasShare import ( "context" + "fmt" "net/http" "net/url" "strings" @@ -47,12 +48,7 @@ func (d *LenovoNasShare) Drop(ctx context.Context) error { func (d *LenovoNasShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { d.checkStoken() // 检查stoken是否过期 - files := make([]File, 0) - - path := dir.GetPath() - if path == "" && !d.ShowRootFolder && d.RootFolderPath != "" { - path = d.RootFolderPath - } + path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) var resp Files query := map[string]string{ @@ -69,15 +65,14 @@ func (d *LenovoNasShare) List(ctx context.Context, dir model.Obj, args model.Lis return nil, err } - files = append(files, resp.Data.List...) - - return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return utils.SliceConvert(resp.Data.List, func(src File) (model.Obj, error) { if src.IsDir() { return src, nil } return &model.ObjThumb{ Object: model.Object{ Name: src.GetName(), + Path: src.GetPath(), Size: src.GetSize(), Modified: src.ModTime(), IsFolder: src.IsDir(), diff --git a/drivers/local/copy_namedpipes.go b/drivers/local/copy_namedpipes.go new file mode 100644 index 000000000..8217e3e02 --- /dev/null +++ b/drivers/local/copy_namedpipes.go @@ -0,0 +1,16 @@ +//go:build !windows && !plan9 && !netbsd && !aix && !illumos && !solaris && !js + +package local + +import ( + "os" + "path/filepath" + "syscall" +) + +func copyNamedPipe(dstPath string, mode os.FileMode, dirMode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(dstPath), dirMode); err != nil { + return err + } + return syscall.Mkfifo(dstPath, uint32(mode)) +} diff --git a/drivers/local/copy_namedpipes_x.go b/drivers/local/copy_namedpipes_x.go new file mode 100644 index 000000000..81f676f1e --- /dev/null +++ b/drivers/local/copy_namedpipes_x.go @@ -0,0 +1,9 @@ +//go:build windows || plan9 || netbsd || aix || illumos || solaris || js + +package local + +import "os" + +func copyNamedPipe(_ string, _, _ os.FileMode) error { + return nil +} diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 45a9a104a..7189648a7 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -23,7 +23,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/times" - cp "github.com/otiai10/copy" log "github.com/sirupsen/logrus" _ "golang.org/x/image/webp" ) @@ -235,6 +234,7 @@ func (d *Local) Get(ctx context.Context, path string) (model.Obj, error) { func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { fullPath := file.GetPath() link := &model.Link{} + var MFile model.File if args.Type == "thumb" && utils.Ext(file.GetName()) != "svg" { var buf *bytes.Buffer var thumbPath *string @@ -261,9 +261,9 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, err } link.ContentLength = int64(stat.Size()) - link.MFile = open + MFile = open } else { - link.MFile = bytes.NewReader(buf.Bytes()) + MFile = bytes.NewReader(buf.Bytes()) link.ContentLength = int64(buf.Len()) } } else { @@ -272,13 +272,11 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, err } link.ContentLength = file.GetSize() - link.MFile = open - } - link.AddIfCloser(link.MFile) - if !d.Config().OnlyLinkMFile { - link.RangeReader = stream.GetRangeReaderFromMFile(link.ContentLength, link.MFile) - link.MFile = nil + MFile = open } + link.SyncClosers.AddIfCloser(MFile) + link.RangeReader = stream.GetRangeReaderFromMFile(link.ContentLength, MFile) + link.RequireReference = link.SyncClosers.Length() > 0 return link, nil } @@ -298,16 +296,9 @@ func (d *Local) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return fmt.Errorf("the destination folder is a subfolder of the source folder") } err := os.Rename(srcPath, dstPath) - if err != nil && strings.Contains(err.Error(), "invalid cross-device link") { - // 跨设备移动，先复制再删除 - if err := d.Copy(ctx, srcObj, dstDir); err != nil { - return err - } - // 复制成功后直接删除源文件/文件夹 - if srcObj.IsDir() { - return os.RemoveAll(srcObj.GetPath()) - } - return os.Remove(srcObj.GetPath()) + if isCrossDeviceError(err) { + // 跨设备移动，变更为移动任务 + return errs.NotImplement } if err == nil { srcParent := filepath.Dir(srcPath) @@ -348,15 +339,14 @@ func (d *Local) Copy(_ context.Context, srcObj, dstDir model.Obj) error { if utils.IsSubPath(srcPath, dstPath) { return fmt.Errorf("the destination folder is a subfolder of the source folder") } - // Copy using otiai10/copy to perform more secure & efficient copy - err := cp.Copy(srcPath, dstPath, cp.Options{ - Sync: true, // Sync file to disk after copy, may have performance penalty in filesystem such as ZFS - PreserveTimes: true, - PreserveOwner: true, - }) + info, err := os.Lstat(srcPath) if err != nil { return err } + // 复制regular文件会返回errs.NotImplement, 转为复制任务 + if err = d.tryCopy(srcPath, dstPath, info); err != nil { + return err + } if d.directoryMap.Has(filepath.Dir(dstPath)) { d.directoryMap.UpdateDirSize(filepath.Dir(dstPath)) @@ -375,18 +365,26 @@ func (d *Local) Remove(ctx context.Context, obj model.Obj) error { err = os.Remove(obj.GetPath()) } } else { - if !utils.Exists(d.RecycleBinPath) { - err = os.MkdirAll(d.RecycleBinPath, 0o755) + objPath := obj.GetPath() + objName := obj.GetName() + var relPath string + relPath, err = filepath.Rel(d.GetRootPath(), filepath.Dir(objPath)) + if err != nil { + return err + } + recycleBinPath := filepath.Join(d.RecycleBinPath, relPath) + if !utils.Exists(recycleBinPath) { + err = os.MkdirAll(recycleBinPath, 0o755) if err != nil { return err } } - dstPath := filepath.Join(d.RecycleBinPath, obj.GetName()) + dstPath := filepath.Join(recycleBinPath, objName) if utils.Exists(dstPath) { - dstPath = filepath.Join(d.RecycleBinPath, obj.GetName()+"_"+time.Now().Format("20060102150405")) + dstPath = filepath.Join(recycleBinPath, objName+"_"+time.Now().Format("20060102150405")) } - err = os.Rename(obj.GetPath(), dstPath) + err = os.Rename(objPath, dstPath) } if err != nil { return err diff --git a/drivers/local/meta.go b/drivers/local/meta.go index b16f3ca5f..a27e9eecd 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -18,12 +18,12 @@ type Addition struct { } var config = driver.Config{ - Name: "Local", - OnlyLinkMFile: false, - LocalSort: true, - NoCache: true, - DefaultRoot: "/", - NoLinkURL: true, + Name: "Local", + LocalSort: true, + OnlyProxy: true, + NoCache: true, + DefaultRoot: "/", + NoLinkURL: true, } func init() { diff --git a/drivers/local/util.go b/drivers/local/util.go index cbf73ad52..be86d9c9b 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -3,6 +3,7 @@ package local import ( "bytes" "encoding/json" + "errors" "fmt" "io/fs" "os" @@ -14,7 +15,9 @@ import ( "strings" "sync" + "github.com/KarpelesLab/reflink" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/disintegration/imaging" @@ -148,7 +151,7 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { return nil, nil, err } if d.ThumbCacheFolder != "" { - err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0666) + err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0o666) if err != nil { return nil, nil, err } @@ -405,3 +408,79 @@ func (m *DirectoryMap) DeleteDirNode(dirname string) error { return nil } + +func (d *Local) tryCopy(srcPath, dstPath string, info os.FileInfo) error { + if info.Mode()&os.ModeDevice != 0 { + return errors.New("cannot copy a device") + } else if info.Mode()&os.ModeSymlink != 0 { + return d.copySymlink(srcPath, dstPath) + } else if info.Mode()&os.ModeNamedPipe != 0 { + return copyNamedPipe(dstPath, info.Mode(), os.FileMode(d.mkdirPerm)) + } else if info.IsDir() { + return d.recurAndTryCopy(srcPath, dstPath) + } else { + return tryReflinkCopy(srcPath, dstPath) + } +} + +func (d *Local) copySymlink(srcPath, dstPath string) error { + linkOrig, err := os.Readlink(srcPath) + if err != nil { + return err + } + dstDir := filepath.Dir(dstPath) + if !filepath.IsAbs(linkOrig) { + srcDir := filepath.Dir(srcPath) + rel, err := filepath.Rel(dstDir, srcDir) + if err != nil { + rel, err = filepath.Abs(srcDir) + } + if err != nil { + return err + } + linkOrig = filepath.Clean(filepath.Join(rel, linkOrig)) + } + err = os.MkdirAll(dstDir, os.FileMode(d.mkdirPerm)) + if err != nil { + return err + } + return os.Symlink(linkOrig, dstPath) +} + +func (d *Local) recurAndTryCopy(srcPath, dstPath string) error { + err := os.MkdirAll(dstPath, os.FileMode(d.mkdirPerm)) + if err != nil { + return err + } + files, err := readDir(srcPath) + if err != nil { + return err + } + for _, f := range files { + if !f.IsDir() { + sp := filepath.Join(srcPath, f.Name()) + dp := filepath.Join(dstPath, f.Name()) + if err = d.tryCopy(sp, dp, f); err != nil { + return err + } + } + } + for _, f := range files { + if f.IsDir() { + sp := filepath.Join(srcPath, f.Name()) + dp := filepath.Join(dstPath, f.Name()) + if err = d.recurAndTryCopy(sp, dp); err != nil { + return err + } + } + } + return nil +} + +func tryReflinkCopy(srcPath, dstPath string) error { + err := reflink.Always(srcPath, dstPath) + if errors.Is(err, reflink.ErrReflinkUnsupported) || errors.Is(err, reflink.ErrReflinkFailed) || isCrossDeviceError(err) { + return errs.NotImplement + } + return err +} diff --git a/drivers/local/util_unix.go b/drivers/local/util_unix.go index 3362df341..6f16a7c99 100644 --- a/drivers/local/util_unix.go +++ b/drivers/local/util_unix.go @@ -3,11 +3,13 @@ package local import ( + "errors" "io/fs" "strings" "syscall" "github.com/OpenListTeam/OpenList/v4/internal/model" + "golang.org/x/sys/unix" ) func isHidden(f fs.FileInfo, _ string) bool { @@ -20,10 +22,14 @@ func getDiskUsage(path string) (model.DiskUsage, error) { if err != nil { return model.DiskUsage{}, err } - total := stat.Blocks * uint64(stat.Bsize) - free := stat.Bfree * uint64(stat.Bsize) + total := int64(stat.Blocks) * int64(stat.Bsize) + free := int64(stat.Bfree) * int64(stat.Bsize) return model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: total - free, }, nil } + +func isCrossDeviceError(err error) bool { + return errors.Is(err, unix.EXDEV) +} diff --git a/drivers/local/util_windows.go b/drivers/local/util_windows.go index 370640098..9c7b4c070 100644 --- a/drivers/local/util_windows.go +++ b/drivers/local/util_windows.go @@ -45,7 +45,11 @@ func getDiskUsage(path string) (model.DiskUsage, error) { return model.DiskUsage{}, err } return model.DiskUsage{ - TotalSpace: totalBytes, - FreeSpace: freeBytes, + TotalSpace: int64(totalBytes), + UsedSpace: int64(totalBytes - freeBytes), }, nil } + +func isCrossDeviceError(err error) bool { + return errors.Is(err, windows.ERROR_NOT_SAME_DEVICE) +} diff --git a/drivers/mediafire/driver.go b/drivers/mediafire/driver.go index bd2502590..49d431021 100644 --- a/drivers/mediafire/driver.go +++ b/drivers/mediafire/driver.go @@ -60,20 +60,24 @@ func (d *Mediafire) GetAddition() driver.Additional { // Init initializes the MediaFire driver with session token and cookie validation func (d *Mediafire) Init(ctx context.Context) error { - if d.SessionToken == "" { - return fmt.Errorf("Init :: [MediaFire] {critical} missing sessionToken") - } - if d.Cookie == "" { return fmt.Errorf("Init :: [MediaFire] {critical} missing Cookie") } + + // If SessionToken is empty, try to get it from cookie + if d.SessionToken == "" { + if _, err := d.getSessionToken(ctx); err != nil { + return fmt.Errorf("Init :: [MediaFire] {critical} failed to get session token from cookie: %w", err) + } + } + // Setup rate limiter if rate limit is configured if d.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) } + // Validate and refresh session token if needed if _, err := d.getSessionToken(ctx); err != nil { - d.renewToken(ctx) // Avoids 10 mins token expiry (6- 9) @@ -387,8 +391,8 @@ func (d *Mediafire) Put(ctx context.Context, dstDir model.Obj, file model.FileSt } } else { pollKey = checkResp.Response.ResumableUpload.UploadKey - up(100.0) } + defer up(100.0) pollResp, err := d.pollUpload(ctx, pollKey) if err != nil { @@ -412,18 +416,18 @@ func (d *Mediafire) GetDetails(ctx context.Context) (*model.StorageDetails, erro if err != nil { return nil, err } - used, err := strconv.ParseUint(resp.Response.UserInfo.UsedStorageSize, 10, 64) + used, err := strconv.ParseInt(resp.Response.UserInfo.UsedStorageSize, 10, 64) if err != nil { return nil, err } - total, err := strconv.ParseUint(resp.Response.UserInfo.StorageLimit, 10, 64) + total, err := strconv.ParseInt(resp.Response.UserInfo.StorageLimit, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/mediafire/meta.go b/drivers/mediafire/meta.go index e80b11f1e..1a3d37c8f 100644 --- a/drivers/mediafire/meta.go +++ b/drivers/mediafire/meta.go @@ -15,6 +15,7 @@ Final opts by @Suyunjing @j2rong4cn @KirCute @Da3zKi7 */ import ( + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) @@ -23,8 +24,8 @@ type Addition struct { driver.RootPath //driver.RootID - SessionToken string `json:"session_token" required:"true" type:"string" help:"Required for MediaFire API"` - Cookie string `json:"cookie" required:"true" type:"string" help:"Required for navigation"` + SessionToken string `json:"session_token" required:"false" type:"string" help:"Optional for MediaFire API, can be auto-acquired from cookie"` + Cookie string `json:"cookie" required:"true" type:"string" help:"Required for MediaFire API authentication"` OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` @@ -36,7 +37,6 @@ type Addition struct { var config = driver.Config{ Name: "MediaFire", LocalSort: false, - OnlyLinkMFile: false, OnlyProxy: false, NoCache: false, NoUpload: false, @@ -50,13 +50,12 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { return &Mediafire{ - appBase: "https://app.mediafire.com", - apiBase: "https://www.mediafire.com/api/1.5", - hostBase: "https://www.mediafire.com", - maxRetries: 3, - secChUa: "\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"139\", \"Google Chrome\";v=\"139\"", - secChUaPlatform: "Windows", - userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36", + appBase: "https://app.mediafire.com", + apiBase: "https://www.mediafire.com/api/1.5", + hostBase: "https://www.mediafire.com", + maxRetries: 3, + userAgent: base.UserAgent, } }) } + diff --git a/drivers/mediafire/types.go b/drivers/mediafire/types.go index 4a59ae568..eebc85460 100644 --- a/drivers/mediafire/types.go +++ b/drivers/mediafire/types.go @@ -244,3 +244,4 @@ type MediafireUserInfoResponse struct { CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } + diff --git a/drivers/mediafire/util.go b/drivers/mediafire/util.go index 6ded6bdf0..24259d0bc 100644 --- a/drivers/mediafire/util.go +++ b/drivers/mediafire/util.go @@ -15,6 +15,7 @@ Final opts by @Suyunjing @j2rong4cn @KirCute @Da3zKi7 */ import ( + "compress/gzip" "context" "encoding/json" "fmt" @@ -60,7 +61,7 @@ func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { } req.Header.Set("Accept", "*/*") - req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") + req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("Accept-Language", "en-US,en;q=0.9") req.Header.Set("Content-Length", "0") req.Header.Set("Cookie", d.Cookie) @@ -83,7 +84,19 @@ func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + var body []byte + // Handle gzip decompression if needed + if resp.Header.Get("Content-Encoding") == "gzip" { + gzipReader, err := gzip.NewReader(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzipReader.Close() + body, _ = io.ReadAll(gzipReader) + } else { + body, err = io.ReadAll(resp.Body) + } + if err != nil { return "", err } @@ -468,39 +481,32 @@ func (d *Mediafire) uploadUnits(ctx context.Context, file model.FileStreamer, ch } var reader io.ReadSeeker - var rateLimitedRd io.Reader var unitHash string // Use lifecycle pattern for proper resource management threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { + Before: func(ctx context.Context) (err error) { // Skip already uploaded units if d.isUnitUploaded(intWords, unitID) { return ss.DiscardSection(start, size) } - - var err error reader, err = ss.GetSectionReader(start, size) - if err != nil { - return err - } - rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader) - return nil + return }, - Do: func(ctx context.Context) error { + Do: func(ctx context.Context) (err error) { if reader == nil { return nil // Skip if reader is not initialized (already uploaded) } + reader.Seek(0, io.SeekStart) if unitHash == "" { - reader.Seek(0, io.SeekStart) var err error unitHash, err = utils.HashReader(utils.SHA256, reader) if err != nil { return err } + reader.Seek(0, io.SeekStart) } - reader.Seek(0, io.SeekStart) // Perform upload @@ -515,7 +521,7 @@ func (d *Mediafire) uploadUnits(ctx context.Context, file model.FileStreamer, ch } url := d.apiBase + "/upload/resumable.php" - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, rateLimitedRd) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } @@ -579,10 +585,10 @@ func (d *Mediafire) uploadUnits(ctx context.Context, file model.FileStreamer, ch finalUploadKey = uploadResp.Response.Doupload.Key keyMutex.Unlock() + up(float64(threadG.Success()+1) * 100 / float64(numUnits+1)) return nil }, After: func(err error) { - up(float64(threadG.Success()) * 100 / float64(numUnits)) if reader != nil { // Cleanup resources ss.FreeSectionReader(reader) @@ -594,7 +600,6 @@ func (d *Mediafire) uploadUnits(ctx context.Context, file model.FileStreamer, ch if err := threadG.Wait(); err != nil { return "", err } - return finalUploadKey, nil } @@ -727,3 +732,4 @@ func (d *Mediafire) getFileByHash(ctx context.Context, hash string) (*model.ObjT file := resp.Response.FileInfo[0] return d.fileToObj(file), nil } + diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index 996f4dce5..f06b3fa98 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -152,7 +152,7 @@ func (d *Mega) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { func (d *Mega) Remove(ctx context.Context, obj model.Obj) error { if node, ok := obj.(*MegaNode); ok { - return d.c.Delete(node.n, false) + return d.c.Delete(node.n, !d.MoveToTrash) } return fmt.Errorf("unable to convert dir to mega n") } @@ -195,6 +195,19 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea return fmt.Errorf("unable to convert dir to mega n") } +func (d *Mega) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.c.GetQuota() + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: int64(quota.Mstrg), + UsedSpace: int64(quota.Cstrg), + }, + }, nil +} + //func (d *Mega) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/mega/meta.go b/drivers/mega/meta.go index 552c71621..17db29d75 100644 --- a/drivers/mega/meta.go +++ b/drivers/mega/meta.go @@ -13,6 +13,7 @@ type Addition struct { Password string `json:"password" required:"true"` TwoFACode string `json:"two_fa_code" required:"false" help:"2FA 6-digit code, filling in the 2FA code alone will not support reloading driver"` TwoFASecret string `json:"two_fa_secret" required:"false" help:"2FA secret"` + MoveToTrash bool `json:"move_to_trash" default:"true" help:"move to trash when deleting files"` } var config = driver.Config{ diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go index 5e7a0d8db..35ebe96cc 100644 --- a/drivers/misskey/util.go +++ b/drivers/misskey/util.go @@ -57,18 +57,22 @@ func setBody(body interface{}) base.ReqCallback { } func handleFolderId(dir model.Obj) interface{} { - if dir.GetID() == "" { - return nil + if isRootFolder(dir) { + return nil // Root folder doesn't need folderId } return dir.GetID() } +func isRootFolder(dir model.Obj) bool { + return dir.GetID() == "" +} + // API layer methods func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { var files []MFile var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} @@ -85,7 +89,7 @@ func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) { var folders []MFolder var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} @@ -197,16 +201,24 @@ func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileSt Reader: stream, UpdateProgress: up, }) + + // Build form data, only add folderId if not root folder + formData := map[string]string{ + "name": stream.GetName(), + "comment": "", + "isSensitive": "false", + "force": "false", + } + + folderId := handleFolderId(dstDir) + if folderId != nil { + formData["folderId"] = folderId.(string) + } + req := base.RestyClient.R(). SetContext(ctx). SetFileReader("file", stream.GetName(), reader). - SetFormData(map[string]string{ - "folderId": handleFolderId(dstDir).(string), - "name": stream.GetName(), - "comment": "", - "isSensitive": "false", - "force": "false", - }). + SetFormData(formData). SetResult(&file). SetAuthToken(d.AccessToken) diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index c611cf029..bab15c1fd 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -330,7 +330,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if resp.StatusCode != http.StatusOK { return fmt.Errorf("upload err,code=%d", resp.StatusCode) } - up(100 * float64(threadG.Success()) / float64(len(parts))) + up(100 * float64(threadG.Success()+1) / float64(len(parts)+1)) initUpdload.PartInfos[i] = "" return nil }) @@ -356,6 +356,19 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre }, nil } +func (d *MoPan) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.client.UsedSpace() + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: int64(quota.Capacity), + UsedSpace: int64(quota.Used), + }, + }, nil +} + var _ driver.Driver = (*MoPan)(nil) var _ driver.MkdirResult = (*MoPan)(nil) var _ driver.MoveResult = (*MoPan)(nil) diff --git a/drivers/netease_music/driver.go b/drivers/netease_music/driver.go index b03c3b2cc..4fcf433c1 100644 --- a/drivers/netease_music/driver.go +++ b/drivers/netease_music/driver.go @@ -42,14 +42,11 @@ func (d *NeteaseMusic) Drop(ctx context.Context) error { return nil } -func (d *NeteaseMusic) Get(ctx context.Context, path string) (model.Obj, error) { - if path == "/" { - return &model.Object{ - IsFolder: true, - Path: path, - }, nil - } +func (Addition) GetRootPath() string { + return "/" +} +func (d *NeteaseMusic) Get(ctx context.Context, path string) (model.Obj, error) { fragments := strings.Split(path, "/") if len(fragments) > 1 { fileName := fragments[1] diff --git a/drivers/onedrive/driver.go b/drivers/onedrive/driver.go index 0af2f8374..e01c1f7c1 100644 --- a/drivers/onedrive/driver.go +++ b/drivers/onedrive/driver.go @@ -22,6 +22,7 @@ type Onedrive struct { AccessToken string root *Object mutex sync.Mutex + ref *Onedrive } func (d *Onedrive) Config() driver.Config { @@ -36,10 +37,22 @@ func (d *Onedrive) Init(ctx context.Context) error { if d.ChunkSize < 1 { d.ChunkSize = 5 } + if d.ref != nil { + return nil + } return d.refreshToken() } +func (d *Onedrive) InitReference(refStorage driver.Driver) error { + if ref, ok := refStorage.(*Onedrive); ok { + d.ref = ref + return nil + } + return errs.NotSupport +} + func (d *Onedrive) Drop(ctx context.Context) error { + d.ref = nil return nil } @@ -85,7 +98,9 @@ func (d *Onedrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src, dir.GetID()), nil + obj := fileToObj(src, dir.GetID()) + obj.Path = path.Join(dir.GetPath(), obj.GetName()) + return obj, nil }) } @@ -218,9 +233,24 @@ func (d *Onedrive) GetDetails(ctx context.Context) (*model.StorageDetails, error return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: drive.Quota.Total, - FreeSpace: drive.Quota.Remaining, + UsedSpace: drive.Quota.Used, }, }, nil } +func (d *Onedrive) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +// GetDirectUploadInfo returns the direct upload info for OneDrive +func (d *Onedrive) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + return d.getDirectUploadInfo(ctx, path.Join(dstDir.GetPath(), fileName)) +} + var _ driver.Driver = (*Onedrive)(nil) diff --git a/drivers/onedrive/meta.go b/drivers/onedrive/meta.go index 3d9ca09eb..00f46bfcc 100644 --- a/drivers/onedrive/meta.go +++ b/drivers/onedrive/meta.go @@ -7,19 +7,20 @@ import ( type Addition struct { driver.RootPath - Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` - TenantID string `json:"tenant_id"` - IsSharepoint bool `json:"is_sharepoint"` - UseOnlineAPI bool `json:"use_online_api" default:"true"` - APIAddress string `json:"api_url_address" default:"https://api.oplist.org/onedrive/renewapi"` - ClientID string `json:"client_id"` - ClientSecret string `json:"client_secret"` - RedirectUri string `json:"redirect_uri" required:"true" default:"https://api.oplist.org/onedrive/callback"` - RefreshToken string `json:"refresh_token" required:"true"` - SiteId string `json:"site_id"` - ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` - CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` - DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` + IsSharepoint bool `json:"is_sharepoint"` + TenantID string `json:"tenant_id"` + UseOnlineAPI bool `json:"use_online_api" default:"true"` + APIAddress string `json:"api_url_address" default:"https://api.oplist.org/onedrive/renewapi"` + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + RedirectUri string `json:"redirect_uri" required:"true" default:"https://api.oplist.org/onedrive/callback"` + RefreshToken string `json:"refresh_token" required:"true"` + SiteId string `json:"site_id"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` + CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` + DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"` } var config = driver.Config{ diff --git a/drivers/onedrive/types.go b/drivers/onedrive/types.go index a00241eb9..78baa07e4 100644 --- a/drivers/onedrive/types.go +++ b/drivers/onedrive/types.go @@ -95,9 +95,9 @@ type DriveResp struct { DriveType string `json:"driveType"` Quota struct { Deleted uint64 `json:"deleted"` - Remaining uint64 `json:"remaining"` + Remaining int64 `json:"remaining"` State string `json:"state"` - Total uint64 `json:"total"` - Used uint64 `json:"used"` + Total int64 `json:"total"` + Used int64 `json:"used"` } `json:"quota"` } diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 8d71bd418..c3ce9c7dd 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -82,7 +82,6 @@ func (d *Onedrive) _refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, @@ -140,7 +139,10 @@ func (d *Onedrive) _refreshToken() error { return nil } -func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { +func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { + if d.ref != nil { + return d.ref.Request(url, method, callback, resp) + } req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { @@ -156,7 +158,7 @@ func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, return nil, err } if e.Error.Code != "" { - if e.Error.Code == "InvalidAuthenticationToken" { + if e.Error.Code == "InvalidAuthenticationToken" && !utils.IsBool(noRetry...) { err = d.refreshToken() if err != nil { return nil, err @@ -289,6 +291,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil return nil } }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), @@ -314,9 +317,36 @@ func (d *Onedrive) getDrive(ctx context.Context) (*DriveResp, error) { var resp DriveResp _, err := d.Request(api, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, &resp) + }, &resp, true) if err != nil { return nil, err } return &resp, nil } + +func (d *Onedrive) getDirectUploadInfo(ctx context.Context, path string) (*model.HttpDirectUploadInfo, error) { + // Create upload session + url := d.GetMetaUrl(false, path) + "/createUploadSession" + metadata := map[string]any{ + "item": map[string]any{ + "@microsoft.graph.conflictBehavior": "rename", + }, + } + + res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) + }, nil) + if err != nil { + return nil, err + } + + uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() + if uploadUrl == "" { + return nil, fmt.Errorf("failed to get upload URL from response") + } + return &model.HttpDirectUploadInfo{ + UploadURL: uploadUrl, + ChunkSize: d.ChunkSize * 1024 * 1024, // Convert MB to bytes + Method: "PUT", + }, nil +} diff --git a/drivers/onedrive_app/driver.go b/drivers/onedrive_app/driver.go index f28adde0a..063552308 100644 --- a/drivers/onedrive_app/driver.go +++ b/drivers/onedrive_app/driver.go @@ -85,7 +85,9 @@ func (d *OnedriveAPP) List(ctx context.Context, dir model.Obj, args model.ListAr return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src, dir.GetID()), nil + obj := fileToObj(src, dir.GetID()) + obj.Path = path.Join(dir.GetPath(), obj.GetName()) + return obj, nil }) } @@ -217,9 +219,23 @@ func (d *OnedriveAPP) GetDetails(ctx context.Context) (*model.StorageDetails, er return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: drive.Quota.Total, - FreeSpace: drive.Quota.Remaining, + UsedSpace: drive.Quota.Used, }, }, nil } +func (d *OnedriveAPP) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +func (d *OnedriveAPP) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + return d.getDirectUploadInfo(ctx, path.Join(dstDir.GetPath(), fileName)) +} + var _ driver.Driver = (*OnedriveAPP)(nil) diff --git a/drivers/onedrive_app/meta.go b/drivers/onedrive_app/meta.go index 32694e6d7..c3f0c02ec 100644 --- a/drivers/onedrive_app/meta.go +++ b/drivers/onedrive_app/meta.go @@ -7,14 +7,15 @@ import ( type Addition struct { driver.RootPath - Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` - ClientID string `json:"client_id" required:"true"` - ClientSecret string `json:"client_secret" required:"true"` - TenantID string `json:"tenant_id"` - Email string `json:"email"` - ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` - CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` - DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` + ClientID string `json:"client_id" required:"true"` + ClientSecret string `json:"client_secret" required:"true"` + TenantID string `json:"tenant_id"` + Email string `json:"email"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` + CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` + DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"` } var config = driver.Config{ diff --git a/drivers/onedrive_app/types.go b/drivers/onedrive_app/types.go index 438eddfb8..935017853 100644 --- a/drivers/onedrive_app/types.go +++ b/drivers/onedrive_app/types.go @@ -78,9 +78,9 @@ type DriveResp struct { DriveType string `json:"driveType"` Quota struct { Deleted uint64 `json:"deleted"` - Remaining uint64 `json:"remaining"` + Remaining int64 `json:"remaining"` State string `json:"state"` - Total uint64 `json:"total"` - Used uint64 `json:"used"` + Total int64 `json:"total"` + Used int64 `json:"used"` } `json:"quota"` } diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 4cc6b9c31..574afb381 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -88,7 +88,7 @@ func (d *OnedriveAPP) _accessToken() error { return nil } -func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { +func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { @@ -104,7 +104,7 @@ func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallba return nil, err } if e.Error.Code != "" { - if e.Error.Code == "InvalidAuthenticationToken" { + if e.Error.Code == "InvalidAuthenticationToken" && !utils.IsBool(noRetry...) { err = d.accessToken() if err != nil { return nil, err @@ -196,6 +196,7 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. return nil } }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), @@ -216,9 +217,36 @@ func (d *OnedriveAPP) getDrive(ctx context.Context) (*DriveResp, error) { var resp DriveResp _, err := d.Request(api, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, &resp) + }, &resp, true) if err != nil { return nil, err } return &resp, nil } + +func (d *OnedriveAPP) getDirectUploadInfo(ctx context.Context, path string) (*model.HttpDirectUploadInfo, error) { + // Create upload session + url := d.GetMetaUrl(false, path) + "/createUploadSession" + metadata := map[string]any{ + "item": map[string]any{ + "@microsoft.graph.conflictBehavior": "rename", + }, + } + + res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) + }, nil) + if err != nil { + return nil, err + } + + uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() + if uploadUrl == "" { + return nil, fmt.Errorf("failed to get upload URL from response") + } + return &model.HttpDirectUploadInfo{ + UploadURL: uploadUrl, + ChunkSize: d.ChunkSize * 1024 * 1024, // Convert MB to bytes + Method: "PUT", + }, nil +} diff --git a/drivers/onedrive_sharelink/driver.go b/drivers/onedrive_sharelink/driver.go index 42d0f1904..0bc3e79b5 100644 --- a/drivers/onedrive_sharelink/driver.go +++ b/drivers/onedrive_sharelink/driver.go @@ -2,21 +2,34 @@ package onedrive_sharelink import ( "context" + "fmt" + "io" + "net/http" + stdpath "path" "strings" + "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/cron" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) +const headerTTL = 25 * time.Minute + type OnedriveSharelink struct { model.Storage cron *cron.Cron Addition + + headerMu sync.RWMutex + sg singleflight.Group[http.Header] } func (d *OnedriveSharelink) Config() driver.Config { @@ -38,17 +51,20 @@ func (d *OnedriveSharelink) Init(ctx context.Context) error { d.cron = cron.NewCron(time.Hour * 1) d.cron.Do(func() { var err error - d.Headers, err = d.getHeaders(ctx) + h, err := d.getHeaders(ctx) if err != nil { log.Errorf("%+v", err) + return } + d.storeHeaders(h) }) // Get initial headers - d.Headers, err = d.getHeaders(ctx) + h, err := d.getHeaders(ctx) if err != nil { return err } + d.storeHeaders(h) return nil } @@ -58,15 +74,16 @@ func (d *OnedriveSharelink) Drop(ctx context.Context) error { } func (d *OnedriveSharelink) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - path := dir.GetPath() - files, err := d.getFiles(ctx, path) + files, err := d.getFiles(ctx, dir.GetPath()) if err != nil { return nil, err } // Convert the slice of files to the required model.Obj format return utils.SliceConvert(files, func(src Item) (model.Obj, error) { - return fileToObj(src), nil + obj := fileToObj(src) + obj.Path = stdpath.Join(dir.GetPath(), obj.GetName()) + return obj, nil }) } @@ -76,21 +93,18 @@ func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model // Cut the first char and the last char uniqueId = uniqueId[1 : len(uniqueId)-1] url := d.downloadLinkPrefix + uniqueId - header := d.Headers - // If the headers are older than 30 minutes, get new headers - if d.HeaderTime < time.Now().Unix()-1800 { - var err error - log.Debug("headers are older than 30 minutes, get new headers") - header, err = d.getHeaders(ctx) - if err != nil { - return nil, err - } + header, err := d.getValidHeaders(ctx) + if err != nil { + return nil, err } return &model.Link{ URL: url, Header: header, + RangeReader: rangeReaderFunc(func(ctx context.Context, hr http_range.Range) (io.ReadCloser, error) { + return d.rangeReadWithRefresh(ctx, url, hr) + }), }, nil } @@ -129,3 +143,102 @@ func (d *OnedriveSharelink) Put(ctx context.Context, dstDir model.Obj, stream mo //} var _ driver.Driver = (*OnedriveSharelink)(nil) + +// rangeReadWithRefresh tries once with current headers, and if the response +// looks invalid (error status or html login page), it refreshes headers and retries. +func (d *OnedriveSharelink) rangeReadWithRefresh(ctx context.Context, url string, hr http_range.Range) (io.ReadCloser, error) { + tryOnce := func(header http.Header) (io.ReadCloser, error) { + h := cloneHeader(header) + if h == nil { + h = http.Header{} + } + h = http_range.ApplyRangeToHttpHeader(hr, h) + resp, err := net.RequestHttp(ctx, http.MethodGet, h, url) + if err != nil { + return nil, err + } + ct := strings.ToLower(resp.Header.Get("Content-Type")) + if strings.Contains(ct, "text/html") { + _ = resp.Body.Close() + return nil, fmt.Errorf("unexpected html response") + } + return resp.Body, nil + } + + header, err := d.getValidHeaders(ctx) + if err != nil { + return nil, err + } + if body, err := tryOnce(header); err == nil { + return body, nil + } + + // refresh and retry once + header, err = d.refreshHeaders(ctx) + if err != nil { + return nil, err + } + return tryOnce(header) +} + +type rangeReaderFunc func(ctx context.Context, hr http_range.Range) (io.ReadCloser, error) + +func (f rangeReaderFunc) RangeRead(ctx context.Context, hr http_range.Range) (io.ReadCloser, error) { + return f(ctx, hr) +} + +func cloneHeader(header http.Header) http.Header { + if header == nil { + return nil + } + return header.Clone() +} + +func (d *OnedriveSharelink) headerSnapshot() http.Header { + d.headerMu.RLock() + defer d.headerMu.RUnlock() + return cloneHeader(d.Headers) +} + +func (d *OnedriveSharelink) storeHeaders(header http.Header) { + if header == nil { + return + } + d.headerMu.Lock() + d.Headers = header + d.HeaderTime = time.Now().Unix() + d.headerMu.Unlock() +} + +func (d *OnedriveSharelink) headersExpired() bool { + d.headerMu.RLock() + defer d.headerMu.RUnlock() + return time.Since(time.Unix(d.HeaderTime, 0)) > headerTTL +} + +func (d *OnedriveSharelink) refreshHeaders(ctx context.Context) (http.Header, error) { + header, err, _ := d.sg.Do("refresh", func() (http.Header, error) { + h, e := d.getHeaders(ctx) + if e != nil { + return nil, e + } + d.storeHeaders(h) + return h, nil + }) + return header, err +} + +func (d *OnedriveSharelink) getValidHeaders(ctx context.Context) (http.Header, error) { + if h := d.headerSnapshot(); h != nil && !d.headersExpired() { + return h, nil + } + h, err := d.refreshHeaders(ctx) + if err != nil { + if h2 := d.headerSnapshot(); h2 != nil { + log.Warnf("onedrive_sharelink: use cached headers after refresh failure: %+v", err) + return h2, nil + } + return nil, err + } + return h, nil +} diff --git a/drivers/openlist/driver.go b/drivers/openlist/driver.go index b539bb5a8..2ca60ff61 100644 --- a/drivers/openlist/driver.go +++ b/drivers/openlist/driver.go @@ -95,6 +95,7 @@ func (d *OpenList) List(ctx context.Context, dir model.Obj, args model.ListArgs) file := model.ObjThumb{ Object: model.Object{ Name: f.Name, + Path: path.Join(dir.GetPath(), f.Name), Modified: f.Modified, Ctime: f.Created, Size: f.Size, @@ -110,19 +111,29 @@ func (d *OpenList) List(ctx context.Context, dir model.Obj, args model.ListArgs) func (d *OpenList) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp common.Resp[FsGetResp] + headers := map[string]string{ + "User-Agent": base.UserAgent, + } // if PassUAToUpsteam is true, then pass the user-agent to the upstream - userAgent := base.UserAgent if d.PassUAToUpsteam { - userAgent = args.Header.Get("user-agent") - if userAgent == "" { - userAgent = base.UserAgent + userAgent := args.Header.Get("user-agent") + if userAgent != "" { + headers["User-Agent"] = userAgent + } + } + // if PassIPToUpsteam is true, then pass the ip address to the upstream + if d.PassIPToUpsteam { + ip := args.IP + if ip != "" { + headers["X-Forwarded-For"] = ip + headers["X-Real-Ip"] = ip } } _, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(FsGetReq{ Path: file.GetPath(), Password: d.MetaPassword, - }).SetHeader("user-agent", userAgent) + }).SetHeaders(headers) }) if err != nil { return nil, err @@ -350,13 +361,21 @@ func (d *OpenList) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.O Name: []string{name}, PutIntoNewDir: args.PutIntoNewDir, SrcDir: dir, + Overwrite: args.Overwrite, }) }) return err } -//func (d *OpenList) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { -// return nil, errs.NotSupport -//} +func (d *OpenList) ResolveLinkCacheMode(_ string) driver.LinkCacheMode { + var mode driver.LinkCacheMode + if d.PassIPToUpsteam { + mode |= driver.LinkCacheIP + } + if d.PassUAToUpsteam { + mode |= driver.LinkCacheUA + } + return mode +} var _ driver.Driver = (*OpenList)(nil) diff --git a/drivers/openlist/meta.go b/drivers/openlist/meta.go index fc76c1424..16c6a155c 100644 --- a/drivers/openlist/meta.go +++ b/drivers/openlist/meta.go @@ -12,6 +12,7 @@ type Addition struct { Username string `json:"username"` Password string `json:"password"` Token string `json:"token"` + PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` } @@ -20,8 +21,8 @@ var config = driver.Config{ Name: "OpenList", LocalSort: true, DefaultRoot: "/", - CheckStatus: true, ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheAuto, } func init() { diff --git a/drivers/openlist/types.go b/drivers/openlist/types.go index f0960d6f6..ffc3e350a 100644 --- a/drivers/openlist/types.go +++ b/drivers/openlist/types.go @@ -167,4 +167,5 @@ type DecompressReq struct { Name []string `json:"name"` PutIntoNewDir bool `json:"put_into_new_dir"` SrcDir string `json:"src_dir"` + Overwrite bool `json:"overwrite"` } diff --git a/drivers/openlist_share/driver.go b/drivers/openlist_share/driver.go index 416987777..0b99edd07 100644 --- a/drivers/openlist_share/driver.go +++ b/drivers/openlist_share/driver.go @@ -68,6 +68,7 @@ func (d *OpenListShare) List(ctx context.Context, dir model.Obj, args model.List file := model.ObjThumb{ Object: model.Object{ Name: f.Name, + Path: stdpath.Join(dir.GetPath(), f.Name), Modified: f.Modified, Ctime: f.Created, Size: f.Size, diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index c728123c1..8b72c3638 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -36,7 +36,6 @@ func (d *PikPak) GetAddition() driver.Additional { } func (d *PikPak) Init(ctx context.Context) (err error) { - if d.Common == nil { d.Common = &Common{ client: base.NewRestyClient(), @@ -247,7 +246,7 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } params := resp.Resumable.Params - //endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") + // endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") // web 端上传 返回的endpoint 为 `mypikpak.net` | android 端上传 返回的endpoint 为 `vip-lixian-07.mypikpak.net`· if d.Addition.Platform == "android" { params.Endpoint = "mypikpak.net" @@ -260,6 +259,30 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return d.UploadByMultipart(ctx, ¶ms, stream.GetSize(), stream, up) } +func (d *PikPak) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := d.request("https://api-drive.mypikpak.com/drive/v1/about", http.MethodGet, func(req *resty.Request) { + req.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + // 离线下载文件 func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { requestBody := base.Json{ @@ -278,7 +301,6 @@ func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir req.SetContext(ctx). SetBody(requestBody) }, &resp) - if err != nil { return nil, err } @@ -325,7 +347,6 @@ func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase [] req.SetContext(ctx). SetQueryParams(params) }, &resp) - if err != nil { return nil, fmt.Errorf("failed to get offline list: %w", err) } diff --git a/drivers/pikpak/types.go b/drivers/pikpak/types.go index bea766fe3..6ae78a455 100644 --- a/drivers/pikpak/types.go +++ b/drivers/pikpak/types.go @@ -78,7 +78,7 @@ type Media struct { type UploadTaskData struct { UploadType string `json:"upload_type"` - //UPLOAD_TYPE_RESUMABLE + // UPLOAD_TYPE_RESUMABLE Resumable *struct { Kind string `json:"kind"` Params S3Params `json:"params"` @@ -195,3 +195,15 @@ type CaptchaTokenResponse struct { ExpiresIn int64 `json:"expires_in"` Url string `json:"url"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + UsageInTrash string `json:"usage_in_trash"` + IsUnlimited bool `json:"is_unlimited"` + Complimentary string `json:"complimentary"` + } `json:"quota"` + ExpiresAt string `json:"expires_at"` + UserType int `json:"user_type"` +} diff --git a/drivers/proton_drive/driver.go b/drivers/proton_drive/driver.go new file mode 100644 index 000000000..936ccee61 --- /dev/null +++ b/drivers/proton_drive/driver.go @@ -0,0 +1,286 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/ProtonMail/gopenpgp/v2/crypto" + proton_api_bridge "github.com/henrybear327/Proton-API-Bridge" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +type ProtonDrive struct { + model.Storage + Addition + + protonDrive *proton_api_bridge.ProtonDrive + + apiBase string + appVersion string + protonJson string + userAgent string + sdkVersion string + webDriveAV string + + c *proton.Client + + // userKR *crypto.KeyRing + addrKRs map[string]*crypto.KeyRing + addrData map[string]proton.Address + + MainShare *proton.Share + + DefaultAddrKR *crypto.KeyRing + MainShareKR *crypto.KeyRing +} + +func (d *ProtonDrive) Config() driver.Config { + return config +} + +func (d *ProtonDrive) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ProtonDrive) Init(ctx context.Context) (err error) { + defer func() { + if r := recover(); err == nil && r != nil { + err = fmt.Errorf("ProtonDrive initialization panic: %v", r) + } + }() + + if d.Email == "" { + return fmt.Errorf("email is required") + } + if d.Password == "" { + return fmt.Errorf("password is required") + } + + config := &common.Config{ + AppVersion: d.appVersion, + UserAgent: d.userAgent, + FirstLoginCredential: &common.FirstLoginCredentialData{ + Username: d.Email, + Password: d.Password, + TwoFA: d.TwoFACode, + }, + EnableCaching: true, + ConcurrentBlockUploadCount: setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers), + //ConcurrentFileCryptoCount: 2, + UseReusableLogin: d.UseReusableLogin && d.ReusableCredential != (common.ReusableCredentialData{}), + ReplaceExistingDraft: true, + ReusableCredential: &d.ReusableCredential, + } + + protonDrive, _, err := proton_api_bridge.NewProtonDrive( + ctx, + config, + d.authHandler, + func() {}, + ) + + if err != nil && config.UseReusableLogin { + config.UseReusableLogin = false + protonDrive, _, err = proton_api_bridge.NewProtonDrive(ctx, + config, + d.authHandler, + func() {}, + ) + if err == nil { + op.MustSaveDriverStorage(d) + } + } + + if err != nil { + return fmt.Errorf("failed to initialize ProtonDrive: %w", err) + } + + if err := d.initClient(ctx); err != nil { + return err + } + + d.protonDrive = protonDrive + d.MainShare = protonDrive.MainShare + if d.RootFolderID == "root" || d.RootFolderID == "" { + d.RootFolderID = protonDrive.RootLink.LinkID + } + d.MainShareKR = protonDrive.MainShareKR + d.DefaultAddrKR = protonDrive.DefaultAddrKR + + return nil +} + +func (d *ProtonDrive) Drop(ctx context.Context) error { + return nil +} + +func (d *ProtonDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + entries, err := d.protonDrive.ListDirectory(ctx, dir.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + } + + objects := make([]model.Obj, 0, len(entries)) + for _, entry := range entries { + obj := &model.Object{ + ID: entry.Link.LinkID, + Name: entry.Name, + Size: entry.Link.Size, + Modified: time.Unix(entry.Link.ModifyTime, 0), + IsFolder: entry.IsFolder, + } + objects = append(objects, obj) + } + + return objects, nil +} + +func (d *ProtonDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + link, err := d.getLink(ctx, file.GetID()) + if err != nil { + return nil, fmt.Errorf("failed get file link: %+v", err) + } + fileSystemAttrs, err := d.protonDrive.GetActiveRevisionAttrs(ctx, link) + if err != nil { + return nil, fmt.Errorf("failed get file revision: %+v", err) + } + // 解密后的文件大小 + size := fileSystemAttrs.Size + + rangeReaderFunc := func(rangeCtx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + length := httpRange.Length + if length < 0 || httpRange.Start+length > size { + length = size - httpRange.Start + } + reader, _, _, err := d.protonDrive.DownloadFile(rangeCtx, link, httpRange.Start) + if err != nil { + return nil, fmt.Errorf("failed start download: %+v", err) + } + return utils.ReadCloser{ + Reader: io.LimitReader(reader, length), + Closer: reader, + }, nil + } + + expiration := time.Minute + return &model.Link{ + RangeReader: stream.RateLimitRangeReaderFunc(rangeReaderFunc), + ContentLength: size, + Expiration: &expiration, + }, nil +} + +func (d *ProtonDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + id, err := d.protonDrive.CreateNewFolderByID(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } + + newDir := &model.Object{ + ID: id, + Name: dirName, + IsFolder: true, + Modified: time.Now(), + } + return newDir, nil +} + +func (d *ProtonDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.DirectMove(ctx, srcObj, dstDir) +} + +func (d *ProtonDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + return d.DirectRename(ctx, srcObj, newName) +} + +func (d *ProtonDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, fmt.Errorf("directory copy not supported") + } + + srcLink, err := d.getLink(ctx, srcObj.GetID()) + if err != nil { + return nil, err + } + + reader, linkSize, fileSystemAttrs, err := d.protonDrive.DownloadFile(ctx, srcLink, 0) + if err != nil { + return nil, fmt.Errorf("failed to download source file: %w", err) + } + defer reader.Close() + + actualSize := linkSize + if fileSystemAttrs != nil && fileSystemAttrs.Size > 0 { + actualSize = fileSystemAttrs.Size + } + + file := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{ + Name: srcObj.GetName(), + // Use the accurate and real size + Size: actualSize, + Modified: srcObj.ModTime(), + }, + Reader: reader, + } + defer file.Close() + return d.Put(ctx, dstDir, file, func(percentage float64) {}) +} + +func (d *ProtonDrive) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + return d.protonDrive.MoveFolderToTrashByID(ctx, obj.GetID(), false) + } else { + return d.protonDrive.MoveFileToTrashByID(ctx, obj.GetID()) + } +} + +func (d *ProtonDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return d.uploadFile(ctx, dstDir.GetID(), file, up) +} + +func (d *ProtonDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + about, err := d.protonDrive.About(ctx) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: about.MaxSpace, + UsedSpace: about.UsedSpace, + }, + }, nil +} + +var _ driver.Driver = (*ProtonDrive)(nil) diff --git a/drivers/proton_drive/meta.go b/drivers/proton_drive/meta.go new file mode 100644 index 000000000..acf656779 --- /dev/null +++ b/drivers/proton_drive/meta.go @@ -0,0 +1,56 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/henrybear327/Proton-API-Bridge/common" +) + +type Addition struct { + driver.RootID + Email string `json:"email" required:"true" type:"string"` + Password string `json:"password" required:"true" type:"string"` + TwoFACode string `json:"two_fa_code" type:"string"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"100"` + UseReusableLogin bool `json:"use_reusable_login" type:"bool" default:"true" help:"Use reusable login credentials instead of username/password"` + ReusableCredential common.ReusableCredentialData +} + +var config = driver.Config{ + Name: "ProtonDrive", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: "root", + NoLinkURL: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ProtonDrive{ + Addition: Addition{ + UseReusableLogin: true, + }, + apiBase: "https://drive.proton.me/api", + appVersion: "windows-drive@1.11.3+rclone+proton", + protonJson: "application/vnd.protonmail.v1+json", + sdkVersion: "js@0.3.0", + userAgent: "ProtonDrive/v1.70.0 (Windows NT 10.0.22000; Win64; x64)", + webDriveAV: "web-drive@5.2.0+0f69f7a8", + } + }) +} diff --git a/drivers/proton_drive/types.go b/drivers/proton_drive/types.go new file mode 100644 index 000000000..ed6dcd192 --- /dev/null +++ b/drivers/proton_drive/types.go @@ -0,0 +1,38 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +type MoveRequest struct { + ParentLinkID string `json:"ParentLinkID"` + NodePassphrase string `json:"NodePassphrase"` + NodePassphraseSignature *string `json:"NodePassphraseSignature"` + Name string `json:"Name"` + NameSignatureEmail string `json:"NameSignatureEmail"` + Hash string `json:"Hash"` + OriginalHash string `json:"OriginalHash"` + ContentHash *string `json:"ContentHash"` // Maybe null +} + +type RenameRequest struct { + Name string `json:"Name"` // PGP encrypted name + NameSignatureEmail string `json:"NameSignatureEmail"` // User's signature email + Hash string `json:"Hash"` // New name hash + OriginalHash string `json:"OriginalHash"` // Current name hash +} + +type RenameResponse struct { + Code int `json:"Code"` +} diff --git a/drivers/proton_drive/util.go b/drivers/proton_drive/util.go new file mode 100644 index 000000000..a59bb6cb9 --- /dev/null +++ b/drivers/proton_drive/util.go @@ -0,0 +1,670 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/henrybear327/go-proton-api" +) + +func (d *ProtonDrive) uploadFile(ctx context.Context, parentLinkID string, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + _, err := d.getLink(ctx, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to get parent link: %w", err) + } + + var reader io.Reader + // Use buffered reader with larger buffer for better performance + var bufferSize int + + // File > 100MB (default) + if file.GetSize() > d.ChunkSize*1024*1024 { + // 256KB for large files + bufferSize = 256 * 1024 + // File > 10MB + } else if file.GetSize() > 10*1024*1024 { + // 128KB for medium files + bufferSize = 128 * 1024 + } else { + // 64KB for small files + bufferSize = 64 * 1024 + } + + // reader = bufio.NewReader(file) + reader = bufio.NewReaderSize(file, bufferSize) + reader = &driver.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: reader, + Size: file.GetSize(), + }, + UpdateProgress: up, + } + reader = driver.NewLimitedUploadStream(ctx, reader) + + id, _, err := d.protonDrive.UploadFileByReader(ctx, parentLinkID, file.GetName(), file.ModTime(), reader, 0) + if err != nil { + return nil, fmt.Errorf("failed to upload file: %w", err) + } + + return &model.Object{ + ID: id, + Name: file.GetName(), + Size: file.GetSize(), + Modified: file.ModTime(), + IsFolder: false, + }, nil +} + +func (d *ProtonDrive) encryptFileName(ctx context.Context, name string, parentLinkID string) (string, error) { + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Temporary file (request) + tempReq := proton.CreateFileReq{ + SignatureAddress: d.MainShare.Creator, + } + + // Encrypt the filename + err = tempReq.SetName(name, d.DefaultAddrKR, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to encrypt filename: %w", err) + } + + return tempReq.Name, nil +} + +func (d *ProtonDrive) generateFileNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func (d *ProtonDrive) getOriginalNameHash(link *proton.Link) (string, error) { + if link == nil { + return "", fmt.Errorf("link cannot be nil") + } + + if link.Hash == "" { + return "", fmt.Errorf("link hash is empty") + } + + return link.Hash, nil +} + +func (d *ProtonDrive) getLink(ctx context.Context, linkID string) (*proton.Link, error) { + if linkID == "" { + return nil, fmt.Errorf("linkID cannot be empty") + } + + link, err := d.c.GetLink(ctx, d.MainShare.ShareID, linkID) + if err != nil { + return nil, err + } + + return &link, nil +} + +func (d *ProtonDrive) getLinkKR(ctx context.Context, link *proton.Link) (*crypto.KeyRing, error) { + if link == nil { + return nil, fmt.Errorf("link cannot be nil") + } + + // Root Link or Root Dir + if link.ParentLinkID == "" { + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + return link.GetKeyRing(d.MainShareKR, signatureVerificationKR) + } + + // Get parent keyring recursively + parentLink, err := d.getLink(ctx, link.ParentLinkID) + if err != nil { + return nil, err + } + + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return nil, err + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + + return link.GetKeyRing(parentNodeKR, signatureVerificationKR) +} + +var ( + ErrKeyPassOrSaltedKeyPassMustBeNotNil = errors.New("either keyPass or saltedKeyPass must be not nil") + ErrFailedToUnlockUserKeys = errors.New("failed to unlock user keys") +) + +func getAccountKRs(ctx context.Context, c *proton.Client, keyPass, saltedKeyPass []byte) (*crypto.KeyRing, map[string]*crypto.KeyRing, map[string]proton.Address, []byte, error) { + user, err := c.GetUser(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("user %#v", user) + + addrsArr, err := c.GetAddresses(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("addr %#v", addr) + + if saltedKeyPass == nil { + if keyPass == nil { + return nil, nil, nil, nil, ErrKeyPassOrSaltedKeyPassMustBeNotNil + } + + // Due to limitations, salts are stored using cacheCredentialToFile + salts, err := c.GetSalts(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("salts %#v", salts) + + saltedKeyPass, err = salts.SaltForKey(keyPass, user.Keys.Primary().ID) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("saltedKeyPass ok") + } + + userKR, addrKRs, err := proton.Unlock(user, addrsArr, saltedKeyPass, nil) + if err != nil { + return nil, nil, nil, nil, err + } else if userKR.CountDecryptionEntities() == 0 { + return nil, nil, nil, nil, ErrFailedToUnlockUserKeys + } + + addrs := make(map[string]proton.Address) + for _, addr := range addrsArr { + addrs[addr.Email] = addr + } + + return userKR, addrKRs, addrs, saltedKeyPass, nil +} + +func (d *ProtonDrive) getSignatureVerificationKeyring(emailAddresses []string, verificationAddrKRs ...*crypto.KeyRing) (*crypto.KeyRing, error) { + ret, err := crypto.NewKeyRing(nil) + if err != nil { + return nil, err + } + + for _, emailAddress := range emailAddresses { + if addr, ok := d.addrData[emailAddress]; ok { + if addrKR, exists := d.addrKRs[addr.ID]; exists { + err = d.addKeysFromKR(ret, addrKR) + if err != nil { + return nil, err + } + } + } + } + + for _, kr := range verificationAddrKRs { + err = d.addKeysFromKR(ret, kr) + if err != nil { + return nil, err + } + } + + if ret.CountEntities() == 0 { + return nil, fmt.Errorf("no keyring for signature verification") + } + + return ret, nil +} + +func (d *ProtonDrive) addKeysFromKR(kr *crypto.KeyRing, newKRs ...*crypto.KeyRing) error { + for i := range newKRs { + for _, key := range newKRs[i].GetKeys() { + err := kr.AddKey(key) + if err != nil { + return err + } + } + } + return nil +} + +func (d *ProtonDrive) DirectRename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + // fmt.Printf("DEBUG DirectRename: path=%s, newName=%s", srcObj.GetPath(), newName) + + if d.MainShare == nil || d.DefaultAddrKR == nil { + return nil, fmt.Errorf("missing required fields: MainShare=%v, DefaultAddrKR=%v", + d.MainShare != nil, d.DefaultAddrKR != nil) + } + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + srcLink, err := d.getLink(ctx, srcObj.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + parentLinkID := srcLink.ParentLinkID + if parentLinkID == "" { + return nil, fmt.Errorf("cannot rename root folder") + } + + encryptedName, err := d.encryptFileName(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateFileNameHash(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + renameReq := RenameRequest{ + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + } + + err = d.executeRenameAPI(ctx, srcLink.LinkID, renameReq) + if err != nil { + return nil, fmt.Errorf("rename API call failed: %w", err) + } + + return &model.Object{ + ID: srcLink.LinkID, + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) executeRenameAPI(ctx context.Context, linkID string, req RenameRequest) error { + renameURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/rename", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal rename request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", renameURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.ReusableCredential.UID) + httpReq.Header.Set("Authorization", "Bearer "+d.ReusableCredential.AccessToken) + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute rename request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("rename failed with status %d", resp.StatusCode) + } + + var renameResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&renameResp); err != nil { + return fmt.Errorf("failed to decode rename response: %w", err) + } + + if renameResp.Code != 1000 { + return fmt.Errorf("rename failed with code %d", renameResp.Code) + } + + return nil +} + +func (d *ProtonDrive) executeMoveAPI(ctx context.Context, linkID string, req MoveRequest) error { + // fmt.Printf("DEBUG Move Request - Name: %s\n", req.Name) + // fmt.Printf("DEBUG Move Request - Hash: %s\n", req.Hash) + // fmt.Printf("DEBUG Move Request - OriginalHash: %s\n", req.OriginalHash) + // fmt.Printf("DEBUG Move Request - ParentLinkID: %s\n", req.ParentLinkID) + + // fmt.Printf("DEBUG Move Request - Name length: %d\n", len(req.Name)) + // fmt.Printf("DEBUG Move Request - NameSignatureEmail: %s\n", req.NameSignatureEmail) + // fmt.Printf("DEBUG Move Request - ContentHash: %v\n", req.ContentHash) + // fmt.Printf("DEBUG Move Request - NodePassphrase length: %d\n", len(req.NodePassphrase)) + // fmt.Printf("DEBUG Move Request - NodePassphraseSignature length: %d\n", len(req.NodePassphraseSignature)) + + // fmt.Printf("DEBUG Move Request - SrcLinkID: %s\n", linkID) + // fmt.Printf("DEBUG Move Request - DstParentLinkID: %s\n", req.ParentLinkID) + // fmt.Printf("DEBUG Move Request - ShareID: %s\n", d.MainShare.ShareID) + + srcLink, _ := d.getLink(ctx, linkID) + if srcLink != nil && srcLink.ParentLinkID == req.ParentLinkID { + return fmt.Errorf("cannot move to same parent directory") + } + + moveURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/move", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal move request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", moveURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Authorization", "Bearer "+d.ReusableCredential.AccessToken) + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.ReusableCredential.UID) + httpReq.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute move request: %w", err) + } + defer resp.Body.Close() + + var moveResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&moveResp); err != nil { + return fmt.Errorf("failed to decode move response: %w", err) + } + + if moveResp.Code != 1000 { + return fmt.Errorf("move operation failed with code: %d", moveResp.Code) + } + + return nil +} + +func (d *ProtonDrive) DirectMove(ctx context.Context, srcObj model.Obj, dstDir model.Obj) (model.Obj, error) { + // fmt.Printf("DEBUG DirectMove: srcPath=%s, dstPath=%s", srcObj.GetPath(), dstDir.GetPath()) + + srcLink, err := d.getLink(ctx, srcObj.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + dstParentLinkID := dstDir.GetID() + + if srcObj.IsDir() { + // Check if destination is a descendant of source + if err := d.checkCircularMove(ctx, srcLink.LinkID, dstParentLinkID); err != nil { + return nil, err + } + } + + // Encrypt the filename for the new location + encryptedName, err := d.encryptFileName(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateNameHash(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + // Re-encrypt node passphrase for new parent context + reencryptedPassphrase, err := d.reencryptNodePassphrase(ctx, srcLink, dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to re-encrypt node passphrase: %w", err) + } + + moveReq := MoveRequest{ + ParentLinkID: dstParentLinkID, + NodePassphrase: reencryptedPassphrase, + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + ContentHash: nil, + + // *** Causes rejection *** + /* NodePassphraseSignature: srcLink.NodePassphraseSignature, */ + } + + //fmt.Printf("DEBUG MoveRequest validation:\n") + //fmt.Printf(" Name length: %d\n", len(moveReq.Name)) + //fmt.Printf(" Hash: %s\n", moveReq.Hash) + //fmt.Printf(" OriginalHash: %s\n", moveReq.OriginalHash) + //fmt.Printf(" NodePassphrase length: %d\n", len(moveReq.NodePassphrase)) + /* fmt.Printf(" NodePassphraseSignature length: %d\n", len(moveReq.NodePassphraseSignature)) */ + //fmt.Printf(" NameSignatureEmail: %s\n", moveReq.NameSignatureEmail) + + err = d.executeMoveAPI(ctx, srcLink.LinkID, moveReq) + if err != nil { + return nil, fmt.Errorf("move API call failed: %w", err) + } + + return &model.Object{ + ID: srcLink.LinkID, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) reencryptNodePassphrase(ctx context.Context, srcLink *proton.Link, dstParentLinkID string) (string, error) { + // Get source parent link with metadata + srcParentLink, err := d.getLink(ctx, srcLink.ParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get source parent link: %w", err) + } + + // Get source parent keyring using link object + srcParentKR, err := d.getLinkKR(ctx, srcParentLink) + if err != nil { + return "", fmt.Errorf("failed to get source parent keyring: %w", err) + } + + // Get destination parent link with metadata + dstParentLink, err := d.getLink(ctx, dstParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get destination parent link: %w", err) + } + + // Get destination parent keyring using link object + dstParentKR, err := d.getLinkKR(ctx, dstParentLink) + if err != nil { + return "", fmt.Errorf("failed to get destination parent keyring: %w", err) + } + + // Re-encrypt the node passphrase from source parent context to destination parent context + reencryptedPassphrase, err := reencryptKeyPacket(srcParentKR, dstParentKR, d.DefaultAddrKR, srcLink.NodePassphrase) + if err != nil { + return "", fmt.Errorf("failed to re-encrypt key packet: %w", err) + } + + return reencryptedPassphrase, nil +} + +func (d *ProtonDrive) generateNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Get signature verification keyring + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func reencryptKeyPacket(srcKR, dstKR, _ *crypto.KeyRing, passphrase string) (string, error) { // addrKR (3) + oldSplitMessage, err := crypto.NewPGPSplitMessageFromArmored(passphrase) + if err != nil { + return "", err + } + + sessionKey, err := srcKR.DecryptSessionKey(oldSplitMessage.KeyPacket) + if err != nil { + return "", err + } + + newKeyPacket, err := dstKR.EncryptSessionKey(sessionKey) + if err != nil { + return "", err + } + + newSplitMessage := crypto.NewPGPSplitMessage(newKeyPacket, oldSplitMessage.DataPacket) + + return newSplitMessage.GetArmored() +} + +func (d *ProtonDrive) checkCircularMove(ctx context.Context, srcLinkID, dstParentLinkID string) error { + currentLinkID := dstParentLinkID + + for currentLinkID != "" && currentLinkID != d.RootFolderID { + if currentLinkID == srcLinkID { + return fmt.Errorf("cannot move folder into itself or its subfolder") + } + + currentLink, err := d.getLink(ctx, currentLinkID) + if err != nil { + return err + } + currentLinkID = currentLink.ParentLinkID + } + + return nil +} + +func (d *ProtonDrive) authHandler(auth proton.Auth) { + if auth.AccessToken != d.ReusableCredential.AccessToken || auth.RefreshToken != d.ReusableCredential.RefreshToken { + d.ReusableCredential.UID = auth.UID + d.ReusableCredential.AccessToken = auth.AccessToken + d.ReusableCredential.RefreshToken = auth.RefreshToken + + if err := d.initClient(context.Background()); err != nil { + fmt.Printf("ProtonDrive: failed to reinitialize client after auth refresh: %v\n", err) + } + + op.MustSaveDriverStorage(d) + } +} + +func (d *ProtonDrive) initClient(ctx context.Context) error { + clientOptions := []proton.Option{ + proton.WithAppVersion(d.appVersion), + proton.WithUserAgent(d.userAgent), + } + manager := proton.New(clientOptions...) + d.c = manager.NewClient(d.ReusableCredential.UID, d.ReusableCredential.AccessToken, d.ReusableCredential.RefreshToken) + + saltedKeyPassBytes, err := base64.StdEncoding.DecodeString(d.ReusableCredential.SaltedKeyPass) + if err != nil { + return fmt.Errorf("failed to decode salted key pass: %w", err) + } + + _, addrKRs, addrs, _, err := getAccountKRs(ctx, d.c, nil, saltedKeyPassBytes) + if err != nil { + return fmt.Errorf("failed to get account keyrings: %w", err) + } + + d.addrKRs = addrKRs + d.addrData = addrs + + return nil +} diff --git a/drivers/qihoo360/driver.go b/drivers/qihoo360/driver.go new file mode 100644 index 000000000..0268c6758 --- /dev/null +++ b/drivers/qihoo360/driver.go @@ -0,0 +1,678 @@ +package qihoo360 + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type Qihoo360 struct { + model.Storage + Addition + authInfo *AuthResp + authExpire int64 +} + +func (d *Qihoo360) Config() driver.Config { + return config +} + +func (d *Qihoo360) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Qihoo360) Init(ctx context.Context) error { + // Test authentication + _, err := d.getAuth() + return err +} + +func (d *Qihoo360) Drop(ctx context.Context) error { + d.authInfo = nil + return nil +} + +func (d *Qihoo360) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + path := dir.GetPath() + if path == "" { + path = d.RootFolderPath + } + if path == "" { + path = "/" + } + + // Ensure directory paths end with / (required by API for non-root paths) + if path != "/" && !strings.HasSuffix(path, "/") { + path += "/" + } + + files, err := d.getFiles(path, 0, 100) + if err != nil { + return nil, err + } + + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return src, nil + }) +} + +func (d *Qihoo360) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + // Get file ID (nid) + nid := file.GetID() + if nid == "" { + return nil, fmt.Errorf("file id is empty") + } + + // Get download URL from API + downloadUrl, err := d.getDownloadUrl(nid) + if err != nil { + return nil, err + } + + if downloadUrl == "" { + return nil, fmt.Errorf("download url is empty") + } + + return &model.Link{ + URL: downloadUrl, + Header: http.Header{ + "User-Agent": []string{"yunpan_mcp_server"}, + }, + }, nil +} + +func (d *Qihoo360) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + userDetail, err := d.getUserDetail() + if err != nil { + return nil, err + } + + // Parse total and used sizes from strings to int64 + totalSize := int64(0) + usedSize := int64(0) + + if total, err := strconv.ParseInt(userDetail.Data.TotalSize, 10, 64); err == nil { + totalSize = total + } + if used, err := strconv.ParseInt(userDetail.Data.UsedSize, 10, 64); err == nil { + usedSize = used + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: totalSize, + UsedSpace: usedSize, + }, + }, nil +} + +func (d *Qihoo360) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + path := parentDir.GetPath() + if path == "" { + path = d.RootFolderPath + } + if path == "" { + path = "/" + } + + // Ensure path ends with / + if path[len(path)-1] != '/' { + path += "/" + } + // Ensure dirName ends with / + if dirName[len(dirName)-1] != '/' { + dirName += "/" + } + + fname := path + dirName + + params := map[string]string{ + "fname": fname, + } + + var resp CommonResp + _, err := d.request("File.makeDir", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("make dir failed: %s", resp.Errmsg) + } + + return nil, nil +} + +func (d *Qihoo360) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcPath := srcObj.GetPath() + if srcPath == "" { + // Try to construct path from name + srcPath = d.RootFolderPath + if srcPath == "" { + srcPath = "/" + } + if srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + srcPath += srcObj.GetName() + if srcObj.IsDir() && srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + } + + dstPath := dstDir.GetPath() + if dstPath == "" { + dstPath = d.RootFolderPath + } + if dstPath == "" { + dstPath = "/" + } + if dstPath[len(dstPath)-1] != '/' { + dstPath += "/" + } + + params := map[string]string{ + "src_name": srcPath, + "new_name": dstPath, + } + + var resp CommonResp + _, err := d.request("File.move", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("move failed: %s", resp.Errmsg) + } + + return nil, nil +} + +func (d *Qihoo360) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + srcPath := srcObj.GetPath() + if srcPath == "" { + // Try to construct path from name + srcPath = d.RootFolderPath + if srcPath == "" { + srcPath = "/" + } + if srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + srcPath += srcObj.GetName() + if srcObj.IsDir() && srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + } + + // new_name should be just the name, not full path + if srcObj.IsDir() && newName[len(newName)-1] != '/' { + newName += "/" + } + + params := map[string]string{ + "src_name": srcPath, + "new_name": newName, + } + + var resp CommonResp + _, err := d.request("File.rename", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("rename failed: %s", resp.Errmsg) + } + + return nil, nil +} + +func (d *Qihoo360) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // Copy is not documented in ecs_mcp_server + return nil, errs.NotSupport +} + +func (d *Qihoo360) Remove(ctx context.Context, obj model.Obj) error { + srcPath := obj.GetPath() + if srcPath == "" { + // Try to construct path from name + srcPath = d.RootFolderPath + if srcPath == "" { + srcPath = "/" + } + if srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + srcPath += obj.GetName() + if obj.IsDir() && srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + } + + params := map[string]string{ + "fname": srcPath, + } + + var resp CommonResp + // fname parameter is excluded from sign calculation + _, err := d.request("File.delete", params, &resp, "fname") + if err != nil { + return err + } + + if resp.Errno != 0 { + return fmt.Errorf("remove failed: %s", resp.Errmsg) + } + + return nil +} + +func (d *Qihoo360) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + dstPath := dstDir.GetPath() + if dstPath == "" { + dstPath = d.RootFolderPath + } + if dstPath == "" { + dstPath = "/" + } + if dstPath[len(dstPath)-1] != '/' { + dstPath += "/" + } + + fname := dstPath + file.GetName() + fsize := file.GetSize() + now := time.Now().Unix() + + // Calculate file hash + const chunkSize = 524288 // 512KB per chunk + numChunks := (fsize + chunkSize - 1) / chunkSize + + var blockHashes []string + var blocks []struct { + data []byte + offset int64 + size int64 + hash string + } + + // Read file and calculate chunk hashes + for i := int64(0); i < numChunks; i++ { + size := chunkSize + if i == numChunks-1 { + size = int(fsize - i*chunkSize) + } + + buf := make([]byte, size) + n, err := io.ReadFull(file, buf) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + buf = buf[:n] + + // Calculate SHA1 hash for this block + hash := sha1.Sum(buf) + blockHash := hex.EncodeToString(hash[:]) + blockHashes = append(blockHashes, blockHash) + + blocks = append(blocks, struct { + data []byte + offset int64 + size int64 + hash string + }{ + data: buf, + offset: i * chunkSize, + size: int64(n), + hash: blockHash, + }) + } + + // Calculate file hash (SHA1 of concatenated block hashes) + fhashStr := strings.Join(blockHashes, "") + fhash := sha1.Sum([]byte(fhashStr)) + fhashHex := hex.EncodeToString(fhash[:]) + + // Get upload address + uploadAddr, err := d.getUploadAddr(fname, fsize, fhashHex, now, now) + if err != nil { + return nil, err + } + + // Check for instant upload (server already has the file) + // When file exists, HTTP is null + httpVal, httpOk := uploadAddr.Data.HTTP.(string) + if !httpOk || httpVal == "" { + return nil, nil // Instant upload success (file already exists) + } + + // Build upload host + uploadHost := httpVal + if uploadAddr.Data.IsHttps == 1 { + uploadHost = "https://" + uploadHost + } else { + uploadHost = "http://" + uploadHost + } + + // Get upload token + var tk string + if tkVal, ok := uploadAddr.Data.Tk.(string); ok { + tk = tkVal + } + + // Prepare block info for preload + blockInfoList := make([]BlockInfo, len(blocks)) + for i, block := range blocks { + blockInfoList[i] = BlockInfo{ + BHash: block.hash, + BIdx: i + 1, + BOffset: block.offset, + BSize: block.size, + } + } + + // Preload - send block info + preloadResp, err := d.preloadBlocks(ctx, uploadHost, fname, fsize, fhashHex, now, now, tk, blockInfoList) + if err != nil { + return nil, err + } + + // Upload each block + for i, block := range blocks { + blockInfo := preloadResp.Data.BlockInfo[i] + // Note: use user token (d.authInfo.Data.Token), not blockInfo.Token + err = d.uploadBlock(ctx, uploadHost, block.data, block.hash, i+1, block.offset, block.size, + fname, fsize, blockInfo.Q, blockInfo.T, d.authInfo.Data.Token, preloadResp.Data.Tid) + if err != nil { + return nil, fmt.Errorf("upload block %d failed: %w", i+1, err) + } + + // Update progress + if up != nil { + up(float64(block.size)) + } + } + + // Commit - merge blocks + // Note: use user token (d.authInfo.Data.Token), not blockInfo.Token + commitResp, err := d.commitUpload(ctx, uploadHost, preloadResp.Data.BlockInfo[0].Q, preloadResp.Data.BlockInfo[0].T, + d.authInfo.Data.Token, preloadResp.Data.Tid) + if err != nil { + return nil, err + } + + // If autoCommit is true (non-zero), file is already added (instant upload), use data from commit + if commitResp.Data.AutoCommit != 0 { + return &File{ + Name: file.GetName(), + Type: "0", + Nid: commitResp.Data.Nid, + CountSize: fmt.Sprintf("%d", commitResp.Data.Size), + CreateTimeTS: fmt.Sprintf("%d", commitResp.Data.CreateTime), + ModifyTimeTS: fmt.Sprintf("%d", commitResp.Data.ModifyTime), + Path: fname, + }, nil + } + + // Call Sync.addFileToApi to finalize the upload and get file info + addFileResp, err := d.addFileToApi(commitResp.Data.Tk) + if err != nil { + return nil, err + } + + // Set the full path + addFileResp.Data.File.Path = fname + + return &addFileResp.Data.File, nil +} + +func (d *Qihoo360) preloadBlocks(ctx context.Context, uploadHost, fname string, fsize int64, fhash string, fctime, fmtime int64, tk string, blocks []BlockInfo) (*PreloadResp, error) { + // Build query parameters + queryParams := map[string]string{ + "method": "Upload.request4Web", + "owner_qid": d.authInfo.Data.Qid, + "qid": d.authInfo.Data.Qid, + "devtype": "ecs_openapi", + "devid": "node-sdk-v16.20.2", // device id + "v": "1.0.1", + "ofmt": "json", + "devname": "EYUN_WEB_UPLOAD", + "rtick": fmt.Sprintf("%d", time.Now().Unix()), + } + + // Build URL + url := fmt.Sprintf("%s/intf.php", uploadHost) + for k, v := range queryParams { + if strings.Contains(url, "?") { + url += "&" + } else { + url += "?" + } + url += k + "=" + v + } + + // Prepare block_info JSON + blockInfoMap := map[string]interface{}{ + "request": map[string]interface{}{ + "block_info": blocks, + }, + } + blockInfoJSON, _ := json.Marshal(blockInfoMap) + + // Create multipart form + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + + // Add form fields + writer.WriteField("owner_qid", d.authInfo.Data.Qid) + writer.WriteField("fname", fname) + writer.WriteField("fsize", fmt.Sprintf("%d", fsize)) + writer.WriteField("fctime", fmt.Sprintf("%d", fctime)) + writer.WriteField("fmtime", fmt.Sprintf("%d", fmtime)) + writer.WriteField("fhash", fhash) + writer.WriteField("qid", d.authInfo.Data.Qid) + writer.WriteField("fattr", "0") + writer.WriteField("token", d.authInfo.Data.Token) + writer.WriteField("tk", tk) + writer.WriteField("devtype", "ecs_openapi") + + // Add file part with block_info JSON + part, _ := writer.CreateFormFile("file", "block_info.json") + part.Write(blockInfoJSON) + writer.Close() + + // Send request + req, _ := http.NewRequestWithContext(ctx, "POST", url, &buf) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Access-Token", d.authInfo.Data.AccessToken) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + var preloadResp PreloadResp + if err := json.Unmarshal(body, &preloadResp); err != nil { + return nil, err + } + + if preloadResp.Errno != 0 { + return nil, fmt.Errorf("preload failed: %s", preloadResp.Errmsg) + } + + return &preloadResp, nil +} + +func (d *Qihoo360) uploadBlock(ctx context.Context, uploadHost string, data []byte, bhash string, bidx int, boffset, bsize int64, filename string, filesize int64, q, t, token, tid string) error { + // Build query parameters + queryParams := map[string]string{ + "method": "Upload.block4Web", + "owner_qid": d.authInfo.Data.Qid, + "qid": d.authInfo.Data.Qid, + "devtype": "ecs_openapi", + "devid": "node-sdk-v16.20.2", + "v": "1.0.1", + "ofmt": "json", + "devname": "EYUN_WEB_UPLOAD", + "rtick": fmt.Sprintf("%d", time.Now().Unix()), + } + + // Build URL + url := fmt.Sprintf("%s/intf.php", uploadHost) + for k, v := range queryParams { + if strings.Contains(url, "?") { + url += "&" + } else { + url += "?" + } + url += k + "=" + v + } + + // Create multipart form + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + + // Add chunk data + part, _ := writer.CreateFormFile("file", "chunk") + part.Write(data) + + // Add form fields + writer.WriteField("bhash", bhash) + writer.WriteField("bidx", strconv.Itoa(bidx)) + writer.WriteField("boffset", fmt.Sprintf("%d", boffset)) + writer.WriteField("bsize", fmt.Sprintf("%d", bsize)) + writer.WriteField("filename", filename) + writer.WriteField("filesize", fmt.Sprintf("%d", filesize)) + writer.WriteField("q", q) + writer.WriteField("t", t) + writer.WriteField("token", token) + writer.WriteField("tid", tid) + writer.Close() + + // Send request + req, _ := http.NewRequestWithContext(ctx, "POST", url, &buf) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Access-Token", d.authInfo.Data.AccessToken) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + var result CommonResp + if err := json.Unmarshal(body, &result); err != nil { + return err + } + + if result.Errno != 0 { + return fmt.Errorf("upload block failed: %s", result.Errmsg) + } + + return nil +} + +func (d *Qihoo360) commitUpload(ctx context.Context, uploadHost, q, t, token, tid string) (*CommitResp, error) { + // Build query parameters + queryParams := map[string]string{ + "method": "Upload.commit4Web", + "owner_qid": d.authInfo.Data.Qid, + "qid": d.authInfo.Data.Qid, + "devtype": "ecs_openapi", + "devid": "node-sdk-v16.20.2", + "v": "1.0.1", + "ofmt": "json", + "devname": "EYUN_WEB_UPLOAD", + "rtick": fmt.Sprintf("%d", time.Now().Unix()), + } + + // Build URL + url := fmt.Sprintf("%s/intf.php", uploadHost) + for k, v := range queryParams { + if strings.Contains(url, "?") { + url += "&" + } else { + url += "?" + } + url += k + "=" + v + } + + // Create multipart form + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + + // Add form fields + writer.WriteField("q", q) + writer.WriteField("t", t) + writer.WriteField("token", token) + writer.WriteField("tid", tid) + writer.Close() + + // Send request + req, _ := http.NewRequestWithContext(ctx, "POST", url, &buf) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Access-Token", d.authInfo.Data.AccessToken) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + var result CommitResp + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + + if result.Errno != 0 { + return nil, fmt.Errorf("commit failed: %s", result.Errmsg) + } + + return &result, nil +} + +func (d *Qihoo360) addFileToApi(tk string) (*AddFileResp, error) { + params := map[string]string{ + "qid": d.authInfo.Data.Qid, + "tk": tk, + } + + var resp AddFileResp + _, err := d.request("Sync.addFileToApi", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("add file to api failed: %s", resp.Errmsg) + } + + return &resp, nil +} + +var _ driver.Driver = (*Qihoo360)(nil) diff --git a/drivers/qihoo360/meta.go b/drivers/qihoo360/meta.go new file mode 100644 index 000000000..57d261f37 --- /dev/null +++ b/drivers/qihoo360/meta.go @@ -0,0 +1,24 @@ +package qihoo360 + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + APIKey string `json:"api_key" required:"true" help:"360 AI Cloud API Key (yunpan_ prefix)"` +} + +var config = driver.Config{ + Name: "Qihoo360", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Qihoo360{} + }) +} diff --git a/drivers/qihoo360/types.go b/drivers/qihoo360/types.go new file mode 100644 index 000000000..a61e4789e --- /dev/null +++ b/drivers/qihoo360/types.go @@ -0,0 +1,167 @@ +package qihoo360 + +import ( + "strconv" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type File struct { + Name string `json:"name"` + Type string `json:"type"` // "1" for directory, "0" for file + Nid string `json:"nid"` + CountSize string `json:"count_size"` + CreateTimeTS string `json:"create_time"` + ModifyTimeTS string `json:"modify_time"` + Path string `json:"-"` // Full path, not from API +} + +func (f File) GetName() string { + return f.Name +} + +func (f File) GetSize() int64 { + size, _ := strconv.ParseInt(f.CountSize, 10, 64) + return size +} + +func (f File) ModTime() time.Time { + timestamp, _ := strconv.ParseInt(f.ModifyTimeTS, 10, 64) + return time.Unix(timestamp, 0) +} + +func (f File) CreateTime() time.Time { + timestamp, _ := strconv.ParseInt(f.CreateTimeTS, 10, 64) + return time.Unix(timestamp, 0) +} + +func (f File) IsDir() bool { + return f.Type == "1" +} + +func (f File) GetID() string { + return f.Nid +} + +func (f File) GetPath() string { + return f.Path +} + +func (f File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +type FileListResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + NodeList []File `json:"node_list"` + } `json:"data"` +} + +type AuthResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + AccessTokenExpire int64 `json:"access_token_expire"` + Qid string `json:"qid"` + } `json:"data"` + TraceId string `json:"trace_id"` +} + +type CommonResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` +} + +type UploadAddrResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + HTTP interface{} `json:"http"` // can be string or null + IsHttps int `json:"is_https"` // 0 or 1, not boolean + Tk interface{} `json:"tk"` // can be string or null + Addr2 string `json:"addr_2"` + NodeInfo []File `json:"node_info"` // returned when file exists (instant upload) + AutoCommit int `json:"autoCommit"` + FileHash string `json:"fhash"` + FileName string `json:"fname"` + FileSize string `json:"fsize"` + IsCreateDir bool `json:"is_createdir"` + } `json:"data"` +} + +type BlockInfo struct { + BHash string `json:"bhash"` + BIdx int `json:"bidx"` + BOffset int64 `json:"boffset"` + BSize int64 `json:"bsize"` + Q string `json:"q,omitempty"` + T string `json:"t,omitempty"` + Token string `json:"token,omitempty"` + Tid string `json:"tid,omitempty"` +} + +type PreloadResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + BlockInfo []BlockInfo `json:"block_info"` + Tid string `json:"tid"` + Tk string `json:"tk"` + HTTP string `json:"http"` + Addr2 string `json:"addr_2"` + IsHttps bool `json:"is_https"` + } `json:"data"` +} + +type DownloadUrlResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + DownloadUrl string `json:"downloadUrl"` + } `json:"data"` +} + +type CommitResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + Nid string `json:"nid"` + Name string `json:"fname"` + Size int64 `json:"fsize"` + CreateTime int64 `json:"fctime"` + ModifyTime int64 `json:"fmtime"` + Tk string `json:"tk"` + AutoCommit int `json:"autoCommit"` // 0 or 1 + } `json:"data"` +} + +type AddFileResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + File File `json:"file"` + } `json:"data"` +} + +type UserDetailResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + Name string `json:"name"` + TotalSize string `json:"total_size"` + UsedSize string `json:"used_size"` + AvailableSize int64 `json:"available_size"` + IsVip bool `json:"is_vip"` + VipDesc string `json:"vip_desc"` + ExpireDay int `json:"expire_day"` + Expire string `json:"expire"` + } `json:"data"` +} + +var _ model.Obj = (*File)(nil) diff --git a/drivers/qihoo360/util.go b/drivers/qihoo360/util.go new file mode 100644 index 000000000..f64b50f89 --- /dev/null +++ b/drivers/qihoo360/util.go @@ -0,0 +1,377 @@ +package qihoo360 + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "net/url" + pathpkg "path" + "sort" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + log "github.com/sirupsen/logrus" +) + +const ( + ApiUrl = "https://openapi.eyun.360.cn/intf.php" + ClientID = "e4757e933b6486c08ed206ecb6d5d9e684fcb4e2" + ClientSecret = "885fd3231f1c1e37c9f462261a09b8c38cde0c2b" + SecretKey = "e7b24b112a44fdd9ee93bdf998c6ca0e" +) + +// phpUrlEncode encodes a string in PHP/JS style used by SDK +// JavaScript's encodeURIComponent keeps - _ . ! ~ * ' ( ) unencoded, +// but the sign function encodes them again +func phpUrlEncode(str string) string { + // First, do standard encoding but keep certain chars + encoded := url.QueryEscape(str) + // url.QueryEscape already encodes most things, but we need to ensure + // these specific characters are encoded as the JS does + replacer := strings.NewReplacer( + "!", "%21", + "'", "%27", + "(", "%28", + ")", "%29", + "*", "%2A", + ",", "%2C", + "~", "%7E", + ) + encoded = replacer.Replace(encoded) + // %20 should be + (last step) + encoded = strings.ReplaceAll(encoded, "%20", "+") + return encoded +} + +// generateSign generates MD5 signature for API request +func generateSign(params map[string]string) string { + // Sort keys alphabetically + keys := make([]string, 0, len(params)) + for k := range params { + keys = append(keys, k) + } + sort.Strings(keys) + + // Build key=encodedValue string + pairs := make([]string, 0, len(keys)) + for _, k := range keys { + encodedValue := phpUrlEncode(params[k]) + pairs = append(pairs, fmt.Sprintf("%s=%s", k, encodedValue)) + } + str := strings.Join(pairs, "&") + + // Append secret key + str += SecretKey + + // Calculate MD5 + hash := md5.Sum([]byte(str)) + return hex.EncodeToString(hash[:]) +} + +func (d *Qihoo360) getAuth() (*AuthResp, error) { + // Check if we have cached auth and it's not expired (with 5 min buffer) + if d.authInfo != nil && d.authExpire > 0 && time.Now().Unix() < d.authExpire-300 { + return d.authInfo, nil + } + + params := map[string]string{ + "method": "Oauth.getAccessTokenByApiKey", + "client_id": ClientID, + "client_secret": ClientSecret, + "api_key": d.APIKey, + "grant_type": "authorization_code", + } + + // Build URL with query parameters (no sign needed for auth request) + var resp AuthResp + req := base.RestyClient.R().SetResult(&resp) + for k, v := range params { + req.SetQueryParam(k, v) + } + + res, err := req.Get(ApiUrl) + if err != nil { + // Clear auth cache on error to force re-authentication next time + d.authInfo = nil + d.authExpire = 0 + return nil, err + } + + log.Debugf("Auth Response: %s", res.String()) + + if resp.Errno != 0 { + // Clear auth cache on error to force re-authentication next time + d.authInfo = nil + d.authExpire = 0 + return nil, fmt.Errorf("auth failed: %s", resp.Errmsg) + } + + // Cache auth info + d.authInfo = &resp + // access_token_expire is already a Unix timestamp, not a duration + if resp.Data.AccessTokenExpire > 0 { + d.authExpire = resp.Data.AccessTokenExpire + } else { + // Default to 1 hour if not provided + d.authExpire = time.Now().Unix() + 3600 + } + + return &resp, nil +} + +func (d *Qihoo360) request(method string, params map[string]string, result interface{}, excluded ...string) ([]byte, error) { + return d.requestWithRetry(method, params, result, 0, excluded...) +} + +func (d *Qihoo360) requestWithRetry(method string, params map[string]string, result interface{}, retryCount int, excluded ...string) ([]byte, error) { + // Prevent infinite retry loops + const maxRetries = 2 + if retryCount >= maxRetries { + return nil, fmt.Errorf("max retries (%d) exceeded for method %s", maxRetries, method) + } + + // Get auth if not already authenticated or expired + if d.authInfo == nil || d.authExpire <= 0 || time.Now().Unix() >= d.authExpire-300 { + _, err := d.getAuth() + if err != nil { + return nil, err + } + } + + // Ensure authInfo is set before proceeding + if d.authInfo == nil { + return nil, fmt.Errorf("authentication failed: no auth info") + } + + // Build excluded params map + excludedMap := make(map[string]bool) + if len(excluded) > 0 { + for _, key := range excluded { + excludedMap[key] = true + } + } + + // Build params for sign (excluding specified params) + signParams := map[string]string{ + "method": method, + "access_token": d.authInfo.Data.AccessToken, + "qid": d.authInfo.Data.Qid, + } + + // Add params to sign if not excluded + for k, v := range params { + if !excludedMap[k] { + signParams[k] = v + } + } + + // Generate sign + sign := generateSign(signParams) + + log.Debugf("Request method: %s", method) + + // File.getList, Sync.getVerifiedDownLoadUrl, and Sync.getUploadFileAddr use GET + var err error + + if method == "File.getList" || method == "Sync.getVerifiedDownLoadUrl" || method == "Sync.getUploadFileAddr" || method == "User.getUserDetail" { + // GET request: params in query string + allParams := map[string]string{ + "method": method, + "access_token": d.authInfo.Data.AccessToken, + "qid": d.authInfo.Data.Qid, + "sign": sign, + } + for k, v := range params { + if !excludedMap[k] { + allParams[k] = v + } + } + req := base.RestyClient.R(). + SetQueryParams(allParams). + SetResult(result). + SetHeader("Access-Token", d.authInfo.Data.AccessToken) + if method == "Sync.getVerifiedDownLoadUrl" { + req.SetHeader("User-Agent", "yunpan_mcp_server") + } + _, err = req.Get(ApiUrl) + if err != nil { + return nil, err + } + } else { + // POST request: basic params in query, all params in form + queryParams := map[string]string{ + "method": method, + "access_token": d.authInfo.Data.AccessToken, + "qid": d.authInfo.Data.Qid, + "sign": sign, + } + + formData := make(map[string]string) + for k, v := range params { + formData[k] = v + } + + _, err = base.RestyClient.R(). + SetQueryParams(queryParams). + SetFormData(formData). + SetResult(result). + SetHeader("Access-Token", d.authInfo.Data.AccessToken). + SetHeader("Content-Type", "application/x-www-form-urlencoded"). + Post(ApiUrl) + if err != nil { + return nil, err + } + } + + log.Debugf("Response data received") + + // Check if we got an authentication error from the API + // If errno is -1 or -2, it usually means token is invalid/expired + if resp, ok := result.(*FileListResp); ok { + if resp.Errno == -1 || resp.Errno == -2 { + log.Debugf("Auth token expired (errno: %d), clearing cache and retrying (attempt %d)", resp.Errno, retryCount+1) + d.authInfo = nil + d.authExpire = 0 + // Retry with fresh auth + return d.requestWithRetry(method, params, result, retryCount+1, excluded...) + } + } else if resp, ok := result.(*DownloadUrlResp); ok { + if resp.Errno == -1 || resp.Errno == -2 { + log.Debugf("Auth token expired (errno: %d), clearing cache and retrying (attempt %d)", resp.Errno, retryCount+1) + d.authInfo = nil + d.authExpire = 0 + return d.requestWithRetry(method, params, result, retryCount+1, excluded...) + } + } else if resp, ok := result.(*UserDetailResp); ok { + if resp.Errno == -1 || resp.Errno == -2 { + log.Debugf("Auth token expired (errno: %d), clearing cache and retrying (attempt %d)", resp.Errno, retryCount+1) + d.authInfo = nil + d.authExpire = 0 + return d.requestWithRetry(method, params, result, retryCount+1, excluded...) + } + } else if resp, ok := result.(*CommonResp); ok { + if resp.Errno == -1 || resp.Errno == -2 { + log.Debugf("Auth token expired (errno: %d), clearing cache and retrying (attempt %d)", resp.Errno, retryCount+1) + d.authInfo = nil + d.authExpire = 0 + return d.requestWithRetry(method, params, result, retryCount+1, excluded...) + } + } + + return nil, nil +} + +func (d *Qihoo360) getFiles(path string, page int, pageSize int) ([]File, error) { + params := map[string]string{ + "path": path, + "page": fmt.Sprintf("%d", page), + "page_size": fmt.Sprintf("%d", pageSize), + } + + var resp FileListResp + _, err := d.request("File.getList", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("get files failed: %s", resp.Errmsg) + } + + // Normalize name display and full path for each file/dir + for i := range resp.Data.NodeList { + rawName := resp.Data.NodeList[i].Name + // Trim leading slash and trailing slash for dir name + trimmed := strings.TrimPrefix(rawName, "/") + trimmed = strings.TrimSuffix(trimmed, "/") + base := pathpkg.Base(trimmed) + resp.Data.NodeList[i].Name = base + + // Construct full path + var fullPath string + if path == "/" { + fullPath = "/" + base + } else { + fullPath = path + base + } + if resp.Data.NodeList[i].Type == "1" && !strings.HasSuffix(fullPath, "/") { + fullPath += "/" + } + resp.Data.NodeList[i].Path = fullPath + } + + return resp.Data.NodeList, nil +} + +func (d *Qihoo360) getDownloadUrl(nid string) (string, error) { + params := map[string]string{ + "nid": nid, + } + + var resp DownloadUrlResp + _, err := d.request("Sync.getVerifiedDownLoadUrl", params, &resp) + if err != nil { + return "", err + } + + if resp.Errno != 0 { + return "", fmt.Errorf("get download url failed: %s", resp.Errmsg) + } + + return resp.Data.DownloadUrl, nil +} + +func (d *Qihoo360) getUploadAddr(fname string, fsize int64, fhash string, fctime, fmtime int64) (*UploadAddrResp, error) { + // Build all query parameters + params := map[string]string{ + "owner_qid": d.authInfo.Data.Qid, + "fname": fname, + "fsize": fmt.Sprintf("%d", fsize), + "fctime": fmt.Sprintf("%d", fctime), + "fmtime": fmt.Sprintf("%d", fmtime), + "fhash": fhash, + "qid": d.authInfo.Data.Qid, + "fattr": "0", + "token": d.authInfo.Data.Token, + "tk": "", + "devtype": "ecs_openapi", + } + + // Calculate sign using only specific parameters (per SDK) + signParams := map[string]string{ + "fhash": fhash, + "qid": d.authInfo.Data.Qid, + "method": "Sync.getUploadFileAddr", + "fname": fname, + "fsize": fmt.Sprintf("%d", fsize), + "access_token": d.authInfo.Data.AccessToken, + } + params["sign"] = generateSign(signParams) + + var resp UploadAddrResp + _, err := d.request("Sync.getUploadFileAddr", params, &resp) + if err != nil { + return nil, err + } + if resp.Errno != 0 { + return nil, fmt.Errorf("get upload addr failed: %s", resp.Errmsg) + } + return &resp, nil +} + +func (d *Qihoo360) getUserDetail() (*UserDetailResp, error) { + params := map[string]string{} + + var resp UserDetailResp + _, err := d.request("User.getUserDetail", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("get user detail failed: %s", resp.Errmsg) + } + + return &resp, nil +} diff --git a/drivers/quark_open/driver.go b/drivers/quark_open/driver.go index 7b6b31331..f0b8baf09 100644 --- a/drivers/quark_open/driver.go +++ b/drivers/quark_open/driver.go @@ -1,7 +1,6 @@ package quark_open import ( - "bytes" "context" "encoding/hex" "errors" @@ -9,12 +8,15 @@ import ( "hash" "io" "net/http" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" ) @@ -174,7 +176,7 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return err } // 如果预上传已经完成，直接返回--秒传 - if pre.Data.Finish == true { + if pre.Data.Finish { up(100) return nil } @@ -188,46 +190,48 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File } // part up + ss, err := streamPkg.NewStreamSectionReader(stream, int(pre.Data.PartSize), &up) + if err != nil { + return err + } total := stream.GetSize() - left := total - part := make([]byte, pre.Data.PartSize) // 用于存储每个分片的ETag，后续commit时需要 - etags := make([]string, len(partInfo)) + etags := make([]string, 0, len(partInfo)) // 遍历上传每个分片 - for i, urlInfo := range upUrlInfo.UploadUrls { + for i := range len(upUrlInfo.UploadUrls) { if utils.IsCanceled(ctx) { return ctx.Err() } - currentSize := int64(urlInfo.PartSize) - if left < currentSize { - part = part[:left] - } else { - part = part[:currentSize] - } - - // 读取分片数据 - n, err := io.ReadFull(stream, part) - if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + offset := int64(i) * pre.Data.PartSize + size := min(pre.Data.PartSize, total-offset) + rd, err := ss.GetSectionReader(offset, size) + if err != nil { return err } - - // 准备上传分片 - reader := driver.NewLimitedUploadStream(ctx, bytes.NewReader(part)) - etag, err := d.upPart(ctx, upUrlInfo, i, reader) + err = retry.Do(func() error { + rd.Seek(0, io.SeekStart) + etag, err := d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) + if err != nil { + return err + } + etags = append(etags, etag) + return nil + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second)) + ss.FreeSectionReader(rd) if err != nil { return fmt.Errorf("failed to upload part %d: %w", i, err) } - // 保存ETag，用于后续commit - etags[i] = etag - - // 更新剩余大小和进度 - left -= int64(n) - up(float64(total-left) / float64(total) * 100) + up(95 * float64(offset+size) / float64(total)) } + defer up(100) return d.upFinish(ctx, pre, partInfo, etags) } diff --git a/drivers/quark_open/util.go b/drivers/quark_open/util.go index 78f4e4a2c..788ca0e99 100644 --- a/drivers/quark_open/util.go +++ b/drivers/quark_open/util.go @@ -341,8 +341,7 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber req.Header.Set("User-Agent", "Go-http-client/1.1") // 发送请求 - client := &http.Client{} - resp, err := client.Do(req) + resp, err := base.HttpClient.Do(req) if err != nil { return "", err } @@ -441,7 +440,6 @@ func (d *QuarkOpen) _refreshToken() (string, string, error) { u := d.APIAddress var resp RefreshTokenOnlineAPIResp _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index f7ebfc603..a1371243c 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -1,7 +1,6 @@ package quark import ( - "bytes" "context" "encoding/hex" "hash" @@ -13,9 +12,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" - log "github.com/sirupsen/logrus" ) type QuarkOrUC struct { @@ -159,56 +159,63 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File if err != nil { return err } - log.Debugln("hash: ", md5Str, sha1Str) // hash finish, err := d.upHash(md5Str, sha1Str, pre.Data.TaskId) if err != nil { return err } if finish { + up(100) return nil } // part up + ss, err := streamPkg.NewStreamSectionReader(stream, pre.Metadata.PartSize, &up) + if err != nil { + return err + } total := stream.GetSize() - left := total partSize := int64(pre.Metadata.PartSize) - part := make([]byte, partSize) - count := int(total / partSize) - if total%partSize > 0 { - count++ - } - md5s := make([]string, 0, count) - partNumber := 1 - for left > 0 { + uploadNums := int((total + partSize - 1) / partSize) + md5s := make([]string, 0, uploadNums) + for partIndex := range uploadNums { if utils.IsCanceled(ctx) { return ctx.Err() } - if left < partSize { - part = part[:left] - } - n, err := io.ReadFull(stream, part) + offset := int64(partIndex) * partSize + size := min(partSize, total-offset) + rd, err := ss.GetSectionReader(offset, size) if err != nil { return err } - left -= int64(n) - log.Debugf("left: %d", left) - reader := driver.NewLimitedUploadStream(ctx, bytes.NewReader(part)) - m, err := d.upPart(ctx, pre, stream.GetMimetype(), partNumber, reader) - // m, err := driver.UpPart(pre, file.GetMIMEType(), partNumber, bytes, account, md5Str, sha1Str) + err = retry.Do(func() error { + rd.Seek(0, io.SeekStart) + m, err := d.upPart(ctx, pre, stream.GetMimetype(), partIndex+1, driver.NewLimitedUploadStream(ctx, rd)) + if err != nil { + return err + } + if m == "finish" { + up(100) + return nil + } + md5s = append(md5s, m) + return nil + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second)) + ss.FreeSectionReader(rd) if err != nil { return err } - if m == "finish" { - return nil - } - md5s = append(md5s, m) - partNumber++ - up(100 * float64(total-left) / float64(total)) + up(95 * float64(offset+size) / float64(total)) } + up(97) err = d.upCommit(pre, md5s) if err != nil { return err } + defer up(100) return d.upFinish(pre) } @@ -220,7 +227,7 @@ func (d *QuarkOrUC) GetDetails(ctx context.Context) (*model.StorageDetails, erro return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: memberInfo.Data.TotalCapacity, - FreeSpace: memberInfo.Data.TotalCapacity - memberInfo.Data.UseCapacity, + UsedSpace: memberInfo.Data.UseCapacity, }, }, nil } diff --git a/drivers/quark_uc/types.go b/drivers/quark_uc/types.go index cea2b8830..07286e293 100644 --- a/drivers/quark_uc/types.go +++ b/drivers/quark_uc/types.go @@ -265,8 +265,8 @@ type MemberResp struct { Data struct { MemberType string `json:"member_type"` CreatedAt uint64 `json:"created_at"` - SecretUseCapacity uint64 `json:"secret_use_capacity"` - UseCapacity uint64 `json:"use_capacity"` + SecretUseCapacity int64 `json:"secret_use_capacity"` + UseCapacity int64 `json:"use_capacity"` IsNewUser bool `json:"is_new_user"` MemberStatus struct { Vip string `json:"VIP"` @@ -274,8 +274,8 @@ type MemberResp struct { MiniVip string `json:"MINI_VIP"` SuperVip string `json:"SUPER_VIP"` } `json:"member_status"` - SecretTotalCapacity uint64 `json:"secret_total_capacity"` - TotalCapacity uint64 `json:"total_capacity"` + SecretTotalCapacity int64 `json:"secret_total_capacity"` + TotalCapacity int64 `json:"total_capacity"` } `json:"data"` Metadata struct { RangeSize int `json:"range_size"` diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index 01f63ef0c..87798c6ef 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "fmt" + "html" "io" "net/http" "strconv" @@ -70,9 +71,11 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { page := 1 size := 100 query := map[string]string{ - "pdir_fid": parent, - "_size": strconv.Itoa(size), - "_fetch_total": "1", + "pdir_fid": parent, + "_size": strconv.Itoa(size), + "_fetch_total": "1", + "fetch_all_file": "1", + "fetch_risk_file_name": "1", } if d.OrderBy != "none" { query["_sort"] = "file_type:asc," + d.OrderBy + ":" + d.OrderDirection @@ -87,6 +90,7 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { return nil, err } for _, file := range resp.Data.List { + file.FileName = html.UnescapeString(file.FileName) if d.OnlyListVideoFile { // 开启后 只列出视频文件和文件夹 if file.IsDir() || file.Category == 1 { @@ -229,25 +233,30 @@ x-oss-user-agent:aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit // } //} u := fmt.Sprintf("https://%s.%s/%s", pre.Data.Bucket, pre.Data.UploadUrl[7:], pre.Data.ObjKey) - res, err := base.RestyClient.R().SetContext(ctx). - SetHeaders(map[string]string{ - "Authorization": resp.Data.AuthKey, - "Content-Type": mineType, - "Referer": "https://pan.quark.cn/", - "x-oss-date": timeStr, - "x-oss-user-agent": "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit", - }). - SetQueryParams(map[string]string{ - "partNumber": strconv.Itoa(partNumber), - "uploadId": pre.Data.UploadId, - }).SetBody(bytes).Put(u) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u, bytes) if err != nil { return "", err } - if res.StatusCode() != 200 { - return "", fmt.Errorf("up status: %d, error: %s", res.StatusCode(), res.String()) + req.Header.Set("Authorization", resp.Data.AuthKey) + req.Header.Set("Content-Type", mineType) + req.Header.Set("Referer", "https://pan.quark.cn/") + req.Header.Set("x-oss-date", timeStr) + req.Header.Set("x-oss-user-agent", "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit") + q := req.URL.Query() + q.Add("partNumber", strconv.Itoa(partNumber)) + q.Add("uploadId", pre.Data.UploadId) + req.URL.RawQuery = q.Encode() + res, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + respBody, _ := io.ReadAll(res.Body) + return "", fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(respBody)) } - return res.Header().Get("Etag"), nil + return res.Header.Get("Etag"), nil } func (d *QuarkOrUC) upCommit(pre UpPreResp, md5s []string) error { diff --git a/drivers/quark_uc_tv/util.go b/drivers/quark_uc_tv/util.go index d68a2f3c3..c0da6eb6b 100644 --- a/drivers/quark_uc_tv/util.go +++ b/drivers/quark_uc_tv/util.go @@ -8,6 +8,7 @@ import ( "errors" "net/http" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -70,8 +71,16 @@ func (d *QuarkUCTV) request(ctx context.Context, pathname string, method string, return nil, err } // 判断 是否需要 刷新 access_token - if e.Status == -1 && e.Errno == 10001 { - // token 过期 + errInfoLower := strings.ToLower(strings.TrimSpace(e.ErrorInfo)) + maybeTokenInvalid := + (e.Status == -1 && (e.Errno == 10001 || e.Errno == 11001)) || + (errInfoLower != "" && + (strings.Contains(errInfoLower, "access token") || + strings.Contains(errInfoLower, "access_token") || + strings.Contains(errInfoLower, "token无效") || + strings.Contains(errInfoLower, "token 无效"))) + if maybeTokenInvalid { + // token 过期 / 无效 err = d.getRefreshTokenByTV(ctx, d.Addition.RefreshToken, true) if err != nil { return nil, err diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index d19ead477..a9edef10b 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" @@ -24,9 +25,10 @@ import ( type S3 struct { model.Storage Addition - Session *session.Session - client *s3.S3 - linkClient *s3.S3 + Session *session.Session + client *s3.S3 + linkClient *s3.S3 + directUploadClient *s3.S3 config driver.Config cron *cron.Cron @@ -52,16 +54,18 @@ func (d *S3) Init(ctx context.Context) error { if err != nil { log.Errorln("Doge init session error:", err) } - d.client = d.getClient(false) - d.linkClient = d.getClient(true) + d.client = d.getClient(ClientTypeNormal) + d.linkClient = d.getClient(ClientTypeLink) + d.directUploadClient = d.getClient(ClientTypeDirectUpload) }) } err := d.initSession() if err != nil { return err } - d.client = d.getClient(false) - d.linkClient = d.getClient(true) + d.client = d.getClient(ClientTypeNormal) + d.linkClient = d.getClient(ClientTypeLink) + d.directUploadClient = d.getClient(ClientTypeDirectUpload) return nil } @@ -210,4 +214,33 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up return err } +func (d *S3) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false) + req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{ + Bucket: &d.Bucket, + Key: &path, + }) + if req == nil { + return nil, fmt.Errorf("failed to create PutObject request") + } + link, err := req.Presign(time.Hour * time.Duration(d.SignURLExpire)) + if err != nil { + return nil, err + } + return &model.HttpDirectUploadInfo{ + UploadURL: link, + Method: "PUT", + }, nil +} + var _ driver.Driver = (*S3)(nil) diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 59c3bbb05..b2439c683 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -21,6 +21,8 @@ type Addition struct { ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false"` + DirectUploadHost string `json:"direct_upload_host" required:"false"` } func init() { diff --git a/drivers/s3/util.go b/drivers/s3/util.go index b58971760..c84380786 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -41,9 +41,15 @@ func (d *S3) initSession() error { return err } -func (d *S3) getClient(link bool) *s3.S3 { +const ( + ClientTypeNormal = iota + ClientTypeLink + ClientTypeDirectUpload +) + +func (d *S3) getClient(clientType int) *s3.S3 { client := s3.New(d.Session) - if link && d.CustomHost != "" { + if clientType == ClientTypeLink && d.CustomHost != "" { client.Handlers.Build.PushBack(func(r *request.Request) { if r.HTTPRequest.Method != http.MethodGet { return @@ -58,6 +64,20 @@ func (d *S3) getClient(link bool) *s3.S3 { } }) } + if clientType == ClientTypeDirectUpload && d.DirectUploadHost != "" { + client.Handlers.Build.PushBack(func(r *request.Request) { + if r.HTTPRequest.Method != http.MethodPut { + return + } + split := strings.SplitN(d.DirectUploadHost, "://", 2) + if utils.SliceContains([]string{"http", "https"}, split[0]) { + r.HTTPRequest.URL.Scheme = split[0] + r.HTTPRequest.URL.Host = split[1] + } else { + r.HTTPRequest.URL.Host = d.DirectUploadHost + } + }) + } return client } @@ -78,8 +98,8 @@ func getPlaceholderName(placeholder string) string { return placeholder } -func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { - prefix = getKey(prefix, true) +func (d *S3) listV1(dirPath string, args model.ListArgs) ([]model.Obj, error) { + prefix := getKey(dirPath, true) log.Debugf("list: %s", prefix) files := make([]model.Obj, 0) marker := "" @@ -97,7 +117,7 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { for _, object := range listObjectsResult.CommonPrefixes { name := path.Base(strings.Trim(*object.Prefix, "/")) file := model.Object{ - //Id: *object.Key, + Path: path.Join(dirPath, name), Name: name, Modified: d.Modified, IsFolder: true, @@ -110,7 +130,7 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { continue } file := model.Object{ - //Id: *object.Key, + Path: path.Join(dirPath, name), Name: name, Size: *object.Size, Modified: *object.LastModified, @@ -129,8 +149,8 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { return files, nil } -func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { - prefix = getKey(prefix, true) +func (d *S3) listV2(dirPath string, args model.ListArgs) ([]model.Obj, error) { + prefix := getKey(dirPath, true) files := make([]model.Obj, 0) var continuationToken, startAfter *string for { @@ -149,7 +169,7 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { for _, object := range listObjectsResult.CommonPrefixes { name := path.Base(strings.Trim(*object.Prefix, "/")) file := model.Object{ - //Id: *object.Key, + Path: path.Join(dirPath, name), Name: name, Modified: d.Modified, IsFolder: true, @@ -165,7 +185,7 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { continue } file := model.Object{ - //Id: *object.Key, + Path: path.Join(dirPath, name), Name: name, Size: *object.Size, Modified: *object.LastModified, @@ -197,9 +217,10 @@ func (d *S3) copy(ctx context.Context, src string, dst string, isDir bool) error func (d *S3) copyFile(ctx context.Context, src string, dst string) error { srcKey := getKey(src, false) dstKey := getKey(dst, false) + encodedKey := strings.ReplaceAll(url.PathEscape(d.Bucket+"/"+srcKey), "+", "%2B") input := &s3.CopyObjectInput{ Bucket: &d.Bucket, - CopySource: aws.String(url.PathEscape(d.Bucket + "/" + srcKey)), + CopySource: aws.String(encodedKey), Key: &dstKey, } _, err := d.client.CopyObject(input) diff --git a/drivers/seafile/driver.go b/drivers/seafile/driver.go index 73c21645e..221ac4361 100644 --- a/drivers/seafile/driver.go +++ b/drivers/seafile/driver.go @@ -4,10 +4,11 @@ import ( "context" "fmt" "net/http" + stdpath "path" "strings" - "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" @@ -18,7 +19,7 @@ type Seafile struct { Addition authorization string - libraryMap map[string]*LibraryInfo + root model.Obj } func (d *Seafile) Config() driver.Config { @@ -31,46 +32,103 @@ func (d *Seafile) GetAddition() driver.Additional { func (d *Seafile) Init(ctx context.Context) error { d.Address = strings.TrimSuffix(d.Address, "/") + err := d.getToken() + if err != nil { + return err + } d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) - d.libraryMap = make(map[string]*LibraryInfo) - return d.getToken() + if d.RepoId != "" { + library, err := d.getLibraryInfo(d.RepoId) + if err != nil { + return err + } + library.path = d.RootFolderPath + library.ObjMask = model.Locked + d.root = &LibraryInfo{ + LibraryItemResp: library, + } + return nil + } + if len(d.RootFolderPath) <= 1 { + d.root = &model.Object{ + Name: "root", + Path: d.RootFolderPath, + IsFolder: true, + Modified: d.Modified, + Mask: model.Locked, + } + return nil + } + + var resp []LibraryItemResp + _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { + req.SetResult(&resp) + }) + if err != nil { + return err + } + for _, library := range resp { + p, found := strings.CutPrefix(d.RootFolderPath[1:], library.Name) + if !found { + continue + } + if p == "" { + p = "/" + } else if p[0] != '/' { + continue + } + // d.RepoId = library.Id + // d.RootFolderPath = p + + library.path = p + library.ObjMask = model.Locked + d.root = &LibraryInfo{ + LibraryItemResp: library, + } + return nil + } + return fmt.Errorf("Library for root folder path %q not found", d.RootFolderPath) } func (d *Seafile) Drop(ctx context.Context) error { + d.root = nil return nil } +func (d *Seafile) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root == nil { + return nil, errs.StorageNotInit + } + return d.root, nil +} + func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) (result []model.Obj, err error) { path := dir.GetPath() - if path == d.RootFolderPath { - libraries, err := d.listLibraries() - if err != nil { - return nil, err - } - if path == "/" && d.RepoId == "" { - return utils.SliceConvert(libraries, func(f LibraryItemResp) (model.Obj, error) { - return &model.Object{ - Name: f.Name, - Modified: time.Unix(f.Modified, 0), - Size: f.Size, - IsFolder: true, - }, nil - }) - } - } - var repo *LibraryInfo - repo, path, err = d.getRepoAndPath(path) - if err != nil { - return nil, err - } - if repo.Encrypted { - err = d.decryptLibrary(repo) - if err != nil { - return nil, err + switch o := dir.(type) { + default: + var resp []LibraryItemResp + _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { + req.SetResult(&resp) + }) + return utils.SliceConvert(resp, func(f LibraryItemResp) (model.Obj, error) { + f.path = path + return &LibraryInfo{ + LibraryItemResp: f, + }, nil + }) + case *LibraryInfo: + if o.Encrypted { + err = d.decryptLibrary(o) + if err != nil { + return nil, err + } } + case *RepoItemResp: + // do nothing } - var resp []RepoDirItemResp - _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/dir/", repo.Id), func(req *resty.Request) { + + var resp []RepoItemResp + _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/dir/", dir.GetID()), func(req *resty.Request) { req.SetResult(&resp).SetQueryParams(map[string]string{ "p": path, }) @@ -78,27 +136,17 @@ func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) if err != nil { return nil, err } - return utils.SliceConvert(resp, func(f RepoDirItemResp) (model.Obj, error) { - return &model.ObjThumb{ - Object: model.Object{ - Name: f.Name, - Modified: time.Unix(f.Modified, 0), - Size: f.Size, - IsFolder: f.Type == "dir", - }, - // Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, - }, nil + return utils.SliceConvert(resp, func(f RepoItemResp) (model.Obj, error) { + f.path = stdpath.Join(path, f.Name) + f.repoID = dir.GetID() + return &f, nil }) } func (d *Seafile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - repo, path, err := d.getRepoAndPath(file.GetPath()) - if err != nil { - return nil, err - } - res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/file/", file.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": file.GetPath(), "reuse": "1", }) }) @@ -111,14 +159,9 @@ func (d *Seafile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Seafile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - repo, path, err := d.getRepoAndPath(parentDir.GetPath()) - if err != nil { - return err - } - path, _ = utils.JoinBasePath(path, dirName) - _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/dir/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/dir/", parentDir.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": stdpath.Join(parentDir.GetPath(), dirName), }).SetFormData(map[string]string{ "operation": "mkdir", }) @@ -127,34 +170,22 @@ func (d *Seafile) MakeDir(ctx context.Context, parentDir model.Obj, dirName stri } func (d *Seafile) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - repo, path, err := d.getRepoAndPath(srcObj.GetPath()) - if err != nil { - return err - } - dstRepo, dstPath, err := d.getRepoAndPath(dstDir.GetPath()) - if err != nil { - return err - } - _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", srcObj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": srcObj.GetPath(), }).SetFormData(map[string]string{ "operation": "move", - "dst_repo": dstRepo.Id, - "dst_dir": dstPath, + "dst_repo": dstDir.GetID(), + "dst_dir": dstDir.GetPath(), }) }, true) return err } func (d *Seafile) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - repo, path, err := d.getRepoAndPath(srcObj.GetPath()) - if err != nil { - return err - } - _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", srcObj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": srcObj.GetPath(), }).SetFormData(map[string]string{ "operation": "rename", "newname": newName, @@ -164,47 +195,31 @@ func (d *Seafile) Rename(ctx context.Context, srcObj model.Obj, newName string) } func (d *Seafile) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - repo, path, err := d.getRepoAndPath(srcObj.GetPath()) - if err != nil { - return err - } - dstRepo, dstPath, err := d.getRepoAndPath(dstDir.GetPath()) - if err != nil { - return err - } - _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", srcObj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": srcObj.GetPath(), }).SetFormData(map[string]string{ "operation": "copy", - "dst_repo": dstRepo.Id, - "dst_dir": dstPath, + "dst_repo": dstDir.GetID(), + "dst_dir": dstDir.GetPath(), }) }) return err } func (d *Seafile) Remove(ctx context.Context, obj model.Obj) error { - repo, path, err := d.getRepoAndPath(obj.GetPath()) - if err != nil { - return err - } - _, err = d.request(http.MethodDelete, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodDelete, fmt.Sprintf("/api2/repos/%s/file/", obj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": obj.GetPath(), }) }) return err } func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - repo, path, err := d.getRepoAndPath(dstDir.GetPath()) - if err != nil { - return err - } - res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/upload-link/", repo.Id), func(req *resty.Request) { + res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/upload-link/", dstDir.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": dstDir.GetPath(), }) }) if err != nil { @@ -220,7 +235,7 @@ func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame }) req.SetFileReader("file", s.GetName(), r). SetFormData(map[string]string{ - "parent_dir": path, + "parent_dir": dstDir.GetPath(), "replace": "1", }). SetContext(ctx) diff --git a/drivers/seafile/types.go b/drivers/seafile/types.go index 47cb322df..29a51ce84 100644 --- a/drivers/seafile/types.go +++ b/drivers/seafile/types.go @@ -1,6 +1,11 @@ package seafile -import "time" +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) type AuthTokenResp struct { Token string `json:"token"` @@ -13,7 +18,41 @@ type RepoItemResp struct { Size int64 `json:"size"` Modified int64 `json:"mtime"` Permission string `json:"permission"` + + path string + model.ObjMask + repoID string +} + +func (l *RepoItemResp) IsDir() bool { + return l.Type == "dir" +} +func (l *RepoItemResp) GetPath() string { + return l.path +} +func (l *RepoItemResp) GetName() string { + return l.Name +} +func (l *RepoItemResp) ModTime() time.Time { + return time.Unix(l.Modified, 0) +} +func (l *RepoItemResp) CreateTime() time.Time { + return l.ModTime() +} +func (l *RepoItemResp) GetSize() int64 { + return l.Size } +func (l *RepoItemResp) GetID() string { + if l.repoID != "" { + return l.repoID + } + return l.Id +} +func (l *RepoItemResp) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +var _ model.Obj = (*RepoItemResp)(nil) type LibraryItemResp struct { RepoItemResp @@ -33,12 +72,12 @@ type LibraryItemResp struct { SizeFormatted string `json:"size_formatted"` } -type RepoDirItemResp struct { - RepoItemResp -} - type LibraryInfo struct { LibraryItemResp decryptedTime time.Time decryptedSuccess bool -} \ No newline at end of file +} + +func (l *LibraryInfo) IsDir() bool { + return true +} diff --git a/drivers/seafile/util.go b/drivers/seafile/util.go index 6b5d09939..f2c0b6659 100644 --- a/drivers/seafile/util.go +++ b/drivers/seafile/util.go @@ -7,9 +7,6 @@ import ( "strings" "time" - "github.com/OpenListTeam/OpenList/v4/internal/errs" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" ) @@ -71,73 +68,12 @@ func (d *Seafile) request(method string, pathname string, callback base.ReqCallb return res.Body(), nil } -func (d *Seafile) getRepoAndPath(fullPath string) (repo *LibraryInfo, path string, err error) { - libraryMap := d.libraryMap - repoId := d.Addition.RepoId - if repoId != "" { - if len(repoId) == 36 /* uuid */ { - for _, library := range libraryMap { - if library.Id == repoId { - return library, fullPath, nil - } - } - } - } else { - var repoName string - str := fullPath[1:] - pos := strings.IndexRune(str, '/') - if pos == -1 { - repoName = str - } else { - repoName = str[:pos] - } - path = utils.FixAndCleanPath(fullPath[1+len(repoName):]) - if library, ok := libraryMap[repoName]; ok { - return library, path, nil - } - } - return nil, "", errs.ObjectNotFound -} - -func (d *Seafile) listLibraries() (resp []LibraryItemResp, err error) { - repoId := d.Addition.RepoId - if repoId == "" { - _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { - req.SetResult(&resp) - }) - } else { - var oneResp LibraryItemResp - _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/", repoId), func(req *resty.Request) { - req.SetResult(&oneResp) - }) - if err == nil { - resp = append(resp, oneResp) - } - } - if err != nil { - return nil, err - } - libraryMap := make(map[string]*LibraryInfo) - var putLibraryMap func(library LibraryItemResp, index int) - putLibraryMap = func(library LibraryItemResp, index int) { - name := library.Name - if index > 0 { - name = fmt.Sprintf("%s (%d)", name, index) - } - if _, exist := libraryMap[name]; exist { - putLibraryMap(library, index+1) - } else { - libraryInfo := LibraryInfo{} - data, _ := utils.Json.Marshal(library) - _ = utils.Json.Unmarshal(data, &libraryInfo) - libraryMap[name] = &libraryInfo - } - } - for _, library := range resp { - putLibraryMap(library, 0) - } - d.libraryMap = libraryMap - return resp, nil +func (d *Seafile) getLibraryInfo(repoId string) (LibraryItemResp, error) { + var oneResp LibraryItemResp + _, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/", repoId), func(req *resty.Request) { + req.SetResult(&oneResp) + }) + return oneResp, err } var repoPwdNotConfigured = errors.New("library password not configured") diff --git a/drivers/sftp/driver.go b/drivers/sftp/driver.go index 17db40380..3aa39537e 100644 --- a/drivers/sftp/driver.go +++ b/drivers/sftp/driver.go @@ -69,15 +69,10 @@ func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* Limiter: stream.ServerDownloadLimit, Ctx: ctx, } - if !d.Config().OnlyLinkMFile { - return &model.Link{ - RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), - SyncClosers: utils.NewSyncClosers(remoteFile), - }, nil - } return &model.Link{ - MFile: mFile, - SyncClosers: utils.NewSyncClosers(remoteFile), + RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), + SyncClosers: utils.NewSyncClosers(remoteFile), + RequireReference: true, }, nil } @@ -136,12 +131,12 @@ func (d *SFTP) GetDetails(ctx context.Context) (*model.StorageDetails, error) { } return nil, err } - total := stat.Blocks * stat.Bsize - free := stat.Bfree * stat.Bsize + total := int64(stat.Blocks * stat.Bsize) + free := int64(stat.Bfree * stat.Bsize) return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: total - free, }, }, nil } diff --git a/drivers/sftp/meta.go b/drivers/sftp/meta.go index 9dada9efa..1c9bd3e33 100644 --- a/drivers/sftp/meta.go +++ b/drivers/sftp/meta.go @@ -16,12 +16,12 @@ type Addition struct { } var config = driver.Config{ - Name: "SFTP", - LocalSort: true, - OnlyLinkMFile: false, - DefaultRoot: "/", - CheckStatus: true, - NoLinkURL: true, + Name: "SFTP", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: "/", + CheckStatus: true, + NoLinkURL: true, } func init() { diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go index 0948834e0..00a32f001 100644 --- a/drivers/sftp/types.go +++ b/drivers/sftp/types.go @@ -11,15 +11,16 @@ import ( func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { symlink := f.Mode()&os.ModeSymlink != 0 + path := stdpath.Join(dir, f.Name()) if !symlink { return &model.Object{ + Path: path, Name: f.Name(), Size: f.Size(), Modified: f.ModTime(), IsFolder: f.IsDir(), }, nil } - path := stdpath.Join(dir, f.Name()) // set target path target, err := d.client.ReadLink(path) if err != nil { @@ -32,6 +33,7 @@ func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { if err != nil { if d.IgnoreSymlinkError { return &model.Object{ + Path: path, Name: f.Name(), Size: f.Size(), Modified: f.ModTime(), diff --git a/drivers/smb/driver.go b/drivers/smb/driver.go index 3aeffbeba..88325e5cf 100644 --- a/drivers/smb/driver.go +++ b/drivers/smb/driver.go @@ -3,6 +3,7 @@ package smb import ( "context" "errors" + "path" "path/filepath" "strings" @@ -54,17 +55,17 @@ func (d *SMB) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m return nil, err } d.updateLastConnTime() - var files []model.Obj + files := make([]model.Obj, 0, len(rawFiles)) for _, f := range rawFiles { - file := model.ObjThumb{ - Object: model.Object{ - Name: f.Name(), - Modified: f.ModTime(), - Size: f.Size(), - IsFolder: f.IsDir(), - Ctime: f.(*smb2.FileStat).CreationTime, - }, + file := model.Object{ + Path: path.Join(fullPath, f.Name()), + Name: f.Name(), + Modified: f.ModTime(), + Size: f.Size(), + IsFolder: f.IsDir(), + Ctime: f.(*smb2.FileStat).CreationTime, } + files = append(files, &file) } return files, nil @@ -86,15 +87,10 @@ func (d *SMB) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*m Limiter: stream.ServerDownloadLimit, Ctx: ctx, } - if !d.Config().OnlyLinkMFile { - return &model.Link{ - RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), - SyncClosers: utils.NewSyncClosers(remoteFile), - }, nil - } return &model.Link{ - MFile: mFile, - SyncClosers: utils.NewSyncClosers(remoteFile), + RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), + SyncClosers: utils.NewSyncClosers(remoteFile), + RequireReference: true, }, nil } @@ -213,10 +209,12 @@ func (d *SMB) GetDetails(ctx context.Context) (*model.StorageDetails, error) { if err != nil { return nil, err } + total := int64(stat.BlockSize() * stat.TotalBlockCount()) + free := int64(stat.BlockSize() * stat.AvailableBlockCount()) return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: stat.BlockSize() * stat.TotalBlockCount(), - FreeSpace: stat.BlockSize() * stat.AvailableBlockCount(), + TotalSpace: total, + UsedSpace: total - free, }, }, nil } diff --git a/drivers/smb/meta.go b/drivers/smb/meta.go index 87a98277e..ce0e5d3b7 100644 --- a/drivers/smb/meta.go +++ b/drivers/smb/meta.go @@ -14,12 +14,12 @@ type Addition struct { } var config = driver.Config{ - Name: "SMB", - LocalSort: true, - OnlyLinkMFile: false, - DefaultRoot: ".", - NoCache: true, - NoLinkURL: true, + Name: "SMB", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: ".", + NoCache: true, + NoLinkURL: true, } func init() { diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go index 010defa90..422f0e1b1 100644 --- a/drivers/strm/driver.go +++ b/drivers/strm/driver.go @@ -12,8 +12,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" + log "github.com/sirupsen/logrus" ) type Strm struct { @@ -39,6 +41,9 @@ func (d *Strm) Init(ctx context.Context) error { if d.Paths == "" { return errors.New("paths is required") } + if d.SaveStrmToLocal && len(d.SaveStrmLocalPath) <= 0 { + return errors.New("SaveStrmLocalPath is required") + } d.pathMap = make(map[string][]string) for _, path := range strings.Split(d.Paths, "\n") { path = strings.TrimSpace(path) @@ -47,6 +52,13 @@ func (d *Strm) Init(ctx context.Context) error { } k, v := getPair(path) d.pathMap[k] = append(d.pathMap[k], v) + if d.SaveStrmToLocal { + err := InsertStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d) + if err != nil { + log.Errorf("insert strmTrie error: %v", err) + continue + } + } } if len(d.pathMap) == 1 { for k := range d.pathMap { @@ -58,26 +70,55 @@ func (d *Strm) Init(ctx context.Context) error { d.autoFlatten = false } - d.supportSuffix = supportSuffix() - if d.FilterFileTypes != "" { - types := strings.Split(d.FilterFileTypes, ",") + var supportTypes []string + if d.FilterFileTypes == "" { + d.FilterFileTypes = "mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" + } + supportTypes = strings.Split(d.FilterFileTypes, ",") + d.supportSuffix = map[string]struct{}{} + for _, ext := range supportTypes { + ext = strings.ToLower(strings.TrimSpace(ext)) + if ext != "" { + d.supportSuffix[ext] = struct{}{} + } + } + + var downloadTypes []string + if d.DownloadFileTypes == "" { + d.DownloadFileTypes = "ass,srt,vtt,sub,strm" + } + downloadTypes = strings.Split(d.DownloadFileTypes, ",") + d.downloadSuffix = map[string]struct{}{} + for _, ext := range downloadTypes { + ext = strings.ToLower(strings.TrimSpace(ext)) + if ext != "" { + d.downloadSuffix[ext] = struct{}{} + } + } + + if d.Version != 5 { + types := strings.Split("mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac", ",") for _, ext := range types { - ext = strings.ToLower(strings.TrimSpace(ext)) - if ext != "" { + if _, ok := d.supportSuffix[ext]; !ok { d.supportSuffix[ext] = struct{}{} + supportTypes = append(supportTypes, ext) } } - } + d.FilterFileTypes = strings.Join(supportTypes, ",") - d.downloadSuffix = downloadSuffix() - if d.DownloadFileTypes != "" { - downloadTypes := strings.Split(d.DownloadFileTypes, ",") - for _, ext := range downloadTypes { - ext = strings.ToLower(strings.TrimSpace(ext)) - if ext != "" { + types = strings.Split("ass,srt,vtt,sub,strm", ",") + for _, ext := range types { + if _, ok := d.downloadSuffix[ext]; !ok { d.downloadSuffix[ext] = struct{}{} + downloadTypes = append(downloadTypes, ext) } } + d.DownloadFileTypes = strings.Join(downloadTypes, ",") + d.PathPrefix = "/d" + d.Version = 5 + } + if len(d.SaveLocalMode) == 0 { + d.SaveLocalMode = SaveLocalInsertMode } return nil } @@ -86,17 +127,17 @@ func (d *Strm) Drop(ctx context.Context) error { d.pathMap = nil d.downloadSuffix = nil d.supportSuffix = nil + for _, path := range strings.Split(d.Paths, "\n") { + RemoveStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d) + } return nil } +func (Addition) GetRootPath() string { + return "/" +} + func (d *Strm) Get(ctx context.Context, path string) (model.Obj, error) { - if utils.PathEqual(path, "/") { - return &model.Object{ - Name: "Root", - IsFolder: true, - Path: "/", - }, nil - } root, sub := d.getRootAndPath(path) dsts, ok := d.pathMap[root] if !ok { @@ -156,7 +197,7 @@ func (d *Strm) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* if file.GetID() == "strm" { link := d.getLink(ctx, file.GetPath()) return &model.Link{ - MFile: strings.NewReader(link), + RangeReader: stream.GetRangeReaderFromMFile(int64(len(link)), strings.NewReader(link)), }, nil } // ftp,s3 diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go new file mode 100644 index 000000000..78ec1e7e6 --- /dev/null +++ b/drivers/strm/hook.go @@ -0,0 +1,237 @@ +package strm + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "os" + stdpath "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" + "github.com/tchap/go-patricia/v2/patricia" +) + +var strmTrie = patricia.NewTrie() + +func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) { + path = utils.FixAndCleanPath(path) + updateLocal := func(driver *Strm, basePath string, objs []model.Obj) { + relParent := strings.TrimPrefix(basePath, utils.GetActualMountPath(driver.MountPath)) + localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent) + for _, obj := range objs { + localPath := stdpath.Join(localParentPath, obj.GetName()) + generateStrm(ctx, driver, obj, localPath) + } + deleteExtraFiles(driver, localParentPath, objs) + } + + _ = strmTrie.VisitPrefixes(patricia.Prefix(path), func(needPathPrefix patricia.Prefix, item patricia.Item) error { + strmDrivers := item.([]*Strm) + needPath := string(needPathPrefix) + restPath := strings.TrimPrefix(path, needPath) + if len(restPath) > 0 && restPath[0] != '/' { + return nil + } + for _, strmDriver := range strmDrivers { + strmObjs := strmDriver.convert2strmObjs(ctx, path, objs) + updateLocal(strmDriver, stdpath.Join(stdpath.Base(needPath), restPath), strmObjs) + } + return nil + }) +} + +func InsertStrm(dstPath string, d *Strm) error { + prefix := patricia.Prefix(strings.TrimRight(dstPath, "/")) + existing := strmTrie.Get(prefix) + + if existing == nil { + if !strmTrie.Insert(prefix, []*Strm{d}) { + return errors.New("failed to insert strm") + } + return nil + } + if lst, ok := existing.([]*Strm); ok { + strmTrie.Set(prefix, append(lst, d)) + } else { + return errors.New("invalid trie item type") + } + + return nil +} + +func RemoveStrm(dstPath string, d *Strm) { + prefix := patricia.Prefix(strings.TrimRight(dstPath, "/")) + existing := strmTrie.Get(prefix) + if existing == nil { + return + } + lst, ok := existing.([]*Strm) + if !ok { + return + } + if len(lst) == 1 && lst[0] == d { + strmTrie.Delete(prefix) + return + } + + for i, di := range lst { + if di == d { + newList := append(lst[:i], lst[i+1:]...) + strmTrie.Set(prefix, newList) + return + } + } +} + +func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) { + if !obj.IsDir() { + if utils.Exists(localPath) && driver.SaveLocalMode == SaveLocalInsertMode { + return + } + link, err := driver.Link(ctx, obj, model.LinkArgs{}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err) + return + } + defer link.Close() + size := link.ContentLength + if size <= 0 { + size = obj.GetSize() + } + rrf, err := stream.GetRangeReaderFromLink(size, link) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to get range reader: %v", localPath, err) + return + } + rc, err := rrf.RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to read range: %v", localPath, err) + return + } + defer rc.Close() + same, err := isSameContent(localPath, size, rc) + if err != nil { + log.Warnf("failed to compare content of obj %s: %v", localPath, err) + return + } + if same { + return + } + rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to reread range: %v", localPath, err) + return + } + defer rc.Close() + file, err := utils.CreateNestedFile(localPath) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err) + return + } + defer file.Close() + if _, err := utils.CopyWithBuffer(file, rc); err != nil { + log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err) + } + } +} + +func isSameContent(localPath string, size int64, rc io.Reader) (bool, error) { + info, err := os.Stat(localPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + if info.Size() != size { + return false, nil + } + localFile, err := os.Open(localPath) + if err != nil { + return false, err + } + defer localFile.Close() + h1 := sha256.New() + h2 := sha256.New() + if _, err := io.Copy(h1, localFile); err != nil { + return false, err + } + if _, err := io.Copy(h2, rc); err != nil { + return false, err + } + return bytes.Equal(h1.Sum(nil), h2.Sum(nil)), nil +} + +func deleteExtraFiles(driver *Strm, localPath string, objs []model.Obj) { + if driver.SaveLocalMode != SaveLocalSyncMode { + return + } + localFiles, localDirs, err := getLocalDirsAndFiles(localPath) + if err != nil { + log.Errorf("Failed to read local files from %s: %v", localPath, err) + return + } + + fileSet := make(map[string]struct{}) + dirSet := make(map[string]struct{}) + for _, obj := range objs { + objPath := stdpath.Join(localPath, obj.GetName()) + if obj.IsDir() { + dirSet[objPath] = struct{}{} + } else { + fileSet[objPath] = struct{}{} + } + } + + for _, localFile := range localFiles { + if _, exists := fileSet[localFile]; !exists { + err := os.Remove(localFile) + if err != nil { + log.Errorf("Failed to delete file: %s, error: %v\n", localFile, err) + } else { + log.Infof("Deleted file %s", localFile) + } + } + } + + for _, localDir := range localDirs { + if _, exists := dirSet[localDir]; !exists { + err := os.RemoveAll(localDir) + if err != nil { + log.Errorf("Failed to delete directory: %s, error: %v\n", localDir, err) + } else { + log.Infof("Deleted directory %s", localDir) + } + } + } +} + +func getLocalDirsAndFiles(localPath string) ([]string, []string, error) { + var files, dirs []string + entries, err := os.ReadDir(localPath) + if err != nil { + return nil, nil, err + } + for _, entry := range entries { + fullPath := stdpath.Join(localPath, entry.Name()) + if entry.IsDir() { + dirs = append(dirs, fullPath) + } else { + files = append(files, fullPath) + } + } + return files, dirs, nil +} + +func init() { + op.RegisterObjsUpdateHook(UpdateLocalStrm) +} diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go index 0f3e0a994..843ff7c66 100644 --- a/drivers/strm/meta.go +++ b/drivers/strm/meta.go @@ -5,24 +5,35 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" ) +const ( + SaveLocalInsertMode = "insert" + SaveLocalUpdateMode = "update" + SaveLocalSyncMode = "sync" +) + type Addition struct { Paths string `json:"paths" required:"true" type:"text"` SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of the strm file"` - FilterFileTypes string `json:"filterFileTypes" type:"text" default:"strm" required:"false" help:"Supports suffix name of strm file"` - DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass" required:"false" help:"Files need to download with strm (usally subtitles)"` + PathPrefix string `json:"PathPrefix" type:"text" required:"false" default:"/d" help:"Path prefix"` + DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Files need to download with strm (usally subtitles)"` + FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Supports suffix name of strm file"` EncodePath bool `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"` - LocalModel bool `json:"localModel" default:"false" help:"enable local mode"` + WithoutUrl bool `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"` + WithSign bool `json:"withSign" default:"false"` + SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"save strm file locally"` + SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"save strm file local path"` + SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"save strm file locally mode" options:"insert,update,sync" default:"insert"` + Version int } var config = driver.Config{ - Name: "Strm", - LocalSort: true, - NoCache: true, - NoUpload: true, - DefaultRoot: "/", - OnlyLinkMFile: true, - OnlyProxy: true, - NoLinkURL: true, + Name: "Strm", + LocalSort: true, + OnlyProxy: true, + NoCache: true, + NoUpload: true, + DefaultRoot: "/", + NoLinkURL: true, } func init() { diff --git a/drivers/strm/types.go b/drivers/strm/types.go deleted file mode 100644 index dfa427d42..000000000 --- a/drivers/strm/types.go +++ /dev/null @@ -1,36 +0,0 @@ -package strm - -func supportSuffix() map[string]struct{} { - return map[string]struct{}{ - // video - "mp4": {}, - "mkv": {}, - "flv": {}, - "avi": {}, - "wmv": {}, - "ts": {}, - "rmvb": {}, - "webm": {}, - // audio - "mp3": {}, - "flac": {}, - "aac": {}, - "wav": {}, - "ogg": {}, - "m4a": {}, - "wma": {}, - "alac": {}, - } -} - -func downloadSuffix() map[string]struct{} { - return map[string]struct{}{ - // strm - "strm": {}, - // subtitles - "ass": {}, - "srt": {}, - "vtt": {}, - "sub": {}, - } -} diff --git a/drivers/strm/util.go b/drivers/strm/util.go index cfa441db8..b9a40da5e 100644 --- a/drivers/strm/util.go +++ b/drivers/strm/util.go @@ -3,7 +3,6 @@ package strm import ( "context" "fmt" - stdpath "path" "strings" @@ -19,6 +18,7 @@ func (d *Strm) listRoot() []model.Obj { var objs []model.Obj for k := range d.pathMap { obj := model.Object{ + Path: "/" + k, Name: k, IsFolder: true, Modified: d.Modified, @@ -58,20 +58,24 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([] if err != nil { return nil, err } + return d.convert2strmObjs(ctx, reqPath, objs), nil +} +func (d *Strm) convert2strmObjs(ctx context.Context, reqPath string, objs []model.Obj) []model.Obj { var validObjs []model.Obj for _, obj := range objs { id, name, path := "", obj.GetName(), "" size := int64(0) if !obj.IsDir() { path = stdpath.Join(reqPath, obj.GetName()) - ext := strings.ToLower(utils.Ext(name)) - if _, ok := d.supportSuffix[ext]; ok { + sourceExt := utils.SourceExt(name) + ext := strings.ToLower(sourceExt) + if _, ok := d.downloadSuffix[ext]; ok { + size = obj.GetSize() + } else if _, ok := d.supportSuffix[ext]; ok { id = "strm" - name = strings.TrimSuffix(name, ext) + "strm" + name = strings.TrimSuffix(name, sourceExt) + "strm" size = int64(len(d.getLink(ctx, path))) - } else if _, ok := d.downloadSuffix[ext]; ok { - size = obj.GetSize() } else { continue } @@ -84,13 +88,11 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([] Modified: obj.ModTime(), IsFolder: obj.IsDir(), } - thumb, ok := model.GetThumb(obj) if !ok { validObjs = append(validObjs, &objRes) continue } - validObjs = append(validObjs, &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ @@ -98,7 +100,7 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([] }, }) } - return validObjs, nil + return validObjs } func (d *Strm) getLink(ctx context.Context, path string) string { @@ -106,11 +108,18 @@ func (d *Strm) getLink(ctx context.Context, path string) string { if d.EncodePath { finalPath = utils.EncodePath(path, true) } - if d.EnableSign { + if d.WithSign { signPath := sign.Sign(path) finalPath = fmt.Sprintf("%s?sign=%s", finalPath, signPath) } - if d.LocalModel { + pathPrefix := d.PathPrefix + if len(pathPrefix) > 0 { + finalPath = stdpath.Join(pathPrefix, finalPath) + } + if !strings.HasPrefix(finalPath, "/") { + finalPath = "/" + finalPath + } + if d.WithoutUrl { return finalPath } apiUrl := d.SiteUrl @@ -119,8 +128,7 @@ func (d *Strm) getLink(ctx context.Context, path string) string { } else { apiUrl = common.GetApiUrl(ctx) } - - return fmt.Sprintf("%s/d%s", + return fmt.Sprintf("%s%s", apiUrl, finalPath) } diff --git a/drivers/teldrive/driver.go b/drivers/teldrive/driver.go index 541d2e3be..d420eb4d0 100644 --- a/drivers/teldrive/driver.go +++ b/drivers/teldrive/driver.go @@ -6,6 +6,8 @@ import ( "math" "net/http" "net/url" + "path" + "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -16,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" + "golang.org/x/sync/errgroup" ) type Teldrive struct { @@ -52,19 +55,60 @@ func (d *Teldrive) Drop(ctx context.Context) error { } func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - var listResp ListResp + var firstResp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "path": dir.GetPath(), - "limit": "1000", // overide default 500, TODO pagination + "limit": "500", + "page": "1", }) - }, &listResp) + }, &firstResp) + if err != nil { return nil, err } - return utils.SliceConvert(listResp.Items, func(src Object) (model.Obj, error) { + pagesData := make([][]Object, firstResp.Meta.TotalPages) + pagesData[0] = firstResp.Items + + if firstResp.Meta.TotalPages > 1 { + g, _ := errgroup.WithContext(ctx) + g.SetLimit(8) + + for i := 2; i <= firstResp.Meta.TotalPages; i++ { + page := i + g.Go(func() error { + var resp ListResp + err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "path": dir.GetPath(), + "limit": "500", + "page": strconv.Itoa(page), + }) + }, &resp) + + if err != nil { + return err + } + + pagesData[page-1] = resp.Items + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + } + + var allItems []Object + for _, items := range pagesData { + allItems = append(allItems, items...) + } + + return utils.SliceConvert(allItems, func(src Object) (model.Obj, error) { return &model.Object{ + Path: path.Join(dir.GetPath(), src.Name), ID: src.ID, Name: src.Name, Size: func() int64 { @@ -182,7 +226,7 @@ func (d *Teldrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStr } if totalParts <= 1 { - return d.doSingleUpload(ctx, dstDir, file, up, totalParts, chunkSize, fileId) + return d.doSingleUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) } return d.doMultiUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) diff --git a/drivers/teldrive/meta.go b/drivers/teldrive/meta.go index 23bae5f94..cc7a5dbf7 100644 --- a/drivers/teldrive/meta.go +++ b/drivers/teldrive/meta.go @@ -11,6 +11,7 @@ type Addition struct { Cookie string `json:"cookie" type:"string" required:"true" help:"access_token=xxx"` UseShareLink bool `json:"use_share_link" type:"bool" default:"false" help:"Create share link when getting link to support 302. If disabled, you need to enable web proxy."` ChunkSize int64 `json:"chunk_size" type:"number" default:"10" help:"Chunk size in MiB"` + RandomChunkName bool `json:"random_chunk_name" type:"bool" default:"true" help:"Random chunk name"` UploadConcurrency int64 `json:"upload_concurrency" type:"number" default:"4" help:"Concurrency upload requests"` } diff --git a/drivers/teldrive/upload.go b/drivers/teldrive/upload.go index 168d9beff..b94f5fc93 100644 --- a/drivers/teldrive/upload.go +++ b/drivers/teldrive/upload.go @@ -1,6 +1,8 @@ package teldrive import ( + "crypto/md5" + "encoding/hex" "fmt" "io" "net/http" @@ -16,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + "github.com/google/uuid" "github.com/pkg/errors" "golang.org/x/net/context" "golang.org/x/sync/errgroup" @@ -38,6 +41,11 @@ func (d *Teldrive) touch(name, path string) error { return nil } +func getMD5Hash(text string) string { + hash := md5.Sum([]byte(text)) + return hex.EncodeToString(hash[:]) +} + func (d *Teldrive) createFileOnUploadSuccess(name, id, path string, uploadedFileParts []FilePart, totalSize int64) error { remoteFileParts, err := d.getFilePart(id) if err != nil { @@ -101,12 +109,10 @@ func (d *Teldrive) getFilePart(fileId string) ([]FilePart, error) { return uploadedParts, nil } -func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback, resp interface{}) error { +func (d *Teldrive) singleUploadRequest(ctx context.Context, fileId string, callback base.ReqCallback, resp any) error { url := d.Address + "/api/uploads/" + fileId client := resty.New().SetTimeout(0) - ctx := context.Background() - req := client.R(). SetContext(ctx) req.SetHeader("Cookie", d.Cookie) @@ -135,16 +141,18 @@ func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback, } func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up model.UpdateProgress, - totalParts int, chunkSize int64, fileId string) error { + maxRetried, totalParts int, chunkSize int64, fileId string) error { totalSize := file.GetSize() var fileParts []FilePart var uploaded int64 = 0 - ss, err := stream.NewStreamSectionReader(file, int(totalSize), &up) + var partName string + chunkSize = min(totalSize, chunkSize) + ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up) if err != nil { return err } - + chunkCnt := 0 for uploaded < totalSize { if utils.IsCanceled(ctx) { return ctx.Err() @@ -154,6 +162,7 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo if err != nil { return err } + chunkCnt += 1 filePart := &FilePart{} if err := retry.Do(func() error { @@ -161,13 +170,19 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo return err } - if err := d.singleUploadRequest(fileId, func(req *resty.Request) { + if d.RandomChunkName { + partName = getMD5Hash(uuid.New().String()) + } else { + partName = file.GetName() + if totalParts > 1 { + partName = fmt.Sprintf("%s.part.%03d", file.GetName(), chunkCnt) + } + } + + if err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ - "partName": func() string { - digits := len(fmt.Sprintf("%d", totalParts)) - return file.GetName() + fmt.Sprintf(".%0*d", digits, 1) - }(), - "partNo": strconv.Itoa(1), + "partName": partName, + "partNo": strconv.Itoa(chunkCnt), "fileName": file.GetName(), } req.SetQueryParams(uploadParams) @@ -179,7 +194,8 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo return nil }, - retry.Attempts(3), + retry.Context(ctx), + retry.Attempts(uint(maxRetried)), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)); err != nil { return err @@ -188,8 +204,11 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo if filePart.Name != "" { fileParts = append(fileParts, *filePart) uploaded += curChunkSize - up(float64(uploaded) / float64(totalSize)) + up(float64(uploaded) / float64(totalSize) * 100) ss.FreeSectionReader(rd) + } else { + // For common situation this code won't reach + return fmt.Errorf("[Teldrive] upload chunk %d failed: filePart Somehow missing", chunkCnt) } } @@ -317,6 +336,7 @@ func (d *Teldrive) doMultiUpload(ctx context.Context, dstDir model.Obj, file mod func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task chunkTask, totalParts, maxRetried int) (*FilePart, error) { filePart := &FilePart{} retryCount := 0 + var partName string defer task.ss.FreeSectionReader(task.reader) for { @@ -330,12 +350,22 @@ func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task ch return &existingPart, nil } - err := d.singleUploadRequest(fileId, func(req *resty.Request) { + if _, err := task.reader.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + if d.RandomChunkName { + partName = getMD5Hash(uuid.New().String()) + } else { + partName = task.fileName + if totalParts > 1 { + partName = fmt.Sprintf("%s.part.%03d", task.fileName, task.chunkIdx) + } + } + + err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ - "partName": func() string { - digits := len(fmt.Sprintf("%d", totalParts)) - return task.fileName + fmt.Sprintf(".%0*d", digits, task.chunkIdx) - }(), + "partName": partName, "partNo": strconv.Itoa(task.chunkIdx), "fileName": task.fileName, } diff --git a/drivers/template/meta.go b/drivers/template/meta.go index a546e676b..f525b5a95 100644 --- a/drivers/template/meta.go +++ b/drivers/template/meta.go @@ -16,7 +16,6 @@ type Addition struct { var config = driver.Config{ Name: "Template", LocalSort: false, - OnlyLinkMFile: false, OnlyProxy: false, NoCache: false, NoUpload: false, diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index 3c9d265c2..5323b1887 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -7,7 +7,6 @@ import ( "encoding/hex" "fmt" "io" - "math" stdpath "path" "strconv" @@ -63,7 +62,9 @@ func (d *Terabox) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil + obj := fileToObj(src) + obj.Path = stdpath.Join(dir.GetPath(), obj.Name) + return obj, nil }) } @@ -193,7 +194,7 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt streamSize := stream.GetSize() chunkSize := calculateChunkSize(streamSize) chunkByteData := make([]byte, chunkSize) - count := int(math.Ceil(float64(streamSize) / float64(chunkSize))) + count := int((streamSize + chunkSize - 1) / chunkSize) left := streamSize uploadBlockList := make([]string, 0, count) h := md5.New() diff --git a/drivers/terabox/util.go b/drivers/terabox/util.go index 518fd4f9d..69944d346 100644 --- a/drivers/terabox/util.go +++ b/drivers/terabox/util.go @@ -88,7 +88,7 @@ func (d *Terabox) request(rurl string, method string, callback base.ReqCallback, return nil, err } errno := utils.Json.Get(res.Body(), "errno").ToInt() - if errno == 4000023 || errno == 4500016 { + if errno == 4000023 || errno == 450016 { // reget jsToken err = d.resetJsToken() if err != nil { diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 7f537ac64..492b9814f 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "net/url" "strconv" "strings" @@ -68,6 +69,7 @@ func (x *Thunder) Init(ctx context.Context) (err error) { PackageName: "com.xunlei.downloadprovider", UserAgent: "ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)", + Space: x.Space, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) @@ -167,6 +169,7 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) { UserAgent: x.UserAgent, DownloadUserAgent: x.DownloadUserAgent, UseVideoUrl: x.UseVideoUrl, + Space: x.Space, refreshCTokenCk: func(token string) { x.CaptchaToken = token @@ -281,7 +284,7 @@ func (xc *XunLeiCommon) Link(ctx context.Context, file model.Obj, args model.Lin _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", file.GetID()) - //r.SetQueryParam("space", "") + r.SetQueryParam("space", xc.Space) }, &lFile) if err != nil { return nil, err @@ -322,6 +325,7 @@ func (xc *XunLeiCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirNam "kind": FOLDER, "name": dirName, "parent_id": parentDir.GetID(), + "space": xc.Space, }) }, nil) return err @@ -331,8 +335,9 @@ func (xc *XunLeiCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) erro _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ - "to": base.Json{"parent_id": dstDir.GetID()}, - "ids": []string{srcObj.GetID()}, + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + "space": xc.Space, }) }, nil) return err @@ -342,7 +347,10 @@ func (xc *XunLeiCommon) Rename(ctx context.Context, srcObj model.Obj, newName st _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", srcObj.GetID()) - r.SetBody(&base.Json{"name": newName}) + r.SetBody(&base.Json{ + "name": newName, + "space": xc.Space, + }) }, nil) return err } @@ -351,8 +359,9 @@ func (xc *XunLeiCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) erro _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ - "to": base.Json{"parent_id": dstDir.GetID()}, - "ids": []string{srcObj.GetID()}, + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + "space": xc.Space, }) }, nil) return err @@ -362,6 +371,7 @@ func (xc *XunLeiCommon) Remove(ctx context.Context, obj model.Obj) error { _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", obj.GetID()) + r.SetQueryParam("space", xc.Space) r.SetBody("{}") }, nil) return err @@ -387,6 +397,7 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi "size": file.GetSize(), "hash": gcid, "upload_type": UPLOAD_TYPE_RESUMABLE, + "space": xc.Space, }) }, &resp) if err != nil { @@ -422,6 +433,32 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi return nil } +func (xc *XunLeiCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string @@ -430,7 +467,7 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model. _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ - "space": "", + "space": xc.Space, "__type": "drive", "refresh": "true", "__sync": "true", @@ -440,6 +477,17 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model. "limit": "100", "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, }) + // 获取硬盘挂载目录等 + if xc.Space != "" { + r.SetQueryParamsFromValues(url.Values{ + "with": []string{ + "withCategoryDiskMountPath", + "withCategoryDriveCachePath", + "withCategoryHistoryDownloadPath", + "withReadOnlyFS", + }, + }) + } }, &fileList) if err != nil { return nil, err @@ -576,6 +624,7 @@ func (xc *XunLeiCommon) OfflineDownload(ctx context.Context, fileUrl string, par "name": fileName, "parent_id": parentDir.GetID(), "upload_type": UPLOAD_TYPE_URL, + "space": xc.Space, "url": base.Json{ "url": fileUrl, }, @@ -602,6 +651,7 @@ func (xc *XunLeiCommon) OfflineList(ctx context.Context, nextPageToken string) ( "type": "offline", "limit": "10000", "page_token": nextPageToken, + "space": xc.Space, }) }, &resp) @@ -618,6 +668,7 @@ func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string SetQueryParams(map[string]string{ "task_ids": strings.Join(taskIDs, ","), "delete_files": strconv.FormatBool(deleteFiles), + "space": xc.Space, }) }, nil) if err != nil { diff --git a/drivers/thunder/meta.go b/drivers/thunder/meta.go index e5c0f977d..be9c712fd 100644 --- a/drivers/thunder/meta.go +++ b/drivers/thunder/meta.go @@ -46,6 +46,8 @@ type ExpertAddition struct { //优先使用视频链接代替下载链接 UseVideoUrl bool `json:"use_video_url"` + + Space string `json:"space" default:"" help:"device id for remote device"` } // 登录特征,用于判断是否重新登录 @@ -80,6 +82,8 @@ type Addition struct { CreditKey string `json:"credit_key" help:"credit key,used for login"` // 登录设备ID DeviceID string `json:"device_id" default:""` + + Space string `json:"space" default:"" help:"device id for remote device"` } // 登录特征,用于判断是否重新登录 @@ -90,7 +94,6 @@ func (i *Addition) GetIdentity() string { var config = driver.Config{ Name: "Thunder", LocalSort: true, - OnlyProxy: true, } var configExpert = driver.Config{ diff --git a/drivers/thunder/types.go b/drivers/thunder/types.go index 7b3ad5692..fcfa1fb1c 100644 --- a/drivers/thunder/types.go +++ b/drivers/thunder/types.go @@ -347,3 +347,21 @@ type ReviewData struct { Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } + +type AboutResponse struct { + // Kind string `json:"kind"` + Quota struct { + // Kind string `json:"kind"` + Limit string `json:"limit"` + Usage string `json:"usage"` + // UsageInTrash string `json:"usage_in_trash"` + // PlayTimesLimit string `json:"play_times_limit"` + // PlayTimesUsage string `json:"play_times_usage"` + // IsUnlimited bool `json:"is_unlimited"` + // UpgradeType string `json:"upgrade_type"` + } `json:"quota"` + // ExpiresAt string `json:"expires_at"` + // Quotas struct { + // } `json:"quotas"` + // IsSearchFlushed bool `json:"is_search_flushed"` +} diff --git a/drivers/thunder/util.go b/drivers/thunder/util.go index f4733613b..1fa01a90e 100644 --- a/drivers/thunder/util.go +++ b/drivers/thunder/util.go @@ -68,6 +68,7 @@ type Common struct { UserAgent string DownloadUserAgent string UseVideoUrl bool + Space string // 验证码token刷新成功回调 refreshCTokenCk func(token string) diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index bf1843a41..e60522e52 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" "time" @@ -542,6 +543,32 @@ func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream return nil } +func (xc *XunLeiBrowserCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, dir model.Obj, path string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string @@ -840,7 +867,7 @@ func (xc *XunLeiBrowserCommon) OfflineList(ctx context.Context, nextPageToken st func (xc *XunLeiBrowserCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string) error { queryParams := map[string]string{ "task_ids": strings.Join(taskIDs, ","), - "_t": fmt.Sprintf("%d", time.Now().UnixMilli()), + "_t": strconv.FormatInt(time.Now().UnixMilli(), 10), } if xc.UseFluentPlay { queryParams["space"] = ThunderBrowserDriveFluentPlayFolderType diff --git a/drivers/thunder_browser/types.go b/drivers/thunder_browser/types.go index 6b2a41023..a5fda2a5d 100644 --- a/drivers/thunder_browser/types.go +++ b/drivers/thunder_browser/types.go @@ -376,3 +376,10 @@ type ReviewData struct { Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + } `json:"quota"` +} diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index 86ff22bdc..acbb82513 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -423,6 +423,32 @@ func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.F return nil } +func (xc *XunLeiXCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiXCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string diff --git a/drivers/thunderx/types.go b/drivers/thunderx/types.go index e5fbaa241..728fdea31 100644 --- a/drivers/thunderx/types.go +++ b/drivers/thunderx/types.go @@ -303,3 +303,10 @@ type Media struct { IsVisible bool `json:"is_visible"` Category string `json:"category"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + } `json:"quota"` +} diff --git a/drivers/url_tree/driver.go b/drivers/url_tree/driver.go index 36579d3cd..c1af0b3fb 100644 --- a/drivers/url_tree/driver.go +++ b/drivers/url_tree/driver.go @@ -44,6 +44,10 @@ func (d *Urls) Drop(ctx context.Context) error { return nil } +func (Addition) GetRootPath() string { + return "/" +} + func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) { d.mutex.RLock() defer d.mutex.RUnlock() diff --git a/drivers/url_tree/meta.go b/drivers/url_tree/meta.go index 1ec57ee0a..9d2f182ba 100644 --- a/drivers/url_tree/meta.go +++ b/drivers/url_tree/meta.go @@ -6,10 +6,6 @@ import ( ) type Addition struct { - // Usually one of two - // driver.RootPath - // driver.RootID - // define other UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://raw.githubusercontent.com/OpenListTeam/OpenList/main/README.md\nhttps://raw.githubusercontent.com/OpenListTeam/OpenList/main/README_cn.md\nfolder:\n CONTRIBUTING.md:1635:https://raw.githubusercontent.com/OpenListTeam/OpenList/main/CONTRIBUTING.md\n CODE_OF_CONDUCT.md:2093:https://raw.githubusercontent.com/OpenListTeam/OpenList/main/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n [FileName:][FileSize:][Modified:]Url"` HeadSize bool `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."` Writable bool `json:"writable" type:"bool" default:"false"` @@ -20,6 +16,7 @@ var config = driver.Config{ LocalSort: true, NoCache: true, CheckStatus: true, + OnlyIndices: true, } func init() { diff --git a/drivers/uss/driver.go b/drivers/uss/driver.go index 8f33a5b0b..3606278d7 100644 --- a/drivers/uss/driver.go +++ b/drivers/uss/driver.go @@ -62,6 +62,7 @@ func (d *USS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m for obj := range objsChan { t := obj.Time f := model.Object{ + Path: path.Join(dir.GetPath(), obj.Name), Name: obj.Name, Size: obj.Size, Modified: t, diff --git a/drivers/virtual/driver.go b/drivers/virtual/driver.go index 1d14427c5..6e6bd0875 100644 --- a/drivers/virtual/driver.go +++ b/drivers/virtual/driver.go @@ -2,11 +2,11 @@ package virtual import ( "context" - "io" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" ) @@ -42,16 +42,14 @@ func (d *Virtual) List(ctx context.Context, dir model.Obj, args model.ListArgs) return res, nil } -type DummyMFile struct { - io.Reader -} +type DummyMFile struct{} func (f DummyMFile) Read(p []byte) (n int, err error) { - return f.Reader.Read(p) + return random.Rand.Read(p) } func (f DummyMFile) ReadAt(p []byte, off int64) (n int, err error) { - return f.Reader.Read(p) + return random.Rand.Read(p) } func (DummyMFile) Seek(offset int64, whence int) (int64, error) { @@ -60,7 +58,7 @@ func (DummyMFile) Seek(offset int64, whence int) (int64, error) { func (d *Virtual) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { return &model.Link{ - MFile: DummyMFile{Reader: random.Rand}, + RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), DummyMFile{}), }, nil } diff --git a/drivers/virtual/meta.go b/drivers/virtual/meta.go index f567830a8..c270ec12b 100644 --- a/drivers/virtual/meta.go +++ b/drivers/virtual/meta.go @@ -14,11 +14,11 @@ type Addition struct { } var config = driver.Config{ - Name: "Virtual", - OnlyLinkMFile: true, - LocalSort: true, - NeedMs: true, - NoLinkURL: true, + Name: "Virtual", + LocalSort: true, + OnlyProxy: true, + NeedMs: true, + NoLinkURL: true, } func init() { diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index a6a18d85d..7a1b5db2c 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -2,11 +2,13 @@ package webdav import ( "context" + "fmt" "net/http" "os" "path" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/cron" @@ -54,6 +56,7 @@ func (d *WebDav) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } return utils.SliceConvert(files, func(src os.FileInfo) (model.Obj, error) { return &model.Object{ + Path: path.Join(dir.GetPath(), src.Name()), Name: src.Name(), Size: src.Size(), Modified: src.ModTime(), @@ -67,6 +70,22 @@ func (d *WebDav) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err != nil { return nil, err } + if args.Redirect { + // get the url after redirect + req := base.NoRedirectClient.R() + req.Header = header + req.SetDoNotParseResponse(true) + res, err := req.Get(url) + if err != nil { + return nil, err + } + _ = res.RawResponse.Body.Close() + if (res.StatusCode() == 302 || res.StatusCode() == 307 || res.StatusCode() == 308) && res.Header().Get("location") != "" { + url = res.Header().Get("location") + } else { + return nil, fmt.Errorf("redirect failed, status: %d", res.StatusCode()) + } + } return &model.Link{ URL: url, Header: header, diff --git a/drivers/webdav/meta.go b/drivers/webdav/meta.go index 5f8197cf6..6e7c27772 100644 --- a/drivers/webdav/meta.go +++ b/drivers/webdav/meta.go @@ -17,8 +17,8 @@ type Addition struct { var config = driver.Config{ Name: "WebDav", LocalSort: true, - OnlyProxy: true, DefaultRoot: "/", + PreferProxy: true, } func init() { diff --git a/drivers/weiyun/driver.go b/drivers/weiyun/driver.go index ef203cfa4..5178c9529 100644 --- a/drivers/weiyun/driver.go +++ b/drivers/weiyun/driver.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" weiyunsdkgo "github.com/foxxorcat/weiyun-sdk-go" + "github.com/go-resty/resty/v2" ) type WeiYun struct { @@ -67,7 +68,7 @@ func (d *WeiYun) Init(ctx context.Context) error { }) // qqCookie保活 - if d.client.LoginType() == 1 { + if d.client.LoginType() == weiyunsdkgo.AccountTypeQQ || d.client.LoginType() == weiyunsdkgo.AccountTypeQQOpenID { d.cron = cron.NewCron(time.Minute * 5) d.cron.Do(func() { _ = d.client.KeepAlive() @@ -391,6 +392,22 @@ func (d *WeiYun) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr }, nil } +func (d *WeiYun) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + info, err := d.client.DiskUserInfoGet(func(request *resty.Request) { + request.SetContext(ctx) + }) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: info.TotalSpace, + UsedSpace: info.UsedSpace, + }, + }, nil +} + // func (d *WeiYun) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport // } @@ -405,3 +422,4 @@ var _ driver.Remove = (*WeiYun)(nil) var _ driver.PutResult = (*WeiYun)(nil) var _ driver.RenameResult = (*WeiYun)(nil) +var _ driver.WithDetails = (*WeiYun)(nil) diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go index 6d9aabca8..3ff39c0cb 100644 --- a/drivers/wopan/driver.go +++ b/drivers/wopan/driver.go @@ -9,8 +9,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/go-resty/resty/v2" "github.com/OpenListTeam/wopan-sdk-go" + "github.com/go-resty/resty/v2" ) type Wopan struct { @@ -166,6 +166,23 @@ func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre return err } +func (d *Wopan) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.client.QueryCloudUsageInfo() + if err != nil { + return nil, err + } + total, err := strconv.ParseInt(quota.UsageInfo.ByteTotalSize, 10, 64) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: quota.UsageInfo.ByteUsedSize, + }, + }, nil +} + //func (d *Wopan) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/wps/driver.go b/drivers/wps/driver.go new file mode 100644 index 000000000..8a3ccb6ce --- /dev/null +++ b/drivers/wps/driver.go @@ -0,0 +1,91 @@ +package wps + +import ( + "context" + "fmt" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type Wps struct { + model.Storage + Addition + companyID string +} + +func (d *Wps) Config() driver.Config { + return config +} + +func (d *Wps) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Wps) Init(ctx context.Context) error { + if d.Cookie == "" { + return fmt.Errorf("cookie is empty") + } + return d.ensureCompanyID(ctx) +} + +func (d *Wps) Drop(ctx context.Context) error { + return nil +} + +func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]model.Obj, error) { + basePath := "/" + if dir != nil { + if p := dir.GetPath(); p != "" { + basePath = p + } + } + return d.list(ctx, basePath) +} + +func (d *Wps) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { + if file == nil { + return nil, errs.NotSupport + } + return d.link(ctx, file.GetPath()) +} + +func (d *Wps) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.makeDir(ctx, parentDir, dirName) +} + +func (d *Wps) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.move(ctx, srcObj, dstDir) +} + +func (d *Wps) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return d.rename(ctx, srcObj, newName) +} + +func (d *Wps) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.copy(ctx, srcObj, dstDir) +} + +func (d *Wps) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(ctx, obj) +} + +func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + return d.put(ctx, dstDir, file, up) +} + +func (d *Wps) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.spaces(ctx) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: quota.Total, + UsedSpace: quota.Used, + }, + }, nil +} + +var _ driver.Driver = (*Wps)(nil) diff --git a/drivers/wps/meta.go b/drivers/wps/meta.go new file mode 100644 index 000000000..7a3362f3a --- /dev/null +++ b/drivers/wps/meta.go @@ -0,0 +1,26 @@ +package wps + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + Cookie string `json:"cookie" required:"true" type:"text"` + Mode string `json:"mode" type:"select" options:"Personal,Business" default:"Business"` +} + +var config = driver.Config{ + Name: "WPS", + LocalSort: true, + DefaultRoot: "/", + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Wps{} + }) +} diff --git a/drivers/wps/types.go b/drivers/wps/types.go new file mode 100644 index 000000000..a04df3d11 --- /dev/null +++ b/drivers/wps/types.go @@ -0,0 +1,107 @@ +package wps + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type workspaceResp struct { + Companies []struct { + ID int64 `json:"id"` + } `json:"companies"` +} + +type Group struct { + CompanyID int64 `json:"company_id"` + GroupID int64 `json:"group_id"` + Name string `json:"name"` + Type string `json:"type"` +} + +type groupsResp struct { + Groups []Group `json:"groups"` +} + +type filePerms struct { + Download int `json:"download"` +} + +type FileInfo struct { + GroupID int64 `json:"groupid"` + ParentID int64 `json:"parentid"` + Name string `json:"fname"` + Size int64 `json:"fsize"` + Type string `json:"ftype"` + Ctime int64 `json:"ctime"` + Mtime int64 `json:"mtime"` + ID int64 `json:"id"` + Deleted bool `json:"deleted"` + FilePerms filePerms `json:"file_perms_acl"` +} + +type filesResp struct { + Files []FileInfo `json:"files"` + NextOffset int `json:"next_offset"` +} + +type downloadResp struct { + URL string `json:"url"` + Result string `json:"result"` +} + +type spacesResp struct { + Id int64 `json:"id"` + Name string `json:"name"` + Result string `json:"result"` + Total int64 `json:"total"` + Used int64 `json:"used"` + UsedParts []struct { + Type string `json:"type"` + Used int64 `json:"used"` + } `json:"used_parts"` +} + +type Obj struct { + id string + name string + size int64 + ctime time.Time + mtime time.Time + isDir bool + hash utils.HashInfo + path string + canDownload bool +} + +func (o *Obj) GetSize() int64 { + return o.size +} + +func (o *Obj) GetName() string { + return o.name +} + +func (o *Obj) ModTime() time.Time { + return o.mtime +} + +func (o *Obj) CreateTime() time.Time { + return o.ctime +} + +func (o *Obj) IsDir() bool { + return o.isDir +} + +func (o *Obj) GetHash() utils.HashInfo { + return o.hash +} + +func (o *Obj) GetID() string { + return o.id +} + +func (o *Obj) GetPath() string { + return o.path +} diff --git a/drivers/wps/util.go b/drivers/wps/util.go new file mode 100644 index 000000000..6f8f342da --- /dev/null +++ b/drivers/wps/util.go @@ -0,0 +1,1072 @@ +package wps + +import ( + "bytes" + "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +const endpoint = "https://365.kdocs.cn" +const personalEndpoint = "https://drive.wps.cn" + +type resolvedNode struct { + kind string + group Group + file *FileInfo +} + +type resolveCacheEntry struct { + node *resolvedNode + expire time.Time +} + +type resolveCacheStore struct { + mu sync.RWMutex + m map[string]resolveCacheEntry +} + +var resolveCaches sync.Map + +type apiResult struct { + Result string `json:"result"` + Msg string `json:"msg"` +} + +type uploadCreateUpdateResp struct { + apiResult + Method string `json:"method"` + URL string `json:"url"` + Store string `json:"store"` + Request struct { + Headers map[string]string `json:"headers"` + FormData map[string]string `json:"formData"` + } `json:"request"` + Response struct { + ExpectCode []int `json:"expect_code"` + ArgsETag string `json:"args_etag"` + ArgsKey string `json:"args_key"` + } `json:"response"` +} + +type uploadPutResp struct { + NewFilename string `json:"newfilename"` + Sha1 string `json:"sha1"` + MD5 string `json:"md5"` +} + +type personalGroupsResp struct { + apiResult + Groups []struct { + ID int64 `json:"id"` + Name string `json:"name"` + } `json:"groups"` +} + +type countingWriter struct { + n *int64 +} + +func (w countingWriter) Write(p []byte) (int, error) { + *w.n += int64(len(p)) + return len(p), nil +} + +func (d *Wps) isPersonal() bool { + return strings.TrimSpace(d.Mode) == "Personal" +} + +func (d *Wps) driveHost() string { + if d.isPersonal() { + return personalEndpoint + } + return endpoint +} + +func (d *Wps) drivePrefix() string { + if d.isPersonal() { + return "" + } + return "/3rd/drive" +} + +func (d *Wps) driveURL(path string) string { + return d.driveHost() + d.drivePrefix() + path +} + +func (d *Wps) origin() string { + return d.driveHost() +} + +func (d *Wps) canDownload(f *FileInfo) bool { + if f == nil || f.Type == "folder" { + return false + } + if f.FilePerms.Download != 0 { + return true + } + return d.isPersonal() +} + +func (d *Wps) request(ctx context.Context) *resty.Request { + return base.RestyClient.R(). + SetHeader("Cookie", d.Cookie). + SetHeader("Accept", "application/json"). + SetContext(ctx) +} + +func (d *Wps) jsonRequest(ctx context.Context) *resty.Request { + return d.request(ctx). + SetHeader("Content-Type", "application/json"). + SetHeader("Origin", d.origin()) +} + +func statusOK(code int, expect []int) bool { + if len(expect) == 0 { + return code >= 200 && code < 300 + } + for _, v := range expect { + if v == code { + return true + } + } + return false +} + +func respArg(arg string, resp *http.Response, body []byte) string { + arg = strings.TrimSpace(arg) + if arg == "" { + return "" + } + l := strings.ToLower(arg) + if strings.HasPrefix(l, "header.") { + h := strings.TrimSpace(arg[len("header."):]) + if h == "" { + return "" + } + return strings.TrimSpace(resp.Header.Get(h)) + } + if strings.HasPrefix(l, "body.") { + k := strings.TrimSpace(arg[len("body."):]) + if k == "" { + return "" + } + var m map[string]interface{} + if err := json.Unmarshal(body, &m); err != nil { + return "" + } + if v, ok := m[k]; ok { + if s, ok := v.(string); ok { + return strings.TrimSpace(s) + } + } + } + return "" +} + +func extractXMLTag(v, tag string) string { + s := strings.TrimSpace(v) + if s == "" { + return "" + } + lt := strings.ToLower(tag) + open := "<" + lt + ">" + clos := "" + ls := strings.ToLower(s) + i := strings.Index(ls, open) + if i < 0 { + return "" + } + i += len(open) + j := strings.Index(ls[i:], clos) + if j < 0 { + return "" + } + r := strings.TrimSpace(s[i : i+j]) + r = strings.ReplaceAll(r, """, "") + return strings.Trim(r, `"'`) +} + +func checkAPI(resp *resty.Response, result apiResult) error { + if result.Result != "" && result.Result != "ok" { + if result.Msg == "" { + result.Msg = "unknown error" + } + return fmt.Errorf("%s: %s", result.Result, result.Msg) + } + if resp != nil && resp.IsError() { + if result.Msg != "" { + return fmt.Errorf("%s", result.Msg) + } + return fmt.Errorf("http error: %d", resp.StatusCode()) + } + return nil +} + +func (d *Wps) ensureCompanyID(ctx context.Context) error { + if d.isPersonal() { + return nil + } + if d.companyID != "" { + return nil + } + var resp workspaceResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(endpoint + "/3rd/plussvr/compose/v1/users/self/workspaces?fields=name&comp_status=active") + if err != nil { + return err + } + if r != nil && r.IsError() { + return fmt.Errorf("http error: %d", r.StatusCode()) + } + if len(resp.Companies) == 0 { + return fmt.Errorf("no company id") + } + d.companyID = strconv.FormatInt(resp.Companies[0].ID, 10) + return nil +} + +func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { + if d.isPersonal() { + var resp personalGroupsResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(d.driveURL("/api/v3/groups")) + if err != nil { + return nil, err + } + if err := checkAPI(r, resp.apiResult); err != nil { + return nil, err + } + res := make([]Group, 0, len(resp.Groups)) + for _, g := range resp.Groups { + res = append(res, Group{GroupID: g.ID, Name: g.Name}) + } + return res, nil + } + if err := d.ensureCompanyID(ctx); err != nil { + return nil, err + } + var resp groupsResp + url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%s/users/self/groups/private", endpoint, d.companyID) + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return resp.Groups, nil +} + +func (d *Wps) getFiles(ctx context.Context, groupID, parentID int64) ([]FileInfo, error) { + var resp filesResp + var files []FileInfo + next_offset := 0 + for range 50 { + url := fmt.Sprintf("%s/api/v5/groups/%d/files", d.driveHost()+d.drivePrefix(), groupID) + r, err := d.request(ctx). + SetQueryParam("parentid", strconv.FormatInt(parentID, 10)). + SetQueryParam("offset", fmt.Sprint(next_offset)). + SetResult(&resp). + SetError(&resp). + Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + files = append(files, resp.Files...) + if resp.NextOffset == -1 { + break + } + next_offset = resp.NextOffset + } + return files, nil +} + +func parseTime(v int64) time.Time { + if v <= 0 { + return time.Time{} + } + return time.Unix(v, 0) +} + +func joinPath(basePath, name string) string { + if basePath == "" || basePath == "/" { + return "/" + name + } + return strings.TrimRight(basePath, "/") + "/" + name +} + +func normalizePath(path string) string { + clean := strings.TrimSpace(path) + if clean == "" || clean == "/" { + return "/" + } + return "/" + strings.Trim(clean, "/") +} + +func (d *Wps) resolveCacheStore() *resolveCacheStore { + if d == nil { + return nil + } + if v, ok := resolveCaches.Load(d); ok { + if s, ok := v.(*resolveCacheStore); ok { + return s + } + } + s := &resolveCacheStore{m: make(map[string]resolveCacheEntry)} + if v, loaded := resolveCaches.LoadOrStore(d, s); loaded { + if s2, ok := v.(*resolveCacheStore); ok { + return s2 + } + } + return s +} + +func (d *Wps) getResolveCache(path string) (*resolvedNode, bool) { + s := d.resolveCacheStore() + if s == nil { + return nil, false + } + s.mu.RLock() + e, ok := s.m[path] + s.mu.RUnlock() + if !ok || e.node == nil { + return nil, false + } + if !e.expire.IsZero() && time.Now().After(e.expire) { + s.mu.Lock() + delete(s.m, path) + s.mu.Unlock() + return nil, false + } + return e.node, true +} + +func (d *Wps) setResolveCache(path string, node *resolvedNode) { + s := d.resolveCacheStore() + if s == nil || node == nil { + return + } + s.mu.Lock() + s.m[path] = resolveCacheEntry{node: node, expire: time.Now().Add(10 * time.Minute)} + s.mu.Unlock() +} + +func (d *Wps) clearResolveCache() { + s := d.resolveCacheStore() + if s == nil { + return + } + s.mu.Lock() + if len(s.m) != 0 { + s.m = make(map[string]resolveCacheEntry) + } + s.mu.Unlock() +} + +func (d *Wps) resolvePath(ctx context.Context, path string) (*resolvedNode, error) { + cacheKey := normalizePath(path) + if n, ok := d.getResolveCache(cacheKey); ok { + return n, nil + } + clean := strings.TrimSpace(path) + if clean == "" { + clean = "/" + } + clean = strings.Trim(clean, "/") + if clean == "" { + n := &resolvedNode{kind: "root"} + d.setResolveCache("/", n) + return n, nil + } + seg := strings.Split(clean, "/") + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + var grp *Group + for i := range groups { + if groups[i].Name == seg[0] { + grp = &groups[i] + break + } + } + if grp == nil { + return nil, fmt.Errorf("group not found") + } + cur := "/" + seg[0] + gn := &resolvedNode{kind: "group", group: *grp} + d.setResolveCache(cur, gn) + if len(seg) == 1 { + return gn, nil + } + parentID := int64(0) + var lastNode *resolvedNode + for i := 1; i < len(seg); i++ { + files, err := d.getFiles(ctx, grp.GroupID, parentID) + if err != nil { + return nil, err + } + var found *FileInfo + for j := range files { + if files[j].Name == seg[i] { + found = &files[j] + break + } + } + if found == nil { + return nil, fmt.Errorf("path not found") + } + if i < len(seg)-1 && found.Type != "folder" { + return nil, fmt.Errorf("path not found") + } + fi := *found + parentID = fi.ID + cur = cur + "/" + seg[i] + kind := "file" + if fi.Type == "folder" { + kind = "folder" + } + n := &resolvedNode{kind: kind, group: *grp, file: &fi} + d.setResolveCache(cur, n) + lastNode = n + } + if lastNode == nil { + return nil, fmt.Errorf("path not found") + } + return lastNode, nil +} + +func (d *Wps) fileToObj(basePath string, f FileInfo) *Obj { + name := f.Name + path := joinPath(basePath, name) + obj := &Obj{ + id: path, + name: name, + size: f.Size, + ctime: parseTime(f.Ctime), + mtime: parseTime(f.Mtime), + isDir: f.Type == "folder", + path: path, + } + if !obj.isDir { + obj.canDownload = d.canDownload(&f) + } + return obj +} + +func (d *Wps) doJSON(ctx context.Context, method, url string, body interface{}) error { + var result apiResult + req := d.jsonRequest(ctx).SetBody(body).SetResult(&result).SetError(&result) + var ( + resp *resty.Response + err error + ) + switch method { + case http.MethodPost: + resp, err = req.Post(url) + case http.MethodPut: + resp, err = req.Put(url) + default: + return errs.NotSupport + } + if err != nil { + return err + } + return checkAPI(resp, result) +} + +func (d *Wps) list(ctx context.Context, basePath string) ([]model.Obj, error) { + if strings.TrimSpace(basePath) == "" { + basePath = "/" + } + node, err := d.resolvePath(ctx, basePath) + if err != nil { + return nil, err + } + if node.kind == "root" { + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0, len(groups)) + for _, g := range groups { + path := joinPath(basePath, g.Name) + obj := &Obj{ + id: path, + name: g.Name, + ctime: parseTime(0), + mtime: parseTime(0), + isDir: true, + path: path, + } + res = append(res, obj) + d.setResolveCache(normalizePath(path), &resolvedNode{kind: "group", group: g}) + } + d.setResolveCache("/", &resolvedNode{kind: "root"}) + return res, nil + } + if node.kind != "group" && node.kind != "folder" { + return nil, nil + } + parentID := int64(0) + if node.file != nil && node.kind == "folder" { + parentID = node.file.ID + } + files, err := d.getFiles(ctx, node.group.GroupID, parentID) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0, len(files)) + for _, f := range files { + res = append(res, d.fileToObj(basePath, f)) + path := normalizePath(joinPath(basePath, f.Name)) + fi := f + kind := "file" + if fi.Type == "folder" { + kind = "folder" + } + d.setResolveCache(path, &resolvedNode{kind: kind, group: node.group, file: &fi}) + } + return res, nil +} + +func (d *Wps) link(ctx context.Context, path string) (*model.Link, error) { + node, err := d.resolvePath(ctx, path) + if err != nil { + return nil, err + } + if node.kind != "file" || node.file == nil { + return nil, errs.NotSupport + } + if !d.canDownload(node.file) { + return nil, fmt.Errorf("no download permission") + } + url := fmt.Sprintf("%s/api/v5/groups/%d/files/%d/download?support_checksums=sha1", d.driveHost()+d.drivePrefix(), node.group.GroupID, node.file.ID) + var resp downloadResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + if resp.URL == "" { + return nil, fmt.Errorf("empty download url") + } + return &model.Link{URL: resp.URL, Header: http.Header{}}, nil +} + +func (d *Wps) makeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if parentDir == nil { + return errs.NotSupport + } + node, err := d.resolvePath(ctx, parentDir.GetPath()) + if err != nil { + return err + } + if node.kind != "group" && node.kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.file != nil && node.kind == "folder" { + parentID = node.file.ID + } + body := map[string]interface{}{ + "groupid": node.group.GroupID, + "name": dirName, + "parentid": parentID, + } + if err := d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/folder"), body); err != nil { + return err + } + d.clearResolveCache() + return nil +} + +func (d *Wps) move(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) + if err != nil { + return err + } + nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) + if err != nil { + return err + } + if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { + return errs.NotSupport + } + if nodeDst.kind != "group" && nodeDst.kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.file != nil && nodeDst.kind == "folder" { + targetParentID = nodeDst.file.ID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.file.ID}, + "target_groupid": nodeDst.group.GroupID, + "target_parentid": targetParentID, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/move", nodeSrc.group.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + d.clearResolveCache() + return nil +} + +func (d *Wps) rename(ctx context.Context, srcObj model.Obj, newName string) error { + if srcObj == nil { + return errs.NotSupport + } + node, err := d.resolvePath(ctx, srcObj.GetPath()) + if err != nil { + return err + } + if node.kind != "file" && node.kind != "folder" { + return errs.NotSupport + } + url := fmt.Sprintf("/api/v3/groups/%d/files/%d", node.group.GroupID, node.file.ID) + body := map[string]string{"fname": newName} + if err := d.doJSON(ctx, http.MethodPut, d.driveURL(url), body); err != nil { + return err + } + d.clearResolveCache() + return nil +} + +func (d *Wps) copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) + if err != nil { + return err + } + nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) + if err != nil { + return err + } + if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { + return errs.NotSupport + } + if nodeDst.kind != "group" && nodeDst.kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.file != nil && nodeDst.kind == "folder" { + targetParentID = nodeDst.file.ID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.file.ID}, + "groupid": nodeSrc.group.GroupID, + "target_groupid": nodeDst.group.GroupID, + "target_parentid": targetParentID, + "duplicated_name_model": 1, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/copy", nodeSrc.group.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + d.clearResolveCache() + return nil +} + +func (d *Wps) remove(ctx context.Context, obj model.Obj) error { + if obj == nil { + return errs.NotSupport + } + node, err := d.resolvePath(ctx, obj.GetPath()) + if err != nil { + return err + } + if node.kind != "file" && node.kind != "folder" { + return errs.NotSupport + } + + body := map[string]interface{}{ + "fileids": []int64{node.file.ID}, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/delete", node.group.GroupID) + + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + // 无法连续创建文件夹删除。如果一定要删除，每0.5s 尝试一次创建下一个删除请求，应当避免递归删除文件夹 + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + d.clearResolveCache() + return nil +} + +func cacheAndHash(file model.FileStreamer, up driver.UpdateProgress) (model.File, int64, string, string, error) { + h1 := sha1.New() + h256 := sha256.New() + size := file.GetSize() + var counted int64 + ws := []io.Writer{h1, h256} + if size <= 0 { + ws = append(ws, countingWriter{n: &counted}) + } + p := up + f, err := file.CacheFullAndWriter(&p, io.MultiWriter(ws...)) + if err != nil { + return nil, 0, "", "", err + } + if size <= 0 { + size = counted + } + return f, size, hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h256.Sum(nil)), nil +} + +func (d *Wps) createUpload(ctx context.Context, groupID, parentID int64, name string, size int64, sha1Hex, sha256Hex string) (*uploadCreateUpdateResp, error) { + body := map[string]string{ + "group_id": strconv.FormatInt(groupID, 10), + "name": name, + "parent_id": strconv.FormatInt(parentID, 10), + "sha1": sha1Hex, + "sha256": sha256Hex, + "size": strconv.FormatInt(size, 10), + } + var resp uploadCreateUpdateResp + r, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&resp). + SetError(&resp). + Put(d.driveURL("/api/v5/files/upload/create_update")) + if err != nil { + return nil, err + } + if err := checkAPI(r, resp.apiResult); err != nil { + return nil, err + } + if resp.URL == "" { + return nil, fmt.Errorf("empty upload url") + } + return &resp, nil +} + +func normalizeETag(v string) string { + v = strings.TrimSpace(v) + if strings.HasPrefix(v, "W/") { + v = strings.TrimSpace(strings.TrimPrefix(v, "W/")) + } + return strings.Trim(v, `"`) +} + +func (d *Wps) commitUpload(ctx context.Context, etag, key string, groupID, parentID int64, name, sha1Hex string, size int64, store string) error { + store = strings.TrimSpace(store) + if store == "" { + store = "ks3" + } + storeKey := "" + if key != "" { + storeKey = key + } + body := map[string]interface{}{ + "etag": etag, + "groupid": groupID, + "key": key, + "name": name, + "parentid": parentID, + "sha1": sha1Hex, + "size": size, + "store": store, + "storekey": storeKey, + } + return d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/file"), body) +} + +func (d *Wps) put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if dstDir == nil || file == nil { + return errs.NotSupport + } + if up == nil { + up = func(float64) {} + } + node, err := d.resolvePath(ctx, dstDir.GetPath()) + if err != nil { + return err + } + if node.kind != "group" && node.kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.file != nil && node.kind == "folder" { + parentID = node.file.ID + } + f, size, sha1Hex, sha256Hex, err := cacheAndHash(file, func(float64) {}) + if err != nil { + return err + } + if c, ok := f.(io.Closer); ok { + defer c.Close() + } + + // 在隐藏文件名前加_上传，这是WPS的限制，无法上传隐藏文件，也无法将任何文件重命名为隐藏文件，所有隐藏文件会被自动加上_ 上传 + // 甚至可以上传前缀是..的文件，但是单个点就是不行 + realName := file.GetName() + uploadName := realName + if strings.HasPrefix(realName, ".") { + uploadName = "_" + realName + } + + info, err := d.createUpload(ctx, node.group.GroupID, parentID, uploadName, size, sha1Hex, sha256Hex) + if err != nil { + return err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + rf := driver.NewLimitedUploadFile(ctx, f) + prog := driver.NewProgress(size, model.UpdateProgressWithRange(up, 0, 1)) + + method := strings.ToUpper(strings.TrimSpace(info.Method)) + if method == "" { + method = http.MethodPut + } + + var req *http.Request + if method == http.MethodPost && len(info.Request.FormData) > 0 { + if size == 0 { + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + return err + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + return err + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + return err + } + if err := mw.Close(); err != nil { + return err + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, bytes.NewReader(buf.Bytes())) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(buf.Len()) + req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10)) + } else { + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + req, err = http.NewRequestWithContext(ctx, method, info.URL, pr) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + go func() { + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + pw.CloseWithError(err) + return + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + pw.CloseWithError(err) + return + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + pw.CloseWithError(err) + return + } + if err := mw.Close(); err != nil { + pw.CloseWithError(err) + return + } + pw.Close() + }() + } + } else { + var body = io.TeeReader(rf, prog) + if size == 0 { + body = bytes.NewReader(nil) + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, body) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.ContentLength = size + req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) + } + + c := *base.RestyClient.GetClient() + c.Timeout = 0 + resp, err := (&c).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !statusOK(resp.StatusCode, info.Response.ExpectCode) { + io.Copy(io.Discard, resp.Body) + return fmt.Errorf("http error: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + etag := normalizeETag(respArg(info.Response.ArgsETag, resp, body)) + if etag == "" { + etag = normalizeETag(resp.Header.Get("ETag")) + } + + key := strings.TrimSpace(respArg(info.Response.ArgsKey, resp, body)) + if key == "" { + key = strings.TrimSpace(resp.Header.Get("x-obs-save-key")) + } + + var pr uploadPutResp + sha1FromServer := "" + if err := json.Unmarshal(body, &pr); err == nil { + sha1FromServer = strings.TrimSpace(pr.NewFilename) + if sha1FromServer == "" { + sha1FromServer = strings.TrimSpace(pr.Sha1) + } + if etag == "" && pr.MD5 != "" { + etag = strings.TrimSpace(pr.MD5) + } + } + + if sha1FromServer == "" { + if v := extractXMLTag(string(body), "ETag"); v != "" { + sha1FromServer = v + if etag == "" { + etag = v + } + } + } + if sha1FromServer == "" && key != "" && len(key) == 40 { + sha1FromServer = key + } + if sha1FromServer == "" { + sha1FromServer = sha1Hex + } + + if etag == "" { + return fmt.Errorf("empty etag") + } + if sha1FromServer == "" { + return fmt.Errorf("empty sha1") + } + + store := strings.TrimSpace(info.Store) + commitKey := "" + if strings.TrimSpace(info.Response.ArgsKey) != "" { + commitKey = key + if commitKey == "" { + commitKey = sha1FromServer + } + } + + if err := d.commitUpload(ctx, etag, commitKey, node.group.GroupID, parentID, uploadName, sha1FromServer, size, store); err != nil { + return err + } + + up(1) + return nil +} + +func (d *Wps) spaces(ctx context.Context) (*spacesResp, error) { + url := fmt.Sprintf("%s/api/v3/spaces", d.driveHost()+d.drivePrefix()) + var resp spacesResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return &resp, nil +} diff --git a/drivers/yandex_disk/driver.go b/drivers/yandex_disk/driver.go index b3591f380..2a7996e22 100644 --- a/drivers/yandex_disk/driver.go +++ b/drivers/yandex_disk/driver.go @@ -41,7 +41,9 @@ func (d *YandexDisk) List(ctx context.Context, dir model.Obj, args model.ListArg return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil + obj := fileToObj(src) + obj.Path = path.Join(dir.GetPath(), obj.Name) + return obj, nil }) } diff --git a/drivers/yandex_disk/types.go b/drivers/yandex_disk/types.go index 2481f051e..78e6203c5 100644 --- a/drivers/yandex_disk/types.go +++ b/drivers/yandex_disk/types.go @@ -42,7 +42,7 @@ type File struct { //Revision int64 `json:"revision"` } -func fileToObj(f File) model.Obj { +func fileToObj(f File) *model.Object { return &model.Object{ Name: f.Name, Size: f.Size, diff --git a/drivers/yandex_disk/util.go b/drivers/yandex_disk/util.go index e6f4cab06..2da2cb73a 100644 --- a/drivers/yandex_disk/util.go +++ b/drivers/yandex_disk/util.go @@ -23,7 +23,6 @@ func (d *YandexDisk) refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, diff --git a/go.mod b/go.mod index 78416068b..c36ac1ca0 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,29 @@ module github.com/OpenListTeam/OpenList/v4 -go 1.23.4 +go 1.24.0 + +toolchain go1.24.13 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 + github.com/KarpelesLab/reflink v1.0.2 + github.com/KirCute/zip v1.0.1 github.com/OpenListTeam/go-cache v0.1.0 github.com/OpenListTeam/sftpd-openlist v1.0.1 - github.com/OpenListTeam/tache v0.2.0 + github.com/OpenListTeam/tache v0.2.2 github.com/OpenListTeam/times v0.1.0 github.com/OpenListTeam/wopan-sdk-go v0.1.5 github.com/ProtonMail/go-crypto v1.3.0 - github.com/SheltonZhu/115driver v1.1.1 + github.com/ProtonMail/gopenpgp/v2 v2.9.0 + github.com/SheltonZhu/115driver v1.2.3 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible + github.com/antchfx/htmlquery v1.3.5 + github.com/antchfx/xpath v1.3.5 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.55.7 github.com/blevesearch/bleve/v2 v2.5.2 + github.com/bmatcuk/doublestar/v4 v4.9.1 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.21.0 github.com/charmbracelet/bubbletea v1.3.6 @@ -30,7 +38,7 @@ require ( github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 github.com/foxxorcat/mopan-sdk-go v0.1.6 - github.com/foxxorcat/weiyun-sdk-go v0.1.3 + github.com/foxxorcat/weiyun-sdk-go v0.1.4 github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 github.com/go-resty/resty/v2 v2.16.5 @@ -38,7 +46,9 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 github.com/hekmon/transmissionrpc/v3 v3.0.0 + github.com/henrybear327/go-proton-api v1.0.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/itsHenry35/gofakes3 v0.0.8 github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 @@ -52,8 +62,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.9 github.com/pquerna/otp v1.5.0 + github.com/quic-go/quic-go v0.54.1 github.com/rclone/rclone v1.70.3 - github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d github.com/shirou/gopsutil/v4 v4.25.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.14.0 @@ -64,13 +74,12 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.6.0 - github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 - golang.org/x/crypto v0.40.0 + golang.org/x/crypto v0.46.0 golang.org/x/image v0.29.0 - golang.org/x/net v0.42.0 - golang.org/x/oauth2 v0.30.0 - golang.org/x/time v0.12.0 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 + golang.org/x/time v0.14.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 @@ -80,15 +89,26 @@ require ( ) require ( - cloud.google.com/go/compute/metadata v0.7.0 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect + github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect + github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect + github.com/ProtonMail/go-srp v0.0.7 // indirect + github.com/PuerkitoBio/goquery v1.10.3 // indirect github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/bradenaw/juniper v0.15.3 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cronokirby/saferith v0.33.0 // indirect github.com/ebitengine/purego v0.8.4 // indirect + github.com/emersion/go-message v0.18.2 // indirect + github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect github.com/geoffgarside/ber v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect @@ -101,13 +121,17 @@ require ( github.com/minio/minlz v1.0.0 // indirect github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/otiai10/mint v1.6.3 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/relvacode/iso8601 v1.6.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + go.uber.org/mock v0.5.0 // indirect + golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect + golang.org/x/mod v0.30.0 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect ) require ( - github.com/OpenListTeam/115-sdk-go v0.2.2 + github.com/OpenListTeam/115-sdk-go v0.2.3 github.com/STARRY-S/zip v0.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/blevesearch/go-faiss v1.0.25 // indirect @@ -137,7 +161,7 @@ require ( github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/yuin/goldmark v1.7.13 - go4.org v0.0.0-20230225012048-214862532bf5 + go4.org v0.0.0-20260112195520-a5071408f32f resty.dev/v3 v3.0.0-beta.2 // indirect ) @@ -194,6 +218,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect + github.com/henrybear327/Proton-API-Bridge v1.0.0 github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.5.0 github.com/jackc/pgpassfile v1.0.0 // indirect @@ -237,7 +262,6 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/otiai10/copy v1.14.1 github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -263,18 +287,22 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.4.0 // indirect golang.org/x/arch v0.18.0 // indirect - golang.org/x/sync v0.16.0 - golang.org/x/sys v0.34.0 - golang.org/x/term v0.33.0 // indirect - golang.org/x/text v0.27.0 - golang.org/x/tools v0.34.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.40.0 + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 + golang.org/x/tools v0.39.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) +replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0 + +replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed + // replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go diff --git a/go.sum b/go.sum index af6b00215..b9a4570bd 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbf cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -37,30 +38,52 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= +github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= +github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= +github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= +github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= +github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/OpenListTeam/115-sdk-go v0.2.2 h1:JCrGHqQjBX3laOA6Hw4CuBovSg7g+FC5s0LEAYsRciU= -github.com/OpenListTeam/115-sdk-go v0.2.2/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/OpenListTeam/115-sdk-go v0.2.3 h1:nDNz0GxgliW+nT2Ds486k/rp/GgJj7Ngznc98ZBUwZo= +github.com/OpenListTeam/115-sdk-go v0.2.3/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s= github.com/OpenListTeam/sftpd-openlist v1.0.1 h1:j4S3iPFOpnXCUKRPS7uCT4mF2VCl34GyqvH6lqwnkUU= github.com/OpenListTeam/sftpd-openlist v1.0.1/go.mod h1:uO/wKnbvbdq3rBLmClMTZXuCnw7XW4wlAq4dZe91a40= -github.com/OpenListTeam/tache v0.2.0 h1:Q4MjuyECn0CZCf1ZF91JaVaZTaps1mOTAm8bFj8sr9Q= -github.com/OpenListTeam/tache v0.2.0/go.mod h1:qmnZ/VpY2DUlmjg3UoDeNFy/LRqrw0biN3hYEEGc/+A= +github.com/OpenListTeam/tache v0.2.2 h1:CWFn6sr1AIYaEjC8ONdKs+LrxHyuErheenAjEqRhh4k= +github.com/OpenListTeam/tache v0.2.2/go.mod h1:qmnZ/VpY2DUlmjg3UoDeNFy/LRqrw0biN3hYEEGc/+A= github.com/OpenListTeam/times v0.1.0 h1:qknxw+qj5CYKgXAwydA102UEpPcpU8TYNGRmwRyPYpg= github.com/OpenListTeam/times v0.1.0/go.mod h1:Jx7qen5NCYzKk2w14YuvU48YYMcPa1P9a+EJePC15Pc= github.com/OpenListTeam/wopan-sdk-go v0.1.5 h1:iKKcVzIqBgtGDbn0QbdWrCazSGxXFmYFyrnFBG+U8dI= github.com/OpenListTeam/wopan-sdk-go v0.1.5/go.mod h1:otynv0CgSNUClPpUgZ44qCZGcMRe0dc83Pkk65xAunI= +github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= +github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= +github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= +github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk= +github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE= +github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s= +github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= +github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.1.1 h1:9EMhe2ZJflGiAaZbYInw2jqxTcqZNF+DtVDsEy70aFU= -github.com/SheltonZhu/115driver v1.1.1/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= +github.com/SheltonZhu/115driver v1.2.3 h1:94XMP/ey7VXIlpoBLIJHEoXu7N8YsELZlXVbxWcDDvk= +github.com/SheltonZhu/115driver v1.2.3/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= @@ -72,6 +95,12 @@ github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAP github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9yDpcdEr6Odf23feCxK3LNUNMxjXg41pZQ= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/antchfx/htmlquery v1.3.5 h1:aYthDDClnG2a2xePf6tys/UyyM/kRcsFRm+ifhFKoU0= +github.com/antchfx/htmlquery v1.3.5/go.mod h1:5oyIPIa3ovYGtLqMPNjBF2Uf25NPCKsMjCnQ8lvjaoA= +github.com/antchfx/xpath v1.3.5 h1:PqbXLC3TkfeZyakF5eeh3NTWEbYl4VHNVeufANzDbKQ= +github.com/antchfx/xpath v1.3.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -156,6 +185,8 @@ github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFx github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= github.com/blevesearch/zapx/v16 v16.2.4 h1:tGgfvleXTAkwsD5mEzgM3zCS/7pgocTCnO1oyAUjlww= github.com/blevesearch/zapx/v16 v16.2.4/go.mod h1:Rti/REtuuMmzwsI8/C/qIzRaEoSK/wiFYw5e5ctUKKs= +github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= +github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4= @@ -164,6 +195,9 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo= +github.com/bradenaw/juniper v0.15.3/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -198,6 +232,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg= @@ -239,6 +274,10 @@ github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJL github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= +github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= +github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg= +github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= @@ -251,8 +290,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= -github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= -github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= +github.com/foxxorcat/weiyun-sdk-go v0.1.4 h1:X2tFvdqikkJ7awCBbMH7XXk7+uQoJlQksJz9CUU6ZgA= +github.com/foxxorcat/weiyun-sdk-go v0.1.4/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= @@ -313,6 +352,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -364,6 +405,8 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 h1:lsK2GVgI2Ox0NkRpQnN09GBOH7jtsjFK5tcIgxXlLr0= +github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -385,6 +428,10 @@ github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg= +github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0= +github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= +github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= +github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -549,10 +596,6 @@ github.com/ncw/swift/v2 v2.0.4 h1:hHWVFxn5/YaTWAASmn4qyq2p6OyP/Hm3vMLzkjEqR7w= github.com/ncw/swift/v2 v2.0.4/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.1.1 h1:OJaYalXdliBUXPmC8CZGQ7oZDxzX1/5mQmgn0/GASew= github.com/nwaples/rardecode/v2 v2.1.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= -github.com/otiai10/copy v1.14.1 h1:5/7E6qsUMBaH5AnQ0sSLzzTg1oTECmcCmT6lvF45Na8= -github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I= -github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= -github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= @@ -585,8 +628,14 @@ github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQP github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= +github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE= github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo= +github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= +github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -599,8 +648,6 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= -github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= @@ -673,8 +720,6 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 h1:K8gF0eekWPEX+57l30ixxzGhHH/qscI3JCnuhbN6V4M= -github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9/go.mod h1:9BnoKCcgJ/+SLhfAXj15352hTOuVmG5Gzo8xNRINfqI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= @@ -702,8 +747,14 @@ go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5J go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= +go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= +go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= @@ -715,12 +766,15 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -756,6 +810,10 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -775,12 +833,16 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -788,6 +850,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -802,6 +866,8 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -821,6 +887,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -835,10 +902,13 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= @@ -846,6 +916,8 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -854,6 +926,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -861,10 +934,14 @@ golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -895,8 +972,10 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -932,6 +1011,8 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -941,10 +1022,14 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/archive/rardecode/rardecode.go b/internal/archive/rardecode/rardecode.go index 13a22e3e8..019841d3b 100644 --- a/internal/archive/rardecode/rardecode.go +++ b/internal/archive/rardecode/rardecode.go @@ -4,6 +4,7 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" @@ -21,7 +22,7 @@ func (RarDecoder) AcceptedExtensions() []string { func (RarDecoder) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{ - ".part1.rar": {".part%d.rar", 2}, + ".part1.rar": {PartFileFormat: regexp.MustCompile(`^.*\.part(\d+)\.rar$`), SecondPartIndex: 2}, } } diff --git a/internal/archive/sevenzip/sevenzip.go b/internal/archive/sevenzip/sevenzip.go index c59cf27c2..bf2cb78ad 100644 --- a/internal/archive/sevenzip/sevenzip.go +++ b/internal/archive/sevenzip/sevenzip.go @@ -2,6 +2,7 @@ package sevenzip import ( "io" + "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" @@ -18,7 +19,7 @@ func (SevenZip) AcceptedExtensions() []string { func (SevenZip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{ - ".7z.001": {".7z.%.3d", 2}, + ".7z.001": {PartFileFormat: regexp.MustCompile(`^.*\.7z\.(\d+)$`), SecondPartIndex: 2}, } } diff --git a/internal/archive/tool/base.go b/internal/archive/tool/base.go index 606094534..571ada748 100644 --- a/internal/archive/tool/base.go +++ b/internal/archive/tool/base.go @@ -2,13 +2,14 @@ package tool import ( "io" + "regexp" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" ) type MultipartExtension struct { - PartFileFormat string + PartFileFormat *regexp.Regexp SecondPartIndex int } diff --git a/internal/archive/zip/utils.go b/internal/archive/zip/utils.go index 0cd1a637a..4f367a631 100644 --- a/internal/archive/zip/utils.go +++ b/internal/archive/zip/utils.go @@ -4,22 +4,15 @@ import ( "bytes" "io" "io/fs" - stdpath "path" "strings" + "github.com/KirCute/zip" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" - "github.com/saintfish/chardet" - "github.com/yeka/zip" - "golang.org/x/text/encoding" - "golang.org/x/text/encoding/charmap" - "golang.org/x/text/encoding/japanese" - "golang.org/x/text/encoding/korean" - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/encoding/traditionalchinese" - "golang.org/x/text/encoding/unicode" - "golang.org/x/text/encoding/unicode/utf32" + "golang.org/x/text/encoding/ianaindex" "golang.org/x/text/transform" ) @@ -37,10 +30,11 @@ func (r *WrapReader) Files() []tool.SubFile { type WrapFileInfo struct { fs.FileInfo + efs bool } func (f *WrapFileInfo) Name() string { - return decodeName(f.FileInfo.Name()) + return decodeName(f.FileInfo.Name(), f.efs) } type WrapFile struct { @@ -48,11 +42,11 @@ type WrapFile struct { } func (f *WrapFile) Name() string { - return decodeName(f.f.Name) + return decodeName(f.f.Name, isEFS(f.f.Flags)) } func (f *WrapFile) FileInfo() fs.FileInfo { - return &WrapFileInfo{FileInfo: f.f.FileInfo()} + return &WrapFileInfo{FileInfo: f.f.FileInfo(), efs: isEFS(f.f.Flags)} } func (f *WrapFile) Open() (io.ReadCloser, error) { @@ -67,16 +61,33 @@ func (f *WrapFile) SetPassword(password string) { f.f.SetPassword(password) } -func getReader(ss []*stream.SeekableStream) (*zip.Reader, error) { - if len(ss) > 1 && stdpath.Ext(ss[1].GetName()) == ".z01" { - // FIXME: Incorrect parsing method for standard multipart zip format - ss = append(ss[1:], ss[0]) - } - reader, err := stream.NewMultiReaderAt(ss) +func makePart(ss *stream.SeekableStream) (zip.SizeReaderAt, error) { + ra, err := stream.NewReadAtSeeker(ss, 0) if err != nil { return nil, err } - return zip.NewReader(reader, reader.Size()) + return &inlineSizeReaderAt{ReaderAt: ra, size: ss.GetSize()}, nil +} + +func (z *Zip) getReader(ss []*stream.SeekableStream) (*zip.Reader, error) { + if len(ss) > 1 && z.traditionalSecondPartRegExp.MatchString(ss[1].GetName()) { + ss = append(ss[1:], ss[0]) + ras := make([]zip.SizeReaderAt, 0, len(ss)) + for _, s := range ss { + ra, err := makePart(s) + if err != nil { + return nil, err + } + ras = append(ras, ra) + } + return zip.NewMultipartReader(ras) + } else { + reader, err := stream.NewMultiReaderAt(ss) + if err != nil { + return nil, err + } + return zip.NewReader(reader, reader.Size()) + } } func filterPassword(err error) error { @@ -86,110 +97,29 @@ func filterPassword(err error) error { return err } -func decodeName(name string) string { - b := []byte(name) - detector := chardet.NewTextDetector() - results, err := detector.DetectAll(b) - if err != nil { +func decodeName(name string, efs bool) string { + if efs { return name } - var ce, re, enc encoding.Encoding - for _, r := range results { - if r.Confidence > 30 { - ce = getCommonEncoding(r.Charset) - if ce != nil { - break - } - } - if re == nil { - re = getEncoding(r.Charset) - } - } - if ce != nil { - enc = ce - } else if re != nil { - enc = re - } else { + enc, err := ianaindex.IANA.Encoding(setting.GetStr(conf.NonEFSZipEncoding)) + if err != nil { return name } - i := bytes.NewReader(b) + i := bytes.NewReader([]byte(name)) decoder := transform.NewReader(i, enc.NewDecoder()) content, _ := io.ReadAll(decoder) return string(content) } -func getCommonEncoding(name string) (enc encoding.Encoding) { - switch name { - case "UTF-8": - enc = unicode.UTF8 - case "UTF-16LE": - enc = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) - case "Shift_JIS": - enc = japanese.ShiftJIS - case "GB-18030": - enc = simplifiedchinese.GB18030 - case "EUC-KR": - enc = korean.EUCKR - case "Big5": - enc = traditionalchinese.Big5 - default: - enc = nil - } - return -} - -func getEncoding(name string) (enc encoding.Encoding) { - switch name { - case "UTF-8": - enc = unicode.UTF8 - case "UTF-16BE": - enc = unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM) - case "UTF-16LE": - enc = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) - case "UTF-32BE": - enc = utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM) - case "UTF-32LE": - enc = utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM) - case "ISO-8859-1": - enc = charmap.ISO8859_1 - case "ISO-8859-2": - enc = charmap.ISO8859_2 - case "ISO-8859-3": - enc = charmap.ISO8859_3 - case "ISO-8859-4": - enc = charmap.ISO8859_4 - case "ISO-8859-5": - enc = charmap.ISO8859_5 - case "ISO-8859-6": - enc = charmap.ISO8859_6 - case "ISO-8859-7": - enc = charmap.ISO8859_7 - case "ISO-8859-8": - enc = charmap.ISO8859_8 - case "ISO-8859-8-I": - enc = charmap.ISO8859_8I - case "ISO-8859-9": - enc = charmap.ISO8859_9 - case "windows-1251": - enc = charmap.Windows1251 - case "windows-1256": - enc = charmap.Windows1256 - case "KOI8-R": - enc = charmap.KOI8R - case "Shift_JIS": - enc = japanese.ShiftJIS - case "GB-18030": - enc = simplifiedchinese.GB18030 - case "EUC-JP": - enc = japanese.EUCJP - case "EUC-KR": - enc = korean.EUCKR - case "Big5": - enc = traditionalchinese.Big5 - case "ISO-2022-JP": - enc = japanese.ISO2022JP - default: - enc = nil - } - return +func isEFS(flags uint16) bool { + return (flags & 0x800) > 0 +} + +type inlineSizeReaderAt struct { + io.ReaderAt + size int64 +} + +func (i *inlineSizeReaderAt) Size() int64 { + return i.size } diff --git a/internal/archive/zip/zip.go b/internal/archive/zip/zip.go index 8ed83ac72..1dcb904d7 100644 --- a/internal/archive/zip/zip.go +++ b/internal/archive/zip/zip.go @@ -3,6 +3,7 @@ package zip import ( "io" stdpath "path" + "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" @@ -12,34 +13,39 @@ import ( ) type Zip struct { + traditionalSecondPartRegExp *regexp.Regexp } -func (Zip) AcceptedExtensions() []string { +func (z *Zip) AcceptedExtensions() []string { return []string{} } -func (Zip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { +func (z *Zip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{ - ".zip": {".z%.2d", 1}, - ".zip.001": {".zip.%.3d", 2}, + ".zip": {PartFileFormat: regexp.MustCompile(`^.*\.z(\d+)$`), SecondPartIndex: 1}, + ".zip.001": {PartFileFormat: regexp.MustCompile(`^.*\.zip\.(\d+)$`), SecondPartIndex: 2}, } } -func (Zip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { - zipReader, err := getReader(ss) +func (z *Zip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + zipReader, err := z.getReader(ss) if err != nil { return nil, err } + efs := true + if len(zipReader.File) > 0 { + efs = isEFS(zipReader.File[0].Flags) + } encrypted, tree := tool.GenerateMetaTreeFromFolderTraversal(&WrapReader{Reader: zipReader}) return &model.ArchiveMetaInfo{ - Comment: zipReader.Comment, + Comment: decodeName(zipReader.Comment, efs), Encrypted: encrypted, Tree: tree, }, nil } -func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { - zipReader, err := getReader(ss) +func (z *Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + zipReader, err := z.getReader(ss) if err != nil { return nil, err } @@ -57,7 +63,7 @@ func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]mod _ = rc.Close() passVerified = true } - name := strings.TrimSuffix(decodeName(file.Name), "/") + name := strings.TrimSuffix(decodeName(file.Name, isEFS(file.Flags)), "/") if strings.Contains(name, "/") { // 有些压缩包不压缩第一个文件夹 strs := strings.Split(name, "/") @@ -70,7 +76,7 @@ func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]mod } continue } - ret = append(ret, tool.MakeModelObj(&WrapFileInfo{FileInfo: file.FileInfo()})) + ret = append(ret, tool.MakeModelObj(&WrapFileInfo{FileInfo: file.FileInfo(), efs: isEFS(file.Flags)})) } if len(ret) == 0 && dir != nil { ret = append(ret, dir) @@ -81,13 +87,13 @@ func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]mod ret := make([]model.Obj, 0) exist := false for _, file := range zipReader.File { - name := decodeName(file.Name) + name := decodeName(file.Name, isEFS(file.Flags)) dir := stdpath.Dir(strings.TrimSuffix(name, "/")) + "/" if dir != innerPath { continue } exist = true - ret = append(ret, tool.MakeModelObj(&WrapFileInfo{file.FileInfo()})) + ret = append(ret, tool.MakeModelObj(&WrapFileInfo{file.FileInfo(), isEFS(file.Flags)})) } if !exist { return nil, errs.ObjectNotFound @@ -96,14 +102,14 @@ func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]mod } } -func (Zip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { - zipReader, err := getReader(ss) +func (z *Zip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + zipReader, err := z.getReader(ss) if err != nil { return nil, 0, err } innerPath := strings.TrimPrefix(args.InnerPath, "/") for _, file := range zipReader.File { - if decodeName(file.Name) == innerPath { + if decodeName(file.Name, isEFS(file.Flags)) == innerPath { if file.IsEncrypted() { file.SetPassword(args.Password) } @@ -117,8 +123,8 @@ func (Zip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io return nil, 0, errs.ObjectNotFound } -func (Zip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { - zipReader, err := getReader(ss) +func (z *Zip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + zipReader, err := z.getReader(ss) if err != nil { return err } @@ -128,5 +134,7 @@ func (Zip) Decompress(ss []*stream.SeekableStream, outputPath string, args model var _ tool.Tool = (*Zip)(nil) func init() { - tool.RegisterTool(Zip{}) + tool.RegisterTool(&Zip{ + traditionalSecondPartRegExp: regexp.MustCompile(`^.*\.z0*1$`), + }) } diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 2209c64f3..74e218f8f 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -39,7 +39,22 @@ func InitConfig() { if !filepath.IsAbs(dataDir) { flags.DataDir = filepath.Join(pwd, flags.DataDir) } - configPath := filepath.Join(flags.DataDir, "config.json") + // Determine config file path: use flags.ConfigPath if provided, otherwise default to /config.json + configPath := flags.ConfigPath + if configPath == "" { + configPath = filepath.Join(flags.DataDir, "config.json") + } else { + // if relative, resolve relative to working directory + if !filepath.IsAbs(configPath) { + if absPath, err := filepath.Abs(configPath); err == nil { + configPath = absPath + } else { + configPath = filepath.Join(pwd, configPath) + } + } + } + configPath = filepath.Clean(configPath) + conf.ConfigPath = configPath log.Infof("reading config file: %s", configPath) if !utils.Exists(configPath) { log.Infof("config file not exists, creating default config file") @@ -126,6 +141,10 @@ func InitConfig() { log.Fatalf("create temp dir error: %+v", err) } log.Debugf("config: %+v", conf.Conf) + + // Validate and display proxy configuration status + validateProxyConfig() + base.InitClient() initURL() } @@ -165,3 +184,14 @@ func CleanTempDir() { } } } + +// validateProxyConfig validates proxy configuration and displays status at startup +func validateProxyConfig() { + if conf.Conf.ProxyAddress != "" { + if _, err := url.Parse(conf.Conf.ProxyAddress); err == nil { + log.Infof("Proxy enabled: %s", conf.Conf.ProxyAddress) + } else { + log.Errorf("Invalid proxy address format: %s, error: %v", conf.Conf.ProxyAddress, err) + } + } +} diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 08024695c..7bff851de 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -148,9 +148,13 @@ func InitialSettings() []model.SettingItem { {Key: "audio_cover", Value: "https://res.oplist.org/logo/logo.svg", MigrationValue: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW}, {Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.PreviewDownloadByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.SharePreviewDownloadByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.SharePreviewArchivesByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.NonEFSZipEncoding, Value: "IBM437", Type: conf.TypeString, Group: model.PREVIEW}, // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, {Key: "package_download", Value: "true", Type: conf.TypeBool, Group: model.GLOBAL}, @@ -173,6 +177,9 @@ func InitialSettings() []model.SettingItem { {Key: conf.ShareArchivePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, {Key: conf.ShareForceProxy, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.ShareSummaryContent, Value: "@{{creator}} shared {{#each files}}{{#if @first}}\"{{filename this}}\"{{/if}}{{#if @last}}{{#unless (eq @index 0)}} and {{@index}} more files{{/unless}}{{/if}}{{/each}} from {{site_title}}: {{base_url}}/@s/{{id}}{{#if pwd}} , the share code is {{pwd}}{{/if}}{{#if expires}}, please access before {{dateLocaleString expires}}.{{/if}}", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PUBLIC}, + {Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, @@ -201,6 +208,7 @@ func InitialSettings() []model.SettingItem { // ldap settings {Key: conf.LdapLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PUBLIC}, {Key: conf.LdapServer, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapSkipTlsVerify, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapManagerDN, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapManagerPassword, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapUserSearchBase, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, diff --git a/internal/bootstrap/patch.go b/internal/bootstrap/patch.go index 1e76190c8..e8baeeac9 100644 --- a/internal/bootstrap/patch.go +++ b/internal/bootstrap/patch.go @@ -2,7 +2,6 @@ package bootstrap import ( "fmt" - "strings" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch" diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go index 44ba17376..5d4c814dc 100644 --- a/internal/bootstrap/patch/all.go +++ b/internal/bootstrap/patch/all.go @@ -4,7 +4,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_24_0" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_32_0" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_41_0" - "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_all" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v4_1_8" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v4_1_9" ) type VersionPatches struct { @@ -34,9 +35,16 @@ var UpgradePatches = []VersionPatches{ }, }, { - Version: "v3.0.0", + Version: "v4.1.8", Patches: []func(){ - v3_all.RenameAlistV3Driver, + v4_1_8.FixAliasConfig, + }, + }, + { + Version: "v4.1.9", + Patches: []func(){ + v4_1_9.EnableWebDavProxy, + v4_1_9.ResetSkipTlsVerify, }, }, } diff --git a/internal/bootstrap/patch/v3_24_0/hash_password.go b/internal/bootstrap/patch/v3_24_0/hash_password.go index 881e3d47b..30a296c47 100644 --- a/internal/bootstrap/patch/v3_24_0/hash_password.go +++ b/internal/bootstrap/patch/v3_24_0/hash_password.go @@ -11,7 +11,8 @@ import ( func HashPwdForOldVersion() { users, _, err := op.GetUsers(1, -1) if err != nil { - utils.Log.Fatalf("[hash pwd for old version] failed get users: %v", err) + utils.Log.Errorf("[hash pwd for old version] failed get users: %v", err) + return } for i := range users { user := users[i] @@ -19,7 +20,7 @@ func HashPwdForOldVersion() { user.SetPassword(user.Password) user.Password = "" if err := db.UpdateUser(&user); err != nil { - utils.Log.Fatalf("[hash pwd for old version] failed update user: %v", err) + utils.Log.Errorf("[hash pwd for old version] failed update user: %v", err) } } } diff --git a/internal/bootstrap/patch/v3_32_0/update_authn.go b/internal/bootstrap/patch/v3_32_0/update_authn.go index fea4bbfb1..721c3f582 100644 --- a/internal/bootstrap/patch/v3_32_0/update_authn.go +++ b/internal/bootstrap/patch/v3_32_0/update_authn.go @@ -11,14 +11,15 @@ import ( func UpdateAuthnForOldVersion() { users, _, err := op.GetUsers(1, -1) if err != nil { - utils.Log.Fatalf("[update authn for old version] failed get users: %v", err) + utils.Log.Errorf("[update authn for old version] failed get users: %v", err) + return } for i := range users { user := users[i] if user.Authn == "" { user.Authn = "[]" if err := db.UpdateUser(&user); err != nil { - utils.Log.Fatalf("[update authn for old version] failed update user: %v", err) + utils.Log.Errorf("[update authn for old version] failed update user: %v", err) } } } diff --git a/internal/bootstrap/patch/v3_all/rename.go b/internal/bootstrap/patch/v3_all/rename.go deleted file mode 100644 index 323b7391f..000000000 --- a/internal/bootstrap/patch/v3_all/rename.go +++ /dev/null @@ -1,33 +0,0 @@ -package v3_all - -import ( - "github.com/OpenListTeam/OpenList/v4/internal/db" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" -) - -// Rename Alist V3 driver to OpenList -func RenameAlistV3Driver() { - storages, _, err := db.GetStorages(1, -1) - if err != nil { - utils.Log.Errorf("[RenameAlistV3Driver] failed to get storages: %s", err.Error()) - return - } - - updatedCount := 0 - for _, s := range storages { - if s.Driver == "AList V3" { - utils.Log.Warnf("[RenameAlistV3Driver] rename storage [%d]%s from Alist V3 to OpenList", s.ID, s.MountPath) - s.Driver = "OpenList" - err = db.UpdateStorage(&s) - if err != nil { - utils.Log.Errorf("[RenameAlistV3Driver] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) - } else { - updatedCount++ - } - } - } - - if updatedCount > 0 { - utils.Log.Infof("[RenameAlistV3Driver] updated %d storages from Alist V3 to OpenList", updatedCount) - } -} diff --git a/internal/bootstrap/patch/v4_1_8/alias.go b/internal/bootstrap/patch/v4_1_8/alias.go new file mode 100644 index 000000000..e41742654 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_8/alias.go @@ -0,0 +1,81 @@ +package v4_1_8 + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// FixAliasConfig upgrade the old version of the Addition of the Alias driver +func FixAliasConfig() { + storages, _, err := db.GetStorages(1, -1) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to get storages: %s", err.Error()) + return + } + for _, s := range storages { + if s.Driver != "Alias" { + continue + } + addition := make(map[string]any) + err = utils.Json.UnmarshalFromString(s.Addition, &addition) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to unmarshal addition of [%d]%s: %s", s.ID, s.MountPath, err.Error()) + continue + } + if _, ok := addition["read_conflict_policy"]; ok { + utils.Log.Infof("[FixAliasConfig] skip fixing [%d]%s because the addition already has \"read_conflict_policy\" key", s.ID, s.MountPath) + continue + } + var protectSameName, parallelWrite, writable bool + protectSameNameAny, ok := addition["protect_same_name"] + if ok { + delete(addition, "protect_same_name") + protectSameName, ok = protectSameNameAny.(bool) + } + if !ok { + protectSameName = false + } + parallelWriteAny, ok := addition["parallel_write"] + if ok { + delete(addition, "parallel_write") + parallelWrite, ok = parallelWriteAny.(bool) + } + if !ok { + parallelWrite = false + } + writableAny, ok := addition["writable"] + if ok { + delete(addition, "writable") + writable, ok = writableAny.(bool) + } + if !ok { + writable = false + } + if !writable { + addition["write_conflict_policy"] = "disabled" + addition["put_conflict_policy"] = "disabled" + } else if !protectSameName && !parallelWrite { + addition["write_conflict_policy"] = "first" + addition["put_conflict_policy"] = "first" + } else if protectSameName && !parallelWrite { + addition["write_conflict_policy"] = "deterministic" + addition["put_conflict_policy"] = "deterministic" + } else if !protectSameName && parallelWrite { + addition["write_conflict_policy"] = "all" + addition["put_conflict_policy"] = "all" + } else { + addition["write_conflict_policy"] = "deterministic_or_all" + addition["put_conflict_policy"] = "deterministic_or_all" + } + addition["read_conflict_policy"] = "first" + s.Addition, err = utils.Json.MarshalToString(addition) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to marshal addition of [%d]%s: %s", s.ID, s.MountPath, err.Error()) + continue + } + err = db.UpdateStorage(&s) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) + } + } +} diff --git a/internal/bootstrap/patch/v4_1_9/skip_tls.go b/internal/bootstrap/patch/v4_1_9/skip_tls.go new file mode 100644 index 000000000..1d9858d95 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_9/skip_tls.go @@ -0,0 +1,32 @@ +package v4_1_9 + +import ( + "os" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +func ResetSkipTlsVerify() { + if !conf.Conf.TlsInsecureSkipVerify { + return + } + if !strings.HasPrefix(conf.Version, "v") { + return + } + + conf.Conf.TlsInsecureSkipVerify = false + + confBody, err := utils.Json.MarshalIndent(conf.Conf, "", " ") + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: marshal config error: %+v", err) + return + } + err = os.WriteFile(conf.ConfigPath, confBody, 0o777) + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: update config struct error: %+v", err) + return + } + utils.Log.Infof("[ResetSkipTlsVerify] succeeded to set tls_insecure_skip_verify to false") +} diff --git a/internal/bootstrap/patch/v4_1_9/webdav.go b/internal/bootstrap/patch/v4_1_9/webdav.go new file mode 100644 index 000000000..2fd97c783 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_9/webdav.go @@ -0,0 +1,30 @@ +package v4_1_9 + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// EnableWebDavProxy updates Webdav driver storages to enable proxy +func EnableWebDavProxy() { + storages, _, err := db.GetStorages(1, -1) + if err != nil { + utils.Log.Errorf("[EnableWebDavProxy] failed to get storages: %s", err.Error()) + return + } + for _, s := range storages { + if s.Driver != "WebDav" { + continue + } + if !s.WebProxy { + s.WebProxy = true + } + if s.WebdavPolicy == "302_redirect" { + s.WebdavPolicy = "native_proxy" + } + err = db.UpdateStorage(&s) + if err != nil { + utils.Log.Errorf("[EnableWebDavProxy] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) + } + } +} diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go new file mode 100644 index 000000000..6740dba65 --- /dev/null +++ b/internal/bootstrap/run.go @@ -0,0 +1,404 @@ +package bootstrap + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "strconv" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/data" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server" + "github.com/OpenListTeam/OpenList/v4/server/middlewares" + "github.com/OpenListTeam/sftpd-openlist" + ftpserver "github.com/fclairamb/ftpserverlib" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/quic-go/quic-go/http3" + log "github.com/sirupsen/logrus" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" +) + +func Init() { + InitConfig() + Log() + InitDB() + data.InitData() + InitStreamLimit() + InitIndex() + InitUpgradePatch() +} + +func Release() { + db.Close() +} + +var ( + running bool + httpSrv *http.Server + httpRunning bool + httpsSrv *http.Server + httpsRunning bool + unixSrv *http.Server + unixRunning bool + quicSrv *http3.Server + quicRunning bool + s3Srv *http.Server + s3Running bool + ftpDriver *server.FtpMainDriver + ftpServer *ftpserver.FtpServer + ftpRunning bool + sftpDriver *server.SftpDriver + sftpServer *sftpd.SftpServer + sftpRunning bool +) + +// Called by OpenList-Mobile +func IsRunning(t string) bool { + switch t { + case "http": + return httpRunning + case "https": + return httpsRunning + case "unix": + return unixRunning + case "quic": + return quicRunning + case "s3": + return s3Running + case "sftp": + return sftpRunning + case "ftp": + return ftpRunning + } + return running +} + +func Start() { + if conf.Conf.DelayedStart != 0 { + utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart) + time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second) + } + InitOfflineDownloadTools() + LoadStorages() + InitTaskManager() + if !flags.Debug && !flags.Dev { + gin.SetMode(gin.ReleaseMode) + } + r := gin.New() + + // gin log + if conf.Conf.Log.Filter.Enable { + r.Use(middlewares.FilteredLogger()) + } else { + r.Use(gin.LoggerWithWriter(log.StandardLogger().Out)) + } + r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out)) + + server.Init(r) + var httpHandler http.Handler = r + if conf.Conf.Scheme.EnableH2c { + httpHandler = h2c.NewHandler(r, &http2.Server{}) + } + if conf.Conf.Scheme.HttpPort != -1 { + httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort) + fmt.Printf("start HTTP server @ %s\n", httpBase) + utils.Log.Infof("start HTTP server @ %s", httpBase) + httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler} + go func() { + httpRunning = true + err := httpSrv.ListenAndServe() + httpRunning = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("http", err) + utils.Log.Errorf("failed to start http: %s", err.Error()) + } else { + handleEndpointShutdownHooks("http") + } + }() + } + if conf.Conf.Scheme.HttpsPort != -1 { + httpsBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpsPort) + fmt.Printf("start HTTPS server @ %s\n", httpsBase) + utils.Log.Infof("start HTTPS server @ %s", httpsBase) + httpsSrv = &http.Server{Addr: httpsBase, Handler: r} + go func() { + httpsRunning = true + err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) + httpsRunning = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("https", err) + utils.Log.Errorf("failed to start https: %s", err.Error()) + } else { + handleEndpointShutdownHooks("https") + } + }() + if conf.Conf.Scheme.EnableH3 { + fmt.Printf("start HTTP3 (quic) server @ %s\n", httpsBase) + utils.Log.Infof("start HTTP3 (quic) server @ %s", httpsBase) + r.Use(func(c *gin.Context) { + if c.Request.TLS != nil { + port := conf.Conf.Scheme.HttpsPort + c.Header("Alt-Svc", fmt.Sprintf("h3=\":%d\"; ma=86400", port)) + } + c.Next() + }) + quicSrv = &http3.Server{Addr: httpsBase, Handler: r} + go func() { + quicRunning = true + err := quicSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) + quicRunning = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("quic", err) + utils.Log.Errorf("failed to start http3 (quic): %s", err.Error()) + } else { + handleEndpointShutdownHooks("quic") + } + }() + } + } + if conf.Conf.Scheme.UnixFile != "" { + fmt.Printf("start unix server @ %s\n", conf.Conf.Scheme.UnixFile) + utils.Log.Infof("start unix server @ %s", conf.Conf.Scheme.UnixFile) + unixSrv = &http.Server{Handler: httpHandler} + go func() { + listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile) + if err != nil { + utils.Log.Errorf("failed to listen unix: %+v", err) + return + } + unixRunning = true + // set socket file permission + mode, err := strconv.ParseUint(conf.Conf.Scheme.UnixFilePerm, 8, 32) + if err != nil { + utils.Log.Errorf("failed to parse socket file permission: %+v", err) + } else { + err = os.Chmod(conf.Conf.Scheme.UnixFile, os.FileMode(mode)) + if err != nil { + utils.Log.Errorf("failed to chmod socket file: %+v", err) + } + } + err = unixSrv.Serve(listener) + unixRunning = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("unix", err) + utils.Log.Errorf("failed to start unix: %s", err.Error()) + } else { + handleEndpointShutdownHooks("unix") + } + }() + } + if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { + s3r := gin.New() + s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) + server.InitS3(s3r) + s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) + fmt.Printf("start S3 server @ %s\n", s3Base) + utils.Log.Infof("start S3 server @ %s", s3Base) + go func() { + s3Running = true + var err error + if conf.Conf.S3.SSL { + s3Srv = &http.Server{Addr: s3Base, Handler: s3r} + err = s3Srv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) + } else { + s3Srv = &http.Server{Addr: s3Base, Handler: s3r} + err = s3Srv.ListenAndServe() + } + s3Running = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("s3", err) + utils.Log.Errorf("failed to start s3 server: %s", err.Error()) + } else { + handleEndpointShutdownHooks("s3") + } + }() + } + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable { + var err error + ftpDriver, err = server.NewMainDriver() + if err != nil { + utils.Log.Errorf("failed to start ftp driver: %s", err.Error()) + } else { + fmt.Printf("start ftp server on %s\n", conf.Conf.FTP.Listen) + utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen) + go func() { + ftpServer = ftpserver.NewFtpServer(ftpDriver) + ftpRunning = true + err = ftpServer.ListenAndServe() + ftpRunning = false + if err != nil { + handleEndpointStartFailedHooks("ftp", err) + utils.Log.Errorf("problem ftp server listening: %s", err.Error()) + } else { + handleEndpointShutdownHooks("ftp") + } + }() + } + } + if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable { + var err error + sftpDriver, err = server.NewSftpDriver() + if err != nil { + utils.Log.Errorf("failed to start sftp driver: %s", err.Error()) + } else { + fmt.Printf("start sftp server on %s", conf.Conf.SFTP.Listen) + utils.Log.Infof("start sftp server on %s", conf.Conf.SFTP.Listen) + go func() { + sftpServer = sftpd.NewSftpServer(sftpDriver) + sftpRunning = true + err = sftpServer.RunServer() + sftpRunning = false + if err != nil { + handleEndpointStartFailedHooks("sftp", err) + utils.Log.Errorf("problem sftp server listening: %s", err.Error()) + } else { + handleEndpointShutdownHooks("sftp") + } + }() + } + } + running = true +} + +func Shutdown(timeout time.Duration) { + utils.Log.Println("Shutdown server...") + fs.ArchiveContentUploadTaskManager.RemoveAll() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + var wg sync.WaitGroup + if httpSrv != nil && conf.Conf.Scheme.HttpPort != -1 { + wg.Add(1) + go func() { + defer wg.Done() + if err := httpSrv.Shutdown(ctx); err != nil { + utils.Log.Error("HTTP server shutdown err: ", err) + } + httpSrv = nil + }() + } + if httpsSrv != nil && conf.Conf.Scheme.HttpsPort != -1 { + wg.Add(1) + go func() { + defer wg.Done() + if err := httpsSrv.Shutdown(ctx); err != nil { + utils.Log.Error("HTTPS server shutdown err: ", err) + } + httpsSrv = nil + }() + if quicSrv != nil && conf.Conf.Scheme.EnableH3 { + wg.Add(1) + go func() { + defer wg.Done() + if err := quicSrv.Shutdown(ctx); err != nil { + utils.Log.Error("HTTP3 (quic) server shutdown err: ", err) + } + quicSrv = nil + }() + } + } + if unixSrv != nil && conf.Conf.Scheme.UnixFile != "" { + wg.Add(1) + go func() { + defer wg.Done() + if err := unixSrv.Shutdown(ctx); err != nil { + utils.Log.Error("Unix server shutdown err: ", err) + } + unixSrv = nil + }() + } + if s3Srv != nil && conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { + wg.Add(1) + go func() { + defer wg.Done() + if err := s3Srv.Shutdown(ctx); err != nil { + utils.Log.Error("S3 server shutdown err: ", err) + } + s3Srv = nil + }() + } + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil { + wg.Add(1) + go func() { + defer wg.Done() + if ftpDriver != nil { + ftpDriver.Stop() + ftpDriver = nil + } + if err := ftpServer.Stop(); err != nil { + utils.Log.Error("FTP server shutdown err: ", err) + } + ftpServer = nil + }() + } + if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable && sftpServer != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := sftpServer.Close(); err != nil { + utils.Log.Error("SFTP server shutdown err: ", err) + } + sftpServer = nil + sftpDriver = nil + }() + } + wg.Wait() + utils.Log.Println("Server exit") + running = false +} + +type EndpointStartFailedHook func(string, string) + +type EndpointShutdownHook func(string) + +var ( + endpointStartFailedHooks map[string]EndpointStartFailedHook + endpointShutdownHooks map[string]EndpointShutdownHook +) + +func RegisterEndpointStartFailedHook(hook EndpointStartFailedHook) string { + id := uuid.NewString() + endpointStartFailedHooks[id] = hook + return id +} + +func RemoveEndpointStartFailedHook(id string) { + delete(endpointStartFailedHooks, id) +} + +func RegisterEndpointShutdownHook(hook EndpointShutdownHook) string { + id := uuid.NewString() + endpointShutdownHooks[id] = hook + return id +} + +func RemoveEndpointShutdownHook(id string) { + delete(endpointShutdownHooks, id) +} + +func handleEndpointStartFailedHooks(t string, err error) { + for _, hook := range endpointStartFailedHooks { + hook(t, err.Error()) + } +} + +func handleEndpointShutdownHooks(t string) { + for _, hook := range endpointShutdownHooks { + hook(t) + } +} + +func init() { + endpointShutdownHooks = make(map[string]EndpointShutdownHook) + endpointStartFailedHooks = make(map[string]EndpointStartFailedHook) +} diff --git a/internal/bootstrap/storage.go b/internal/bootstrap/storage.go index d111c7d37..1389095bf 100644 --- a/internal/bootstrap/storage.go +++ b/internal/bootstrap/storage.go @@ -25,6 +25,6 @@ func LoadStorages() { storages[i].MountPath, storages[i].Driver, storages[i].Order) } } - conf.StoragesLoaded = true + conf.SendStoragesLoadedSignal() }(storages) } diff --git a/internal/cache/keyed_cache.go b/internal/cache/keyed_cache.go new file mode 100644 index 000000000..07bf8cd9f --- /dev/null +++ b/internal/cache/keyed_cache.go @@ -0,0 +1,101 @@ +package cache + +import ( + "sync" + "time" +) + +type KeyedCache[T any] struct { + entries map[string]*CacheEntry[T] + mu sync.RWMutex + ttl time.Duration +} + +func NewKeyedCache[T any](ttl time.Duration) *KeyedCache[T] { + c := &KeyedCache[T]{ + entries: make(map[string]*CacheEntry[T]), + ttl: ttl, + } + gcFuncs = append(gcFuncs, c.GC) + return c +} + +func (c *KeyedCache[T]) Set(key string, value T) { + c.SetWithExpirable(key, value, ExpirationTime(time.Now().Add(c.ttl))) +} + +func (c *KeyedCache[T]) SetWithTTL(key string, value T, ttl time.Duration) { + c.SetWithExpirable(key, value, ExpirationTime(time.Now().Add(ttl))) +} + +func (c *KeyedCache[T]) SetWithExpirable(key string, value T, exp Expirable) { + c.mu.Lock() + defer c.mu.Unlock() + + c.entries[key] = &CacheEntry[T]{ + data: value, + Expirable: exp, + } +} + +func (c *KeyedCache[T]) Get(key string) (T, bool) { + c.mu.RLock() + entry, exists := c.entries[key] + if !exists { + c.mu.RUnlock() + return *new(T), false + } + + expired := entry.Expired() + c.mu.RUnlock() + + if !expired { + return entry.data, true + } + + c.mu.Lock() + if c.entries[key] == entry { + delete(c.entries, key) + c.mu.Unlock() + return *new(T), false + } + c.mu.Unlock() + return *new(T), false +} + +func (c *KeyedCache[T]) Delete(key string) { + c.mu.Lock() + defer c.mu.Unlock() + + delete(c.entries, key) +} + +func (c *KeyedCache[T]) Pop(key string) (T, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if entry, exists := c.entries[key]; exists { + delete(c.entries, key) + return entry.data, true + } + return *new(T), false +} + +func (c *KeyedCache[T]) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + c.entries = make(map[string]*CacheEntry[T]) +} + +func (c *KeyedCache[T]) GC() { + c.mu.Lock() + defer c.mu.Unlock() + expiredKeys := make([]string, 0, len(c.entries)) + for key, entry := range c.entries { + if entry.Expired() { + expiredKeys = append(expiredKeys, key) + } + } + for _, key := range expiredKeys { + delete(c.entries, key) + } +} diff --git a/internal/cache/type.go b/internal/cache/type.go new file mode 100644 index 000000000..4eccba52a --- /dev/null +++ b/internal/cache/type.go @@ -0,0 +1,18 @@ +package cache + +import "time" + +type Expirable interface { + Expired() bool +} + +type ExpirationTime time.Time + +func (e ExpirationTime) Expired() bool { + return time.Now().After(time.Time(e)) +} + +type CacheEntry[T any] struct { + Expirable + data T +} diff --git a/internal/cache/typed_cache.go b/internal/cache/typed_cache.go new file mode 100644 index 000000000..7ba126be8 --- /dev/null +++ b/internal/cache/typed_cache.go @@ -0,0 +1,113 @@ +package cache + +import ( + "sync" + "time" +) + +type TypedCache[T any] struct { + entries map[string]map[string]*CacheEntry[T] + mu sync.RWMutex + ttl time.Duration +} + +func NewTypedCache[T any](ttl time.Duration) *TypedCache[T] { + c := &TypedCache[T]{ + entries: make(map[string]map[string]*CacheEntry[T]), + ttl: ttl, + } + gcFuncs = append(gcFuncs, c.GC) + return c +} + +func (c *TypedCache[T]) SetType(key, typeKey string, value T) { + c.SetTypeWithExpirable(key, typeKey, value, ExpirationTime(time.Now().Add(c.ttl))) +} + +func (c *TypedCache[T]) SetTypeWithTTL(key, typeKey string, value T, ttl time.Duration) { + c.SetTypeWithExpirable(key, typeKey, value, ExpirationTime(time.Now().Add(ttl))) +} + +func (c *TypedCache[T]) SetTypeWithExpirable(key, typeKey string, value T, exp Expirable) { + c.mu.Lock() + defer c.mu.Unlock() + cache, exists := c.entries[key] + if !exists { + cache = make(map[string]*CacheEntry[T]) + c.entries[key] = cache + } + + cache[typeKey] = &CacheEntry[T]{ + data: value, + Expirable: exp, + } +} + +func (c *TypedCache[T]) GetType(key, typeKey string) (T, bool) { + c.mu.RLock() + cache, exists := c.entries[key] + if !exists { + c.mu.RUnlock() + return *new(T), false + } + entry, exists := cache[typeKey] + if !exists { + c.mu.RUnlock() + return *new(T), false + } + expired := entry.Expired() + c.mu.RUnlock() + + if !expired { + return entry.data, true + } + + c.mu.Lock() + if cache[typeKey] == entry { + delete(cache, typeKey) + if len(cache) == 0 { + delete(c.entries, key) + } + c.mu.Unlock() + return *new(T), false + } + c.mu.Unlock() + return *new(T), false +} + +func (c *TypedCache[T]) DeleteKey(key string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.entries, key) +} + +func (c *TypedCache[T]) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + c.entries = make(map[string]map[string]*CacheEntry[T]) +} + +func (c *TypedCache[T]) GC() { + c.mu.Lock() + defer c.mu.Unlock() + expiredKeys := make(map[string][]string) + for tk, entries := range c.entries { + for key, entry := range entries { + if !entry.Expired() { + continue + } + if _, ok := expiredKeys[tk]; !ok { + expiredKeys[tk] = make([]string, 0, len(entries)) + } + expiredKeys[tk] = append(expiredKeys[tk], key) + } + } + for tk, keys := range expiredKeys { + for _, key := range keys { + delete(c.entries[tk], key) + } + if len(c.entries[tk]) == 0 { + delete(c.entries, tk) + } + } +} diff --git a/internal/cache/utils.go b/internal/cache/utils.go new file mode 100644 index 000000000..82b7cf857 --- /dev/null +++ b/internal/cache/utils.go @@ -0,0 +1,24 @@ +package cache + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/cron" + log "github.com/sirupsen/logrus" +) + +var ( + cacheGcCron *cron.Cron + gcFuncs []func() +) + +func init() { + // TODO Move to bootstrap + cacheGcCron = cron.NewCron(time.Hour) + cacheGcCron.Do(func() { + log.Infof("Start cache GC") + for _, f := range gcFuncs { + f() + } + }) +} diff --git a/internal/conf/config.go b/internal/conf/config.go index af198e916..f347380d8 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -35,6 +35,7 @@ type Scheme struct { UnixFile string `json:"unix_file" env:"UNIX_FILE"` UnixFilePerm string `json:"unix_file_perm" env:"UNIX_FILE_PERM"` EnableH2c bool `json:"enable_h2c" env:"ENABLE_H2C"` + EnableH3 bool `json:"enable_h3" env:"ENABLE_H3"` } type LogConfig struct { @@ -130,6 +131,7 @@ type Config struct { FTP FTP `json:"ftp" envPrefix:"FTP_"` SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` LastLaunchedVersion string `json:"last_launched_version"` + ProxyAddress string `json:"proxy_address" env:"PROXY_ADDRESS"` } func DefaultConfig(dataDir string) *Config { @@ -180,7 +182,7 @@ func DefaultConfig(dataDir string) *Config { MmapThreshold: 4, MaxConnections: 0, MaxConcurrency: 64, - TlsInsecureSkipVerify: true, + TlsInsecureSkipVerify: false, Tasks: TasksConfig{ Download: TaskConfig{ Workers: 5, @@ -243,5 +245,6 @@ func DefaultConfig(dataDir string) *Config { Listen: ":5222", }, LastLaunchedVersion: "", + ProxyAddress: "", } } diff --git a/internal/conf/const.go b/internal/conf/const.go index 5543f6083..b99d8849c 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -24,17 +24,21 @@ const ( HideStorageDetailsInManagePage = "hide_storage_details_in_manage_page" // preview - TextTypes = "text_types" - AudioTypes = "audio_types" - VideoTypes = "video_types" - ImageTypes = "image_types" - ProxyTypes = "proxy_types" - ProxyIgnoreHeaders = "proxy_ignore_headers" - AudioAutoplay = "audio_autoplay" - VideoAutoplay = "video_autoplay" - PreviewArchivesByDefault = "preview_archives_by_default" - ReadMeAutoRender = "readme_autorender" - FilterReadMeScripts = "filter_readme_scripts" + TextTypes = "text_types" + AudioTypes = "audio_types" + VideoTypes = "video_types" + ImageTypes = "image_types" + ProxyTypes = "proxy_types" + ProxyIgnoreHeaders = "proxy_ignore_headers" + AudioAutoplay = "audio_autoplay" + VideoAutoplay = "video_autoplay" + PreviewDownloadByDefault = "preview_download_by_default" + PreviewArchivesByDefault = "preview_archives_by_default" + SharePreviewDownloadByDefault = "share_preview_download_by_default" + SharePreviewArchivesByDefault = "share_preview_archives_by_default" + ReadMeAutoRender = "readme_autorender" + FilterReadMeScripts = "filter_readme_scripts" + NonEFSZipEncoding = "non_efs_zip_encoding" // global HideFiles = "hide_files" @@ -52,6 +56,9 @@ const ( ShareArchivePreview = "share_archive_preview" ShareForceProxy = "share_force_proxy" ShareSummaryContent = "share_summary_content" + HandleHookAfterWriting = "handle_hook_after_writing" + HandleHookRateLimit = "handle_hook_rate_limit" + IgnoreSystemFiles = "ignore_system_files" // index SearchIndex = "search_index" @@ -70,6 +77,9 @@ const ( // 115 Pan115TempDir = "115_temp_dir" + // 123 + Pan123TempDir = "123_temp_dir" + // 115_open Pan115OpenTempDir = "115_open_temp_dir" @@ -108,6 +118,7 @@ const ( // ldap LdapLoginEnabled = "ldap_login_enabled" LdapServer = "ldap_server" + LdapSkipTlsVerify = "ldap_skip_tls_verify" LdapManagerDN = "ldap_manager_dn" LdapManagerPassword = "ldap_manager_password" LdapUserSearchBase = "ldap_user_search_base" @@ -125,6 +136,10 @@ const ( QbittorrentUrl = "qbittorrent_url" QbittorrentSeedtime = "qbittorrent_seedtime" + // 123 open offline download + Pan123OpenOfflineDownloadCallbackUrl = "123_open_callback_url" + Pan123OpenTempDir = "123_open_temp_dir" + // ftp FTPPublicHost = "ftp_public_host" FTPPasvPortMap = "ftp_pasv_port_map" @@ -159,7 +174,7 @@ const ( ) // ContextKey is the type of context keys. -type ContextKey int +type ContextKey int8 const ( _ ContextKey = iota @@ -175,4 +190,5 @@ const ( UserAgentKey PathKey SharingIDKey + SkipHookKey ) diff --git a/internal/conf/var.go b/internal/conf/var.go index de23b5c60..972f69997 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -3,6 +3,7 @@ package conf import ( "net/url" "regexp" + "sync" ) var ( @@ -14,8 +15,9 @@ var ( ) var ( - Conf *Config - URL *url.URL + Conf *Config + URL *url.URL + ConfigPath string ) var SlicesMap = make(map[string][]string) @@ -23,8 +25,6 @@ var FilenameCharMap = make(map[string]string) var PrivacyReg []*regexp.Regexp var ( - // StoragesLoaded loaded success if empty - StoragesLoaded = false // 单个Buffer最大限制 MaxBufferLimit = 16 * 1024 * 1024 // 超过该阈值的Buffer将使用 mmap 分配，可主动释放内存 @@ -35,3 +35,39 @@ var ( ManageHtml string IndexHtml string ) + +var ( + // StoragesLoaded loaded success if empty + StoragesLoaded = false + storagesLoadMu sync.RWMutex + storagesLoadSignal chan struct{} = make(chan struct{}) +) + +func StoragesLoadSignal() <-chan struct{} { + storagesLoadMu.RLock() + ch := storagesLoadSignal + storagesLoadMu.RUnlock() + return ch +} +func SendStoragesLoadedSignal() { + storagesLoadMu.Lock() + select { + case <-storagesLoadSignal: + // already closed + default: + StoragesLoaded = true + close(storagesLoadSignal) + } + storagesLoadMu.Unlock() +} +func ResetStoragesLoadSignal() { + storagesLoadMu.Lock() + select { + case <-storagesLoadSignal: + StoragesLoaded = false + storagesLoadSignal = make(chan struct{}) + default: + // not closed -> nothing to do + } + storagesLoadMu.Unlock() +} diff --git a/internal/db/sharing.go b/internal/db/sharing.go index 3748796b0..8670b15f3 100644 --- a/internal/db/sharing.go +++ b/internal/db/sharing.go @@ -38,18 +38,26 @@ func GetSharingsByCreatorId(creator uint, pageIndex, pageSize int) (sharings []m } func CreateSharing(s *model.SharingDB) (string, error) { - id := random.String(8) - for len(id) < 12 { - old := model.SharingDB{ - ID: id, + if s.ID == "" { + id := random.String(8) + for len(id) < 12 { + old := model.SharingDB{ + ID: id, + } + if err := db.Where(old).First(&old).Error; err != nil { + s.ID = id + return id, errors.WithStack(db.Create(s).Error) + } + id += random.String(1) } - if err := db.Where(old).First(&old).Error; err != nil { - s.ID = id - return id, errors.WithStack(db.Create(s).Error) + return "", errors.New("failed find valid id") + } else { + query := model.SharingDB{ID: s.ID} + if err := db.Where(query).First(&query).Error; err == nil { + return "", errors.New("sharing already exist") } - id += random.String(1) + return s.ID, errors.WithStack(db.Create(s).Error) } - return "", errors.New("failed find valid id") } func UpdateSharing(s *model.SharingDB) error { @@ -60,3 +68,7 @@ func DeleteSharingById(id string) error { s := model.SharingDB{ID: id} return errors.WithStack(db.Where(s).Delete(&s).Error) } + +func DeleteSharingsByCreatorId(creatorId uint) error { + return errors.WithStack(db.Where("creator_id = ?", creatorId).Delete(&model.SharingDB{}).Error) +} diff --git a/internal/db/tasks.go b/internal/db/tasks.go index dcb9dfeab..36054898b 100644 --- a/internal/db/tasks.go +++ b/internal/db/tasks.go @@ -1,6 +1,7 @@ package db import ( + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" ) @@ -30,6 +31,7 @@ func GetTaskDataFunc(type_s string, enabled bool) func() ([]byte, error) { return nil } return func() ([]byte, error) { + <-conf.StoragesLoadSignal() return []byte(task.PersistData), nil } } diff --git a/internal/driver/config.go b/internal/driver/config.go index bec6d47bf..2063975ba 100644 --- a/internal/driver/config.go +++ b/internal/driver/config.go @@ -3,11 +3,9 @@ package driver type Config struct { Name string `json:"name"` LocalSort bool `json:"local_sort"` - // if the driver returns Link with MFile, this should be set to true - OnlyLinkMFile bool `json:"only_local"` - OnlyProxy bool `json:"only_proxy"` - NoCache bool `json:"no_cache"` - NoUpload bool `json:"no_upload"` + OnlyProxy bool `json:"only_proxy"` + NoCache bool `json:"no_cache"` + NoUpload bool `json:"no_upload"` // if need get message from user, such as validate code NeedMs bool `json:"need_ms"` DefaultRoot string `json:"default_root"` @@ -19,8 +17,32 @@ type Config struct { ProxyRangeOption bool `json:"-"` // if the driver returns Link without URL, this should be set to true NoLinkURL bool `json:"-"` + // Link cache behaviour: + // - LinkCacheAuto: let driver decide per-path (implement driver.LinkCacheModeResolver) + // - LinkCacheNone: no extra info added to cache key (default) + // - flags (OR-able) can add more attributes to cache key (IP, UA, ...) + LinkCacheMode `json:"-"` + // if the driver only store indices of files (e.g. UrlTree) + OnlyIndices bool `json:"only_indices"` + // prefer proxy download even if direct link is available + PreferProxy bool `json:"prefer_proxy"` } +type LinkCacheMode int8 + +const ( + LinkCacheAuto LinkCacheMode = -1 // Let the driver decide per-path (use driver.LinkCacheModeResolver) + LinkCacheNone LinkCacheMode = 0 // No extra info added to cache key (default) +) + +const ( + LinkCacheIP LinkCacheMode = 1 << iota // include client IP in cache key + LinkCacheUA // include User-Agent in cache key +) func (c Config) MustProxy() bool { - return c.OnlyProxy || c.OnlyLinkMFile || c.NoLinkURL + return c.OnlyProxy || c.NoLinkURL +} + +func (c Config) DefaultProxy() bool { + return c.PreferProxy } diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 4a8e88255..373bb5653 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -47,11 +47,6 @@ type Getter interface { Get(ctx context.Context, path string) (model.Obj, error) } -type GetObjInfo interface { - // GetObjInfo get file info by path - GetObjInfo(ctx context.Context, path string) (model.Obj, error) -} - //type Writer interface { // Mkdir // Move @@ -115,15 +110,6 @@ type PutURL interface { PutURL(ctx context.Context, dstDir model.Obj, name, url string) error } -//type WriteResult interface { -// MkdirResult -// MoveResult -// RenameResult -// CopyResult -// PutResult -// Remove -//} - type MkdirResult interface { MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) } @@ -218,3 +204,17 @@ type WithDetails interface { type Reference interface { InitReference(storage Driver) error } + +type LinkCacheModeResolver interface { + // ResolveLinkCacheMode returns the LinkCacheMode for the given path. + ResolveLinkCacheMode(path string) LinkCacheMode +} + +type DirectUploader interface { + // GetDirectUploadTools returns available frontend-direct upload tools + GetDirectUploadTools() []string + // GetDirectUploadInfo returns the information needed for direct upload from client to storage + // actualPath is the path relative to the storage root (after removing mount path prefix) + // return errs.NotImplement if the driver does not support the given direct upload tool + GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error) +} diff --git a/internal/driver/utils.go b/internal/driver/utils.go index 9e4b6b015..41d9c6ce3 100644 --- a/internal/driver/utils.go +++ b/internal/driver/utils.go @@ -20,7 +20,7 @@ func (p *Progress) Write(b []byte) (n int, err error) { n = len(b) p.Done += int64(n) p.up(float64(p.Done) / float64(p.Total) * 100) - return + return n, err } func NewProgress(total int64, up UpdateProgress) *Progress { diff --git a/internal/errs/errors.go b/internal/errs/errors.go index fa599124b..fdf7f2189 100644 --- a/internal/errs/errors.go +++ b/internal/errs/errors.go @@ -12,13 +12,12 @@ var ( NotSupport = errors.New("not support") RelativePath = errors.New("using relative path is not allowed") - MoveBetweenTwoStorages = errors.New("can't move files between two storages, try to copy") - UploadNotSupported = errors.New("upload not supported") - - MetaNotFound = errors.New("meta not found") - StorageNotFound = errors.New("storage not found") - StreamIncomplete = errors.New("upload/download stream incomplete, possible network issue") - StreamPeekFail = errors.New("StreamPeekFail") + UploadNotSupported = errors.New("upload not supported") + MetaNotFound = errors.New("meta not found") + StorageNotFound = errors.New("storage not found") + StorageNotInit = errors.New("storage not init") + StreamIncomplete = errors.New("upload/download stream incomplete, possible network issue") + StreamPeekFail = errors.New("StreamPeekFail") UnknownArchiveFormat = errors.New("unknown archive format") WrongArchivePassword = errors.New("wrong archive password") diff --git a/internal/errs/object.go b/internal/errs/object.go index 00e8232ff..ab78f314a 100644 --- a/internal/errs/object.go +++ b/internal/errs/object.go @@ -7,9 +7,11 @@ import ( ) var ( - ObjectNotFound = errors.New("object not found") - NotFolder = errors.New("not a folder") - NotFile = errors.New("not a file") + ObjectNotFound = errors.New("object not found") + ObjectAlreadyExists = errors.New("object already exists") + NotFolder = errors.New("not a folder") + NotFile = errors.New("not a file") + IgnoredSystemFile = errors.New("system file upload ignored") ) func IsObjectNotFound(err error) bool { diff --git a/internal/errs/unwrap.go b/internal/errs/unwrap.go index 1cec96519..f34953c76 100644 --- a/internal/errs/unwrap.go +++ b/internal/errs/unwrap.go @@ -1,12 +1,11 @@ package errs -import "errors" - func UnwrapOrSelf(err error) error { - // errors.Unwrap has no fallback mechanism - unwrapped := errors.Unwrap(err) - if unwrapped == nil { + u, ok := err.(interface { + Unwrap() error + }) + if !ok { return err } - return unwrapped + return u.Unwrap() } diff --git a/internal/fs/archive.go b/internal/fs/archive.go index e1e4c4486..784ba587a 100644 --- a/internal/fs/archive.go +++ b/internal/fs/archive.go @@ -38,8 +38,17 @@ func (t *ArchiveDownloadTask) GetName() string { } func (t *ArchiveDownloadTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err + if t.SrcStorage == nil { + if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil { + t.SrcStorage = srcStorage + } else { + return err + } + if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil { + t.DstStorage = dstStorage + } else { + return err + } } t.ClearEndTime() t.SetStartTime(time.Now()) @@ -113,6 +122,7 @@ func (t *ArchiveDownloadTask) RunWithoutPushUploadTask() (*ArchiveContentUploadT DstActualPath: t.DstActualPath, dstStorage: t.DstStorage, DstStorageMp: t.DstStorageMp, + overwrite: t.Overwrite, } return uploadTask, nil } @@ -130,6 +140,7 @@ type ArchiveContentUploadTask struct { DstStorageMp string finalized bool groupID string + overwrite bool } func (t *ArchiveContentUploadTask) GetName() string { @@ -141,24 +152,22 @@ func (t *ArchiveContentUploadTask) GetStatus() string { } func (t *ArchiveContentUploadTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err - } t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() return t.RunWithNextTaskCallback(func(nextTsk *ArchiveContentUploadTask) error { + task_group.TransferCoordinator.AddTask(t.groupID, nil) ArchiveContentUploadTaskManager.Add(nextTsk) return nil }) } func (t *ArchiveContentUploadTask) OnSucceeded() { - task_group.TransferCoordinator.Done(t.groupID, true) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, true) } func (t *ArchiveContentUploadTask) OnFailed() { - task_group.TransferCoordinator.Done(t.groupID, false) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, false) } func (t *ArchiveContentUploadTask) SetRetry(retry int, maxRetry int) { @@ -190,8 +199,8 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch if err != nil { return err } - if !t.InPlace && len(t.groupID) > 0 { - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToRefresh(nextDstActualPath)) + if !t.InPlace { + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(nextDstActualPath)) } var es error for _, entry := range entries { @@ -205,9 +214,6 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch es = stderrors.Join(es, err) continue } - if len(t.groupID) > 0 { - task_group.TransferCoordinator.AddTask(t.groupID, nil) - } err = f(&ArchiveContentUploadTask{ TaskExtension: task.TaskExtension{ Creator: t.Creator, @@ -220,6 +226,7 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch dstStorage: t.dstStorage, DstStorageMp: t.DstStorageMp, groupID: t.groupID, + overwrite: t.overwrite, }) if err != nil { es = stderrors.Join(es, err) @@ -229,6 +236,12 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch return es } } else { + if !t.overwrite { + dstPath := stdpath.Join(t.DstActualPath, t.ObjName) + if res, _ := op.Get(t.Ctx(), t.dstStorage, dstPath); res != nil { + return errs.ObjectAlreadyExists + } + } file, err := os.Open(t.FilePath) if err != nil { return err @@ -246,7 +259,7 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch } fs.Closers.Add(file) t.status = "uploading" - err = op.Put(t.Ctx(), t.dstStorage, t.DstActualPath, fs, t.SetProgress, true) + err = op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.dstStorage, t.DstActualPath, fs, t.SetProgress) if err != nil { return err } @@ -384,14 +397,22 @@ func archiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args } defer uploadTask.deleteSrcFile() var callback func(t *ArchiveContentUploadTask) error + var hasSuccess bool callback = func(t *ArchiveContentUploadTask) error { t.Base.SetCtx(ctx) e := t.RunWithNextTaskCallback(callback) + if e == nil { + hasSuccess = true + } t.deleteSrcFile() return e } uploadTask.Base.SetCtx(ctx) - return nil, uploadTask.RunWithNextTaskCallback(callback) + uploadTask.groupID = stdpath.Join(uploadTask.DstStorageMp, uploadTask.DstActualPath) + task_group.TransferCoordinator.AddTask(uploadTask.groupID, nil) + err = uploadTask.RunWithNextTaskCallback(callback) + task_group.TransferCoordinator.Done(context.WithoutCancel(ctx), uploadTask.groupID, hasSuccess) + return nil, err } else { tsk.Creator, _ = ctx.Value(conf.UserKey).(*model.User) tsk.ApiUrl = common.GetApiUrl(ctx) diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index dbdd28358..e78fc9be8 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -22,16 +22,22 @@ import ( type taskType uint8 func (t taskType) String() string { - if t == 0 { + switch t { + case copy: return "copy" - } else { + case move: return "move" + case merge: + return "merge" + default: + return "unknown" } } const ( copy taskType = iota move + merge ) type FileTransferTask struct { @@ -45,16 +51,25 @@ func (t *FileTransferTask) GetName() string { } func (t *FileTransferTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err + if t.SrcStorage == nil { + if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil { + t.SrcStorage = srcStorage + } else { + return err + } + if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil { + t.DstStorage = dstStorage + } else { + return err + } } + t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() return t.RunWithNextTaskCallback(func(nextTask *FileTransferTask) error { - nextTask.groupID = t.groupID task_group.TransferCoordinator.AddTask(t.groupID, nil) - if t.TaskType == copy { + if t.TaskType == copy || t.TaskType == merge { CopyTaskManager.Add(nextTask) } else { MoveTaskManager.Add(nextTask) @@ -64,15 +79,15 @@ func (t *FileTransferTask) Run() error { } func (t *FileTransferTask) OnSucceeded() { - task_group.TransferCoordinator.Done(t.groupID, true) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, true) } func (t *FileTransferTask) OnFailed() { - task_group.TransferCoordinator.Done(t.groupID, false) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, false) } func (t *FileTransferTask) SetRetry(retry int, maxRetry int) { - t.TaskExtension.SetRetry(retry, maxRetry) + t.TaskData.SetRetry(retry, maxRetry) if retry == 0 && (len(t.groupID) == 0 || // 重启恢复 (t.GetErr() == nil && t.GetState() != tache.StatePending)) { // 手动重试 @@ -85,7 +100,7 @@ func (t *FileTransferTask) SetRetry(retry int, maxRetry int) { } } -func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { +func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { return nil, errors.WithMessage(err, "failed get src storage") @@ -96,13 +111,16 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str } if srcStorage.GetStorage() == dstStorage.GetStorage() { - if taskType == copy { - err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) + if utils.IsBool(skipHook...) { + ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) + } + if taskType == copy || taskType == merge { + err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath) if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { return nil, err } } else { - err = op.Move(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) + err = op.Move(ctx, srcStorage, srcObjActualPath, dstDirActualPath) if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { return nil, err } @@ -122,6 +140,8 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str TaskType: taskType, } + t.groupID = stdpath.Join(t.DstStorageMp, t.DstActualPath) + task_group.TransferCoordinator.AddTask(t.groupID, nil) if ctx.Value(conf.NoTaskKey) != nil { var callback func(nextTask *FileTransferTask) error hasSuccess := false @@ -135,24 +155,22 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str } t.Base.SetCtx(ctx) err = t.RunWithNextTaskCallback(callback) - if hasSuccess || err == nil { - if taskType == move { - task_group.RefreshAndRemove(dstDirPath, task_group.SrcPathToRemove(srcObjPath)) - } else { - op.DeleteCache(t.DstStorage, dstDirActualPath) - } + if err == nil { + hasSuccess = true } + if taskType == move { + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.SrcPathToRemove(srcObjPath)) + } + task_group.TransferCoordinator.Done(context.WithoutCancel(ctx), t.groupID, hasSuccess) return nil, err } t.Creator, _ = ctx.Value(conf.UserKey).(*model.User) t.ApiUrl = common.GetApiUrl(ctx) - t.groupID = dstDirPath - if taskType == copy { - task_group.TransferCoordinator.AddTask(dstDirPath, nil) + if taskType == copy || taskType == merge { CopyTaskManager.Add(t) } else { - task_group.TransferCoordinator.AddTask(dstDirPath, task_group.SrcPathToRemove(srcObjPath)) + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.SrcPathToRemove(srcObjPath)) MoveTaskManager.Add(t) } return t, nil @@ -164,6 +182,7 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", t.SrcActualPath) } + if srcObj.IsDir() { t.Status = "src object is dir, listing objs" objs, err := op.List(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.ListArgs{}) @@ -171,17 +190,36 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcActualPath) } dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) - if t.TaskType == copy { - if t.Ctx().Value(conf.NoTaskKey) != nil { - defer op.DeleteCache(t.DstStorage, dstActualPath) - } else { - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToRefresh(dstActualPath)) + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) + + existedObjs := make(map[string]bool) + if t.TaskType == merge { + dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) + if err != nil && !errors.Is(err, errs.ObjectNotFound) { + // 目标文件夹不存在的情况不是错误，会在之后新建文件夹 + // 这种情况显然不需要统计existedObjs，dstObjs保持为nil，下面这个for将不会执行 + return errors.WithMessagef(err, "failed list dst [%s] objs", dstActualPath) + } + for _, obj := range dstObjs { + if err := t.Ctx().Err(); err != nil { + return err + } + if !obj.IsDir() { + existedObjs[obj.GetName()] = true + } } } + for _, obj := range objs { - if utils.IsCanceled(t.Ctx()) { - return nil + if err := t.Ctx().Err(); err != nil { + return err + } + + if t.TaskType == merge && !obj.IsDir() && existedObjs[obj.GetName()] { + // skip existed file + continue } + err = f(&FileTransferTask{ TaskType: t.TaskType, TaskData: TaskData{ @@ -196,6 +234,7 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer SrcStorageMp: t.SrcStorageMp, DstStorageMp: t.DstStorageMp, }, + groupID: t.groupID, }) if err != nil { return err @@ -205,7 +244,8 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return nil } - link, _, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) + t.Status = "getting src object link" + link, srcObj, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) if err != nil { return errors.WithMessagef(err, "failed get [%s] link", t.SrcActualPath) } @@ -220,7 +260,7 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer } t.SetTotalBytes(ss.GetSize()) t.Status = "uploading" - return op.Put(t.Ctx(), t.DstStorage, t.DstActualPath, ss, t.SetProgress, true) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } var ( diff --git a/internal/fs/fs.go b/internal/fs/fs.go index ca199ed40..3d459c6be 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -60,32 +60,40 @@ func Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, m return res, file, nil } -func MakeDir(ctx context.Context, path string, lazyCache ...bool) error { - err := makeDir(ctx, path, lazyCache...) +func MakeDir(ctx context.Context, path string) error { + err := makeDir(ctx, path) if err != nil { log.Errorf("failed make dir %s: %+v", path, err) } return err } -func Move(ctx context.Context, srcPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { - req, err := transfer(ctx, move, srcPath, dstDirPath, lazyCache...) +func Move(ctx context.Context, srcPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { + req, err := transfer(ctx, move, srcPath, dstDirPath, skipHook...) if err != nil { log.Errorf("failed move %s to %s: %+v", srcPath, dstDirPath, err) } return req, err } -func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { - res, err := transfer(ctx, copy, srcObjPath, dstDirPath, lazyCache...) +func Copy(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { + res, err := transfer(ctx, copy, srcObjPath, dstDirPath, skipHook...) if err != nil { log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) } return res, err } -func Rename(ctx context.Context, srcPath, dstName string, lazyCache ...bool) error { - err := rename(ctx, srcPath, dstName, lazyCache...) +func Merge(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { + res, err := transfer(ctx, merge, srcObjPath, dstDirPath, skipHook...) + if err != nil { + log.Errorf("failed merge %s to %s: %+v", srcObjPath, dstDirPath, err) + } + return res, err +} + +func Rename(ctx context.Context, srcPath, dstName string, skipHook ...bool) error { + err := rename(ctx, srcPath, dstName, skipHook...) if err != nil { log.Errorf("failed rename %s to %s: %+v", srcPath, dstName, err) } @@ -100,8 +108,8 @@ func Remove(ctx context.Context, path string) error { return err } -func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, lazyCache ...bool) error { - err := putDirectly(ctx, dstDirPath, file, lazyCache...) +func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, skipHook ...bool) error { + err := putDirectly(ctx, dstDirPath, file, skipHook...) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) } @@ -190,3 +198,11 @@ func PutURL(ctx context.Context, path, dstName, urlStr string) error { } return op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) } + +func GetDirectUploadInfo(ctx context.Context, tool, path, dstName string, fileSize int64) (any, error) { + info, err := getDirectUploadInfo(ctx, tool, path, dstName, fileSize) + if err != nil { + log.Errorf("failed get %s direct upload info for %s(%d bytes): %+v", path, dstName, fileSize, err) + } + return info, err +} diff --git a/internal/fs/get.go b/internal/fs/get.go index 4e91c5bde..8a920065e 100644 --- a/internal/fs/get.go +++ b/internal/fs/get.go @@ -3,7 +3,6 @@ package fs import ( "context" stdpath "path" - "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -15,9 +14,10 @@ func get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) { path = utils.FixAndCleanPath(path) // maybe a virtual file if path != "/" { - virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, stdpath.Dir(path), !args.WithStorageDetails) + dir, name := stdpath.Split(path) + virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, dir, !args.WithStorageDetails, false, name) for _, f := range virtualFiles { - if f.GetName() == stdpath.Base(path) { + if f.GetName() == name { return f, nil } } @@ -28,9 +28,8 @@ func get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) { if path == "/" { return &model.Object{ Name: "root", - Size: 0, - Modified: time.Time{}, IsFolder: true, + Mask: model.ReadOnly | model.Virtual, }, nil } return nil, errors.WithMessage(err, "failed get storage") diff --git a/internal/fs/list.go b/internal/fs/list.go index 5245795fc..1f92c7d46 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -15,7 +15,7 @@ import ( func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) { meta, _ := ctx.Value(conf.MetaKey).(*model.Meta) user, _ := ctx.Value(conf.UserKey).(*model.User) - virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, path, !args.WithStorageDetails) + virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, path, !args.WithStorageDetails, args.Refresh, "") storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil && len(virtualFiles) == 0 { return nil, errors.WithMessage(err, "failed get storage") diff --git a/internal/fs/other.go b/internal/fs/other.go index 8d16b0003..a23beb73b 100644 --- a/internal/fs/other.go +++ b/internal/fs/other.go @@ -3,27 +3,32 @@ package fs import ( "context" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/task" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) -func makeDir(ctx context.Context, path string, lazyCache ...bool) error { +func makeDir(ctx context.Context, path string) error { storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { return errors.WithMessage(err, "failed get storage") } - return op.MakeDir(ctx, storage, actualPath, lazyCache...) + return op.MakeDir(ctx, storage, actualPath) } -func rename(ctx context.Context, srcPath, dstName string, lazyCache ...bool) error { +func rename(ctx context.Context, srcPath, dstName string, skipHook ...bool) error { storage, srcActualPath, err := op.GetStorageAndActualPath(srcPath) if err != nil { return errors.WithMessage(err, "failed get storage") } - return op.Rename(ctx, storage, srcActualPath, dstName, lazyCache...) + if utils.IsBool(skipHook...) { + ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) + } + return op.Rename(ctx, storage, srcActualPath, dstName) } func remove(ctx context.Context, path string) error { diff --git a/internal/fs/put.go b/internal/fs/put.go index 881330b04..be829ae47 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -6,6 +6,7 @@ import ( stdpath "path" "time" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -38,15 +39,15 @@ func (t *UploadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() - return op.Put(t.Ctx(), t.storage, t.dstDirActualPath, t.file, t.SetProgress, true) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.storage, t.dstDirActualPath, t.file, t.SetProgress) } func (t *UploadTask) OnSucceeded() { - task_group.TransferCoordinator.Done(stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), true) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), true) } func (t *UploadTask) OnFailed() { - task_group.TransferCoordinator.Done(stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), false) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), false) } func (t *UploadTask) SetRetry(retry int, maxRetry int) { @@ -87,13 +88,13 @@ func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) file: file, } t.SetTotalBytes(file.GetSize()) - task_group.TransferCoordinator.AddTask(dstDirPath, nil) + task_group.TransferCoordinator.AddTask(stdpath.Join(storage.GetStorage().MountPath, dstDirActualPath), nil) UploadTaskManager.Add(t) return t, nil } // putDirect put the file and return after finish -func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, lazyCache ...bool) error { +func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, skipHook ...bool) error { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { _ = file.Close() @@ -103,5 +104,16 @@ func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer _ = file.Close() return errors.WithStack(errs.UploadNotSupported) } - return op.Put(ctx, storage, dstDirActualPath, file, nil, lazyCache...) + if utils.IsBool(skipHook...) { + ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) + } + return op.Put(ctx, storage, dstDirActualPath, file, nil) +} + +func getDirectUploadInfo(ctx context.Context, tool, dstDirPath, dstName string, fileSize int64) (any, error) { + storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return nil, errors.WithMessage(err, "failed get storage") + } + return op.GetDirectUploadInfo(ctx, tool, storage, dstDirActualPath, dstName, fileSize) } diff --git a/internal/model/args.go b/internal/model/args.go index 2ec95f149..073c94a63 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -15,6 +15,7 @@ type ListArgs struct { S3ShowPlaceholder bool Refresh bool WithStorageDetails bool + SkipHook bool } type LinkArgs struct { @@ -28,16 +29,17 @@ type Link struct { URL string `json:"url"` // most common way Header http.Header `json:"header"` // needed header (for url) RangeReader RangeReaderIF `json:"-"` // recommended way if can't use URL - MFile File `json:"-"` // best for local,smb... file system, which exposes MFile Expiration *time.Duration // local cache expire Duration //for accelerating request, use multi-thread downloading Concurrency int `json:"concurrency"` PartSize int `json:"part_size"` - ContentLength int64 `json:"-"` // 转码视频、缩略图 + ContentLength int64 `json:"content_length"` // 转码视频、缩略图 utils.SyncClosers `json:"-"` + // 如果SyncClosers中的资源被关闭后Link将不可用，则此值应为 true + RequireReference bool `json:"-"` } type OtherArgs struct { @@ -76,6 +78,7 @@ type ArchiveDecompressArgs struct { ArchiveInnerArgs CacheFull bool PutIntoNewDir bool + Overwrite bool } type SharingListArgs struct { diff --git a/internal/model/direct_upload.go b/internal/model/direct_upload.go new file mode 100644 index 000000000..89bbfeb5d --- /dev/null +++ b/internal/model/direct_upload.go @@ -0,0 +1,8 @@ +package model + +type HttpDirectUploadInfo struct { + UploadURL string `json:"upload_url"` // The URL to upload the file + ChunkSize int64 `json:"chunk_size"` // The chunk size for uploading, 0 means no chunking required + Headers map[string]string `json:"headers,omitempty"` // Optional headers to include in the upload request + Method string `json:"method,omitempty"` // HTTP method, default is PUT +} diff --git a/internal/model/file.go b/internal/model/file.go index 9fd20adc0..4ca7201e1 100644 --- a/internal/model/file.go +++ b/internal/model/file.go @@ -27,6 +27,9 @@ func (f *FileCloser) Close() error { return errors.Join(errs...) } +// FileRangeReader 是对 RangeReaderIF 的轻量包装，表明由 RangeReaderIF.RangeRead +// 返回的 io.ReadCloser 同时实现了 model.File（即支持 Read/ReadAt/Seek）。 +// 只有满足这些才需要使用 FileRangeReader，否则直接使用 RangeReaderIF 即可。 type FileRangeReader struct { RangeReaderIF } diff --git a/internal/model/obj.go b/internal/model/obj.go index b3bf5ebe5..ba7467f74 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -48,7 +48,6 @@ type FileStreamer interface { // for a non-seekable Stream, if Read is called, this function won't work. // caches the full Stream and writes it to writer (if provided, even if the stream is already cached). CacheFullAndWriter(up *UpdateProgress, writer io.Writer) (File, error) - SetTmpFile(file File) // if the Stream is not a File and is not cached, returns nil. GetFile() File } @@ -138,62 +137,55 @@ func WrapObjName(objs Obj) Obj { } func WrapObjsName(objs []Obj) { - for i := 0; i < len(objs); i++ { + for i := range objs { objs[i] = &ObjWrapName{Name: utils.MappingName(objs[i].GetName()), Obj: objs[i]} } } -func UnwrapObj(obj Obj) Obj { - if unwrap, ok := obj.(ObjUnwrap); ok { - obj = unwrap.Unwrap() +func UnwrapObjName(obj Obj) Obj { + if n, ok := obj.(*ObjWrapName); ok { + return n.Obj } return obj } func GetThumb(obj Obj) (thumb string, ok bool) { - if obj, ok := obj.(Thumb); ok { - return obj.Thumb(), true - } - if unwrap, ok := obj.(ObjUnwrap); ok { - return GetThumb(unwrap.Unwrap()) + for { + switch o := obj.(type) { + case Thumb: + return o.Thumb(), true + case ObjUnwrap: + obj = o.Unwrap() + default: + return + } } - return thumb, false } func GetUrl(obj Obj) (url string, ok bool) { - if obj, ok := obj.(URL); ok { - return obj.URL(), true - } - if unwrap, ok := obj.(ObjUnwrap); ok { - return GetUrl(unwrap.Unwrap()) + for { + switch o := obj.(type) { + case URL: + return o.URL(), true + case ObjUnwrap: + obj = o.Unwrap() + default: + return + } } - return url, false } func GetProvider(obj Obj) (string, bool) { - if obj, ok := obj.(ObjWithProvider); ok { - return obj.GetProvider(), true - } - if unwrap, ok := obj.(ObjUnwrap); ok { - return GetProvider(unwrap.Unwrap()) - } - return "unknown", false -} - -func GetRawObject(obj Obj) *Object { - switch v := obj.(type) { - case *ObjThumbURL: - return &v.Object - case *ObjThumb: - return &v.Object - case *ObjectURL: - return &v.Object - case *ObjectProvider: - return &v.Object - case *Object: - return v + for { + switch o := obj.(type) { + case ObjWithProvider: + return o.GetProvider(), true + case ObjUnwrap: + obj = o.Unwrap() + default: + return "unknown", false + } } - return nil } // Merge @@ -243,3 +235,52 @@ func (om *ObjMerge) InitHideReg(hides string) { func (om *ObjMerge) Reset() { om.set.Clear() } + +type ObjMask uint8 + +func (m ObjMask) GetObjMask() ObjMask { + return m +} + +const ( + Virtual ObjMask = 1 << iota + NoRename + NoRemove + NoMove + NoCopy + NoWrite + Temp +) +const ( + Locked = NoRename | NoRemove | NoMove + ReadOnly = Locked | NoWrite // NoRename | NoDelete | NoMove | NoWrite +) + +type ObjWrapMask struct { + Obj + Mask ObjMask +} + +func (m *ObjWrapMask) Unwrap() Obj { + return m.Obj +} +func (m *ObjWrapMask) GetObjMask() ObjMask { + return m.Mask +} + +func GetObjMask(obj Obj) ObjMask { + for { + switch o := obj.(type) { + case interface{ GetObjMask() ObjMask }: + return o.GetObjMask() + case ObjUnwrap: + obj = o.Unwrap() + default: + return 0 + } + } +} + +func ObjHasMask(obj Obj, mask ObjMask) bool { + return GetObjMask(obj)&mask != 0 +} diff --git a/internal/model/object.go b/internal/model/object.go index 8e5cdf047..b6cb0d7f7 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -28,6 +28,7 @@ type Object struct { Ctime time.Time // file create time IsFolder bool HashInfo utils.HashInfo + Mask ObjMask } func (o *Object) GetName() string { @@ -68,6 +69,10 @@ func (o *Object) GetHash() utils.HashInfo { return o.HashInfo } +func (o *Object) GetObjMask() ObjMask { + return o.Mask +} + type Thumbnail struct { Thumbnail string } diff --git a/internal/model/sharing.go b/internal/model/sharing.go index dfbc81b02..c5dd95e9e 100644 --- a/internal/model/sharing.go +++ b/internal/model/sharing.go @@ -33,7 +33,7 @@ func (s *Sharing) Valid() bool { if len(s.Files) == 0 { return false } - if !s.Creator.CanShare() { + if s.Creator == nil || !s.Creator.CanShare() { return false } if s.Expires != nil && !s.Expires.IsZero() && s.Expires.Before(time.Now()) { diff --git a/internal/model/storage.go b/internal/model/storage.go index 8c754e0fe..a6b4745a6 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -1,22 +1,24 @@ package model import ( + "encoding/json" "time" ) type Storage struct { - ID uint `json:"id" gorm:"primaryKey"` // unique key - MountPath string `json:"mount_path" gorm:"unique" binding:"required"` // must be standardized - Order int `json:"order"` // use to sort - Driver string `json:"driver"` // driver used - CacheExpiration int `json:"cache_expiration"` // cache expire time - Status string `json:"status"` - Addition string `json:"addition" gorm:"type:text"` // Additional information, defined in the corresponding driver - Remark string `json:"remark"` - Modified time.Time `json:"modified"` - Disabled bool `json:"disabled"` // if disabled - DisableIndex bool `json:"disable_index"` - EnableSign bool `json:"enable_sign"` + ID uint `json:"id" gorm:"primaryKey"` // unique key + MountPath string `json:"mount_path" gorm:"unique" binding:"required"` // must be standardized + Order int `json:"order"` // use to sort + Driver string `json:"driver"` // driver used + CacheExpiration int `json:"cache_expiration"` // cache expire time + CustomCachePolicies string `json:"custom_cache_policies" gorm:"type:text"` + Status string `json:"status"` + Addition string `json:"addition" gorm:"type:text"` // Additional information, defined in the corresponding driver + Remark string `json:"remark"` + Modified time.Time `json:"modified"` + Disabled bool `json:"disabled"` // if disabled + DisableIndex bool `json:"disable_index"` + EnableSign bool `json:"enable_sign"` Sort Proxy } @@ -32,7 +34,7 @@ type Proxy struct { WebdavPolicy string `json:"webdav_policy"` ProxyRange bool `json:"proxy_range"` DownProxyURL string `json:"down_proxy_url"` - //Disable sign for DownProxyURL + // Disable sign for DownProxyURL DisableProxySign bool `json:"disable_proxy_sign"` } @@ -57,33 +59,44 @@ func (p Proxy) WebdavProxyURL() bool { } type DiskUsage struct { - TotalSpace uint64 `json:"total_space"` - FreeSpace uint64 `json:"free_space"` + TotalSpace int64 + UsedSpace int64 } -type StorageDetails struct { - DiskUsage +func (d DiskUsage) FreeSpace() int64 { + return d.TotalSpace - d.UsedSpace } -type StorageDetailsWithName struct { - *StorageDetails - DriverName string `json:"driver_name"` +func (d DiskUsage) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "total_space": d.TotalSpace, + "used_space": d.UsedSpace, + "free_space": d.FreeSpace(), + }) +} + +type StorageDetails struct { + DiskUsage } type ObjWithStorageDetails interface { - GetStorageDetails() *StorageDetailsWithName + GetStorageDetails() *StorageDetails } type ObjStorageDetails struct { Obj - StorageDetailsWithName + *StorageDetails +} + +func (o *ObjStorageDetails) Unwrap() Obj { + return o.Obj } -func (o ObjStorageDetails) GetStorageDetails() *StorageDetailsWithName { - return &o.StorageDetailsWithName +func (o *ObjStorageDetails) GetStorageDetails() *StorageDetails { + return o.StorageDetails } -func GetStorageDetails(obj Obj) (*StorageDetailsWithName, bool) { +func GetStorageDetails(obj Obj) (*StorageDetails, bool) { if obj, ok := obj.(ObjWithStorageDetails); ok { return obj.GetStorageDetails(), true } diff --git a/internal/model/user.go b/internal/model/user.go index ae2d7d58c..3bad4ebb9 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -20,7 +20,15 @@ const ( ADMIN ) -const StaticHashSalt = "https://github.com/alist-org/alist" +const ( + StaticHashSalt = "https://github.com/alist-org/alist" + + InvalidUsernameOrPassword = "Invalid username or password" + Invalid2FACode = "Invalid 2FA code" + TooManyAttempts = "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later." + GuestCannotUpdateProfile = "Guest user can not update profile" + GuestCannotGenerate2FA = "Guest user can not generate 2FA code" +) var LoginCache = cache.NewMemCache[int]() @@ -59,6 +67,7 @@ type User struct { OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform Authn string `gorm:"type:text" json:"-"` + AllowLdap bool `json:"allow_ldap" gorm:"default:true"` } func (u *User) IsGuest() bool { @@ -90,64 +99,124 @@ func (u *User) SetPassword(pwd string) *User { return u } +func CanSeeHides(permission int32) bool { + return permission&1 == 1 +} + func (u *User) CanSeeHides() bool { - return u.Permission&1 == 1 + return CanSeeHides(u.Permission) +} + +func CanAccessWithoutPassword(permission int32) bool { + return (permission>>1)&1 == 1 } func (u *User) CanAccessWithoutPassword() bool { - return (u.Permission>>1)&1 == 1 + return CanAccessWithoutPassword(u.Permission) +} + +func CanAddOfflineDownloadTasks(permission int32) bool { + return (permission>>2)&1 == 1 } func (u *User) CanAddOfflineDownloadTasks() bool { - return (u.Permission>>2)&1 == 1 + return CanAddOfflineDownloadTasks(u.Permission) +} + +func CanWrite(permission int32) bool { + return (permission>>3)&1 == 1 } func (u *User) CanWrite() bool { - return (u.Permission>>3)&1 == 1 + return CanWrite(u.Permission) +} + +func CanRename(permission int32) bool { + return (permission>>4)&1 == 1 } func (u *User) CanRename() bool { - return (u.Permission>>4)&1 == 1 + return CanRename(u.Permission) +} + +func CanMove(permission int32) bool { + return (permission>>5)&1 == 1 } func (u *User) CanMove() bool { - return (u.Permission>>5)&1 == 1 + return CanMove(u.Permission) +} + +func CanCopy(permission int32) bool { + return (permission>>6)&1 == 1 } func (u *User) CanCopy() bool { - return (u.Permission>>6)&1 == 1 + return CanCopy(u.Permission) +} + +func CanRemove(permission int32) bool { + return (permission>>7)&1 == 1 } func (u *User) CanRemove() bool { - return (u.Permission>>7)&1 == 1 + return CanRemove(u.Permission) +} + +func CanWebdavRead(permission int32) bool { + return (permission>>8)&1 == 1 } func (u *User) CanWebdavRead() bool { - return (u.Permission>>8)&1 == 1 + return CanWebdavRead(u.Permission) +} + +func CanWebdavManage(permission int32) bool { + return (permission>>9)&1 == 1 } func (u *User) CanWebdavManage() bool { - return (u.Permission>>9)&1 == 1 + return CanWebdavManage(u.Permission) +} + +func CanFTPAccess(permission int32) bool { + return (permission>>10)&1 == 1 } func (u *User) CanFTPAccess() bool { - return (u.Permission>>10)&1 == 1 + return CanFTPAccess(u.Permission) +} + +func CanFTPManage(permission int32) bool { + return (permission>>11)&1 == 1 } func (u *User) CanFTPManage() bool { - return (u.Permission>>11)&1 == 1 + return CanFTPManage(u.Permission) +} + +func CanReadArchives(permission int32) bool { + return (permission>>12)&1 == 1 } func (u *User) CanReadArchives() bool { - return (u.Permission>>12)&1 == 1 + return CanReadArchives(u.Permission) +} + +func CanDecompress(permission int32) bool { + return (permission>>13)&1 == 1 } func (u *User) CanDecompress() bool { - return (u.Permission>>13)&1 == 1 + return CanDecompress(u.Permission) +} + +func CanShare(permission int32) bool { + return (permission>>14)&1 == 1 } func (u *User) CanShare() bool { - return (u.Permission>>14)&1 == 1 + return CanShare(u.Permission) } func (u *User) JoinPath(reqPath string) (string, error) { diff --git a/internal/net/request.go b/internal/net/request.go index 1306bc549..8d380ea4e 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -12,6 +12,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/rclone/rclone/lib/mmap" @@ -403,7 +404,7 @@ var errInfiniteRetry = errors.New("infinite retry") func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { resp, err := d.cfg.HttpClient(d.ctx, params) if err != nil { - statusCode, ok := errors.Unwrap(err).(HttpStatusCodeError) + statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError) if !ok { return 0, err } diff --git a/internal/net/serve.go b/internal/net/serve.go index f8d2e3ca0..6a20460b1 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -283,11 +283,15 @@ func HttpClient() *http.Client { } func NewHttpClient() *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + } + + SetProxyIfConfigured(transport) + return &http.Client{ - Timeout: time.Hour * 48, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, - }, + Timeout: time.Hour * 48, + Transport: transport, } } diff --git a/internal/net/util.go b/internal/net/util.go index 40b5e145e..4bf2ba6cf 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -1,18 +1,20 @@ package net import ( - "fmt" "io" - "math" "mime/multipart" "net/http" "net/textproto" + "net/url" "strings" "time" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/rclone/rclone/lib/readers" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -305,39 +307,9 @@ func rangesMIMESize(ranges []http_range.Range, contentType string, contentSize i return encSize, nil } -// LimitedReadCloser wraps a io.ReadCloser and limits the number of bytes that can be read from it. -type LimitedReadCloser struct { - rc io.ReadCloser - remaining int -} - -func (l *LimitedReadCloser) Read(buf []byte) (int, error) { - if l.remaining <= 0 { - return 0, io.EOF - } - - if len(buf) > l.remaining { - buf = buf[0:l.remaining] - } - - n, err := l.rc.Read(buf) - l.remaining -= n - - return n, err -} - -func (l *LimitedReadCloser) Close() error { - return l.rc.Close() -} - // GetRangedHttpReader some http server doesn't support "Range" header, // so this function read readCloser with whole data, skip offset, then return ReaderCloser. func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.ReadCloser, error) { - var length_int int - if length > math.MaxInt { - return nil, fmt.Errorf("doesnot support length bigger than int32 max ") - } - length_int = int(length) if offset > 100*1024*1024 { log.Warnf("offset is more than 100MB, if loading data from internet, high-latency and wasting of bandwidth is expected") @@ -348,5 +320,25 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea } // return an io.ReadCloser that is limited to `length` bytes. - return &LimitedReadCloser{readCloser, length_int}, nil + return readers.NewLimitedReadCloser(readCloser, length), nil +} + +// SetProxyIfConfigured sets proxy for HTTP Transport if configured +func SetProxyIfConfigured(transport *http.Transport) { + // If proxy address is configured, override environment variable settings + if conf.Conf.ProxyAddress != "" { + if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + } +} + +// SetRestyProxyIfConfigured sets proxy for Resty client if configured +func SetRestyProxyIfConfigured(client *resty.Client) { + // If proxy address is configured, override environment variable settings + if conf.Conf.ProxyAddress != "" { + if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { + client.SetProxy(proxyURL.String()) + } + } } diff --git a/internal/offline_download/123/client.go b/internal/offline_download/123/client.go new file mode 100644 index 000000000..2c4f47048 --- /dev/null +++ b/internal/offline_download/123/client.go @@ -0,0 +1,142 @@ +package _123_pan + +import ( + "context" + "fmt" + "strconv" + + _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" +) + +type Pan123 struct{} + +func (*Pan123) Name() string { + return "123Pan" +} + +func (*Pan123) Items() []model.SettingItem { + return []model.SettingItem{ + {Key: conf.Pan123TempDir, Value: "", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (*Pan123) Run(_ *tool.DownloadTask) error { + return errs.NotSupport +} + +func (*Pan123) Init() (string, error) { + return "ok", nil +} + +func (*Pan123) IsReady() bool { + tempDir := setting.GetStr(conf.Pan123TempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*_123.Pan123); !ok { + return false + } + return true +} + +func (*Pan123) AddURL(args *tool.AddUrlArgs) (string, error) { + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver123, ok := storage.(*_123.Pan123) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") + } + ctx := context.Background() + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + taskID, err := driver123.OfflineDownload(ctx, args.Url, parentDir) + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + return strconv.FormatInt(taskID, 10), nil +} + +func (*Pan123) Remove(task *tool.DownloadTask) error { + taskID, err := strconv.ParseInt(task.GID, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse task ID: %s", task.GID) + } + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + driver123, ok := storage.(*_123.Pan123) + if !ok { + return fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") + } + return driver123.DeleteOfflineTasks(context.Background(), []int64{taskID}) +} + +func (*Pan123) Status(task *tool.DownloadTask) (*tool.Status, error) { + taskID, err := strconv.ParseInt(task.GID, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse task ID: %s", task.GID) + } + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver123, ok := storage.(*_123.Pan123) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") + } + + t, err := driver123.GetOfflineTask(context.Background(), taskID) + if err != nil { + return nil, err + } + + var statusStr string + completed := false + var taskErr error + switch t.Status { + case 0: + statusStr = "downloading" + case 2: + statusStr = "succeed" + completed = true + case 1: + statusStr = "failed" + taskErr = fmt.Errorf("offline download failed") + case 3: + statusStr = "retrying" + default: + statusStr = fmt.Sprintf("status_%d", t.Status) + } + + return &tool.Status{ + TotalBytes: t.Size, + Progress: t.Progress, + Completed: completed, + Status: statusStr, + Err: taskErr, + }, nil +} + +var _ tool.Tool = (*Pan123)(nil) + +func init() { + tool.Tools.Add(&Pan123{}) +} diff --git a/internal/offline_download/123_open/client.go b/internal/offline_download/123_open/client.go new file mode 100644 index 000000000..ce1453c32 --- /dev/null +++ b/internal/offline_download/123_open/client.go @@ -0,0 +1,119 @@ +package _123_open + +import ( + "context" + "fmt" + "strconv" + + _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" +) + +type Open123 struct{} + +func (*Open123) Name() string { + return "123 Open" +} + +func (*Open123) Items() []model.SettingItem { + return nil +} + +func (*Open123) Run(_ *tool.DownloadTask) error { + return errs.NotSupport +} + +func (*Open123) Init() (string, error) { + return "ok", nil +} + +func (*Open123) IsReady() bool { + tempDir := setting.GetStr(conf.Pan123OpenTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*_123_open.Open123); !ok { + return false + } + return true +} + +func (*Open123) AddURL(args *tool.AddUrlArgs) (string, error) { + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver123Open, ok := storage.(*_123_open.Open123) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only 123 Open is supported") + } + ctx := context.Background() + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + cb := setting.GetStr(conf.Pan123OpenOfflineDownloadCallbackUrl) + taskID, err := driver123Open.OfflineDownload(ctx, args.Url, parentDir, cb) + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + return strconv.Itoa(taskID), nil +} + +func (*Open123) Remove(_ *tool.DownloadTask) error { + return errs.NotSupport +} + +func (*Open123) Status(task *tool.DownloadTask) (*tool.Status, error) { + taskID, err := strconv.Atoi(task.GID) + if err != nil { + return nil, fmt.Errorf("failed to parse task ID: %s", task.GID) + } + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver123Open, ok := storage.(*_123_open.Open123) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only 123 Open is supported") + } + process, status, err := driver123Open.OfflineDownloadProcess(context.Background(), taskID) + if err != nil { + return nil, err + } + var statusStr string + switch status { + case 0: + statusStr = "downloading" + case 1: + err = fmt.Errorf("offline download failed") + case 2: + statusStr = "succeed" + case 3: + statusStr = "retrying" + } + return &tool.Status{ + Progress: process, + Completed: status == 2, + Status: statusStr, + Err: err, + }, nil +} + +var _ tool.Tool = (*Open123)(nil) + +func init() { + tool.Tools.Add(&Open123{}) +} diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index f79a8312d..4fcbdb9c4 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -3,6 +3,8 @@ package offline_download import ( _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/115" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/115_open" + _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123" + _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123_open" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/aria2" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/http" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/pikpak" diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index aea88e2a4..0f574571e 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,18 +2,17 @@ package tool import ( "context" - "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" - - _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" - "github.com/OpenListTeam/OpenList/v4/server/common" - "net/url" stdpath "path" "path/filepath" _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" + _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" + _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" + "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -22,6 +21,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/task" + "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/google/uuid" "github.com/pkg/errors" ) @@ -104,6 +104,19 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } else { tempDir = filepath.Join(setting.GetStr(conf.Pan115OpenTempDir), uid) } + case "123 Open": + if _, ok := storage.(*_123_open.Open123); ok && dstDirActualPath != "/" { + // directly offline downloading to the root path is not allowed via 123 open platform + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.Pan123OpenTempDir), uid) + } + case "123Pan": + if _, ok := storage.(*_123.Pan123); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.Pan123TempDir), uid) + } case "PikPak": if _, ok := storage.(*pikpak.PikPak); ok { tempDir = args.DstDirPath diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 2d402dcf9..50a4f6343 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -2,6 +2,7 @@ package tool import ( "fmt" + "path" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -32,9 +33,6 @@ type DownloadTask struct { } func (t *DownloadTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err - } t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() @@ -111,6 +109,9 @@ outer: if t.tool.Name() == "115 Open" { return nil } + if t.tool.Name() == "123 Open" { + return nil + } t.Status = "offline download completed, maybe transferring" // hack for qBittorrent if t.tool.Name() == "qBittorrent" { @@ -174,7 +175,7 @@ func (t *DownloadTask) Update() (bool, error) { func (t *DownloadTask) Transfer() error { toolName := t.tool.Name() - if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" { + if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "123 Open" || toolName == "123Pan" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" { // 如果不是直接下载到目标路径，则进行转存 if t.TempDir != t.DstDirPath { return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) @@ -198,11 +199,11 @@ func (t *DownloadTask) Transfer() error { DstStorage: dstStorage, DstStorageMp: dstStorage.GetStorage().MountPath, }, - groupID: t.DstDirPath, DeletePolicy: t.DeletePolicy, Url: t.Url, } tsk.SetTotalBytes(t.GetTotalBytes()) + tsk.groupID = path.Join(tsk.DstStorageMp, tsk.DstActualPath) task_group.TransferCoordinator.AddTask(tsk.groupID, nil) TransferTaskManager.Add(tsk) return nil diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index 1c1284a02..fd6b8f464 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path" stdpath "path" "path/filepath" "time" @@ -31,8 +32,19 @@ type TransferTask struct { } func (t *TransferTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err + if t.SrcStorage == nil && t.SrcStorageMp != "" { + if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil { + t.SrcStorage = srcStorage + } else { + return err + } + if t.DstStorage == nil { + if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil { + t.DstStorage = dstStorage + } else { + return err + } + } } t.ClearEndTime() t.SetStartTime(time.Now()) @@ -61,12 +73,11 @@ func (t *TransferTask) Run() error { Mimetype: mimetype, Closers: utils.NewClosers(r), } - return op.Put(t.Ctx(), t.DstStorage, t.DstActualPath, s, t.SetProgress) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) } return transferStdPath(t) - } else { - return transferObjPath(t) } + return transferObjPath(t) } func (t *TransferTask) GetName() string { @@ -84,7 +95,7 @@ func (t *TransferTask) OnSucceeded() { removeObjTemp(t) } } - task_group.TransferCoordinator.Done(t.groupID, true) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, true) } func (t *TransferTask) OnFailed() { @@ -95,7 +106,7 @@ func (t *TransferTask) OnFailed() { removeObjTemp(t) } } - task_group.TransferCoordinator.Done(t.groupID, false) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, false) } func (t *TransferTask) SetRetry(retry int, maxRetry int) { @@ -105,7 +116,7 @@ func (t *TransferTask) SetRetry(retry int, maxRetry int) { t.groupID = stdpath.Join(t.DstStorageMp, t.DstActualPath) task_group.TransferCoordinator.AddTask(t.groupID, nil) } - t.TaskExtension.SetRetry(retry, maxRetry) + t.TaskData.SetRetry(retry, maxRetry) } var ( @@ -134,10 +145,10 @@ func transferStd(ctx context.Context, tempDir, dstDirPath string, deletePolicy D DstStorage: dstStorage, DstStorageMp: dstStorage.GetStorage().MountPath, }, - groupID: dstDirPath, DeletePolicy: deletePolicy, } - task_group.TransferCoordinator.AddTask(dstDirPath, nil) + t.groupID = path.Join(t.DstStorageMp, t.DstActualPath) + task_group.TransferCoordinator.AddTask(t.groupID, nil) TransferTaskManager.Add(t) } return nil @@ -156,7 +167,7 @@ func transferStdPath(t *TransferTask) error { return err } dstDirActualPath := stdpath.Join(t.DstActualPath, info.Name()) - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToRefresh(dstDirActualPath)) + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstDirActualPath)) for _, entry := range entries { srcRawPath := stdpath.Join(t.SrcActualPath, entry.Name()) task := &TransferTask{ @@ -206,7 +217,7 @@ func transferStdFile(t *TransferTask) error { Closers: utils.NewClosers(rc), } t.SetTotalBytes(info.Size()) - return op.Put(t.Ctx(), t.DstStorage, t.DstActualPath, s, t.SetProgress) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) } func removeStdTemp(t *TransferTask) { @@ -247,10 +258,10 @@ func transferObj(ctx context.Context, tempDir, dstDirPath string, deletePolicy D SrcStorageMp: srcStorage.GetStorage().MountPath, DstStorageMp: dstStorage.GetStorage().MountPath, }, - groupID: dstDirPath, DeletePolicy: deletePolicy, } - task_group.TransferCoordinator.AddTask(dstDirPath, nil) + t.groupID = path.Join(t.DstStorageMp, t.DstActualPath) + task_group.TransferCoordinator.AddTask(t.groupID, nil) TransferTaskManager.Add(t) } return nil @@ -269,7 +280,7 @@ func transferObjPath(t *TransferTask) error { return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcActualPath) } dstDirActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToRefresh(dstDirActualPath)) + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstDirActualPath)) for _, obj := range objs { if utils.IsCanceled(t.Ctx()) { return nil @@ -300,11 +311,11 @@ func transferObjPath(t *TransferTask) error { } func transferObjFile(t *TransferTask) error { - srcFile, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcActualPath) + _, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcActualPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", t.SrcActualPath) } - link, _, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) + link, srcFile, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) if err != nil { return errors.WithMessagef(err, "failed get [%s] link", t.SrcActualPath) } @@ -318,7 +329,7 @@ func transferObjFile(t *TransferTask) error { return errors.WithMessagef(err, "failed get [%s] stream", t.SrcActualPath) } t.SetTotalBytes(ss.GetSize()) - return op.Put(t.Ctx(), t.DstStorage, t.DstActualPath, ss, t.SetProgress) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } func removeObjTemp(t *TransferTask) { diff --git a/internal/op/archive.go b/internal/op/archive.go index 4d85d2064..d0a53a919 100644 --- a/internal/op/archive.go +++ b/internal/op/archive.go @@ -3,31 +3,35 @@ package op import ( "context" stderrors "errors" - "fmt" "io" stdpath "path" + "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" - "github.com/OpenListTeam/OpenList/v4/internal/stream" - + "github.com/OpenListTeam/OpenList/v4/internal/cache" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/go-cache" + gocache "github.com/OpenListTeam/go-cache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) -var archiveMetaCache = cache.NewMemCache(cache.WithShards[*model.ArchiveMetaProvider](64)) -var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider] +var ( + archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64)) + archiveMetaG singleflight.Group[*model.ArchiveMetaProvider] +) func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) key := Key(storage, path) @@ -37,14 +41,14 @@ func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg return nil, errors.Wrapf(err, "failed to get %s archive met: %+v", path, err) } if m.Expiration != nil { - archiveMetaCache.Set(key, m, cache.WithEx[*model.ArchiveMetaProvider](*m.Expiration)) + archiveMetaCache.Set(key, m, gocache.WithEx[*model.ArchiveMetaProvider](*m.Expiration)) } return m, nil } - if storage.Config().OnlyLinkMFile { - meta, err := fn() - return meta, err - } + // if storage.Config().NoLinkSingleflight { + // meta, err := fn() + // return meta, err + // } if !args.Refresh { if meta, ok := archiveMetaCache.Get(key); ok { log.Debugf("use cache when get %s archive meta", path) @@ -60,20 +64,25 @@ func GetArchiveToolAndStream(ctx context.Context, storage driver.Driver, path st if err != nil { return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] link", path) } - baseName, ext, found := strings.Cut(obj.GetName(), ".") - if !found { - _ = l.Close() - return nil, nil, nil, errors.Errorf("failed get archive tool: the obj does not have an extension.") - } - partExt, t, err := tool.GetArchiveTool("." + ext) - if err != nil { - var e error - partExt, t, e = tool.GetArchiveTool(stdpath.Ext(obj.GetName())) - if e != nil { + + // Get archive tool + var partExt *tool.MultipartExtension + var t tool.Tool + ext := obj.GetName() + for { + var found bool + _, ext, found = strings.Cut(ext, ".") + if !found { _ = l.Close() - return nil, nil, nil, errors.WithMessagef(stderrors.Join(err, e), "failed get archive tool: %s", ext) + return nil, nil, nil, errors.Errorf("failed get archive tool: the obj does not have an extension.") + } + partExt, t, err = tool.GetArchiveTool("." + ext) + if err == nil { + break } } + + // Get first part stream ss, err := stream.NewSeekableStream(&stream.FileStream{Ctx: ctx, Obj: obj}, l) if err != nil { _ = l.Close() @@ -82,29 +91,62 @@ func GetArchiveToolAndStream(ctx context.Context, storage driver.Driver, path st ret := []*stream.SeekableStream{ss} if partExt == nil { return obj, t, ret, nil - } else { - index := partExt.SecondPartIndex - dir := stdpath.Dir(path) - for { - p := stdpath.Join(dir, baseName+fmt.Sprintf(partExt.PartFileFormat, index)) - var o model.Obj - l, o, err = Link(ctx, storage, p, args) - if err != nil { - break - } - ss, err = stream.NewSeekableStream(&stream.FileStream{Ctx: ctx, Obj: o}, l) - if err != nil { - _ = l.Close() - for _, s := range ret { - _ = s.Close() - } - return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path) + } + + // Merge multi-part archive + dir := stdpath.Dir(path) + objs, err := List(ctx, storage, dir, model.ListArgs{}) + if err != nil { + return obj, t, ret, nil + } + for _, o := range objs { + submatch := partExt.PartFileFormat.FindStringSubmatch(o.GetName()) + if submatch == nil { + continue + } + partIdx, e := strconv.Atoi(submatch[1]) + if e != nil { + continue + } + partIdx = partIdx - partExt.SecondPartIndex + 1 + if partIdx < 1 { + continue + } + p := stdpath.Join(dir, o.GetName()) + l1, o1, e := Link(ctx, storage, p, args) + if e != nil { + err = errors.WithMessagef(e, "failed get [%s] link", p) + break + } + ss1, e := stream.NewSeekableStream(&stream.FileStream{Ctx: ctx, Obj: o1}, l1) + if e != nil { + _ = l1.Close() + err = errors.WithMessagef(e, "failed get [%s] stream", p) + break + } + for partIdx >= len(ret) { + ret = append(ret, nil) + } + ret[partIdx] = ss1 + } + closeAll := func(r []*stream.SeekableStream) { + for _, s := range r { + if s != nil { + _ = s.Close() } - ret = append(ret, ss) - index++ } - return obj, t, ret, nil } + if err != nil { + closeAll(ret) + return nil, nil, nil, err + } + for i, ss1 := range ret { + if ss1 == nil { + closeAll(ret) + return nil, nil, nil, errors.Errorf("failed merge [%s] parts, missing part %d", path, i) + } + } + return obj, t, ret, nil } func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (model.Obj, *model.ArchiveMetaProvider, error) { @@ -158,12 +200,14 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg return obj, archiveMetaProvider, err } -var archiveListCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) -var archiveListG singleflight.Group[[]model.Obj] +var ( + archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64)) + archiveListG singleflight.Group[[]model.Obj] +) func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) metaKey := Key(storage, path) @@ -179,16 +223,10 @@ func ListArchive(ctx context.Context, storage driver.Driver, path string, args m // } } objs, err, _ := archiveListG.Do(key, func() ([]model.Obj, error) { - obj, files, err := listArchive(ctx, storage, path, args) + files, err := listArchive(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed to list archive [%s]%s: %+v", path, args.InnerPath, err) } - // set path - for _, f := range files { - if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && obj.GetPath() != "" { - s.SetPath(stdpath.Join(obj.GetPath(), args.InnerPath, f.GetName())) - } - } // warp obj name model.WrapObjsName(files) // sort objs @@ -199,7 +237,7 @@ func ListArchive(ctx context.Context, storage driver.Driver, path string, args m if !storage.Config().NoCache { if len(files) > 0 { log.Debugf("set cache: %s => %+v", key, files) - archiveListCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + archiveListCache.Set(key, files, gocache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) } else { log.Debugf("del cache: %s", key) archiveListCache.Del(key) @@ -210,24 +248,24 @@ func ListArchive(ctx context.Context, storage driver.Driver, path string, args m return objs, err } -func _listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { +func _listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { storageAr, ok := storage.(driver.ArchiveReader) if ok { obj, err := GetUnwrap(ctx, storage, path) if err != nil { - return nil, nil, errors.WithMessage(err, "failed to get file") + return nil, errors.WithMessage(err, "failed to get file") } if obj.IsDir() { - return nil, nil, errors.WithStack(errs.NotFile) + return nil, errors.WithStack(errs.NotFile) } files, err := storageAr.ListArchive(ctx, obj, args.ArchiveInnerArgs) if !errors.Is(err, errs.NotImplement) { - return obj, files, err + return files, err } } - obj, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + _, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { - return nil, nil, err + return nil, err } defer func() { var e error @@ -239,11 +277,11 @@ func _listArchive(ctx context.Context, storage driver.Driver, path string, args } }() files, err := t.List(ss, args.ArchiveInnerArgs) - return obj, files, err + return files, err } -func listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { - obj, files, err := _listArchive(ctx, storage, path, args) +func listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { + files, err := _listArchive(ctx, storage, path, args) if errors.Is(err, errs.NotSupport) { var meta model.ArchiveMeta meta, err = GetArchiveMeta(ctx, storage, path, model.ArchiveMetaArgs{ @@ -251,20 +289,17 @@ func listArchive(ctx context.Context, storage driver.Driver, path string, args m Refresh: args.Refresh, }) if err != nil { - return nil, nil, err + return nil, err } files, err = getChildrenFromArchiveMeta(meta, args.InnerPath) if err != nil { - return nil, nil, err + return nil, err } } - if err == nil && obj == nil { - obj, err = GetUnwrap(ctx, storage, path) - } if err != nil { - return nil, nil, err + return nil, err } - return obj, files, err + return files, err } func getChildrenFromArchiveMeta(meta model.ArchiveMeta, innerPath string) ([]model.Obj, error) { @@ -309,7 +344,7 @@ func splitPath(path string) []string { func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) af, err := GetUnwrap(ctx, storage, path) @@ -354,75 +389,52 @@ func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args mo return nil, nil, errors.WithStack(errs.ObjectNotFound) } -type extractLink struct { - *model.Link - Obj model.Obj +type objWithLink struct { + link *model.Link + obj model.Obj } -var extractCache = cache.NewMemCache(cache.WithShards[*extractLink](16)) -var extractG = singleflight.Group[*extractLink]{Remember: true} +var ( + extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute) + extractG = singleflight.Group[*objWithLink]{} +) func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } key := stdpath.Join(Key(storage, path), args.InnerPath) - if link, ok := extractCache.Get(key); ok { - return link.Link, link.Obj, nil + if ol, ok := extractCache.Get(key); ok { + if ol.link.Expiration != nil || ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil + } } - var forget any - var linkM *extractLink - fn := func() (*extractLink, error) { - link, err := driverExtract(ctx, storage, path, args) + fn := func() (*objWithLink, error) { + ol, err := driverExtract(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed extract archive") } - if link.MFile != nil && forget != nil { - linkM = link - return nil, errLinkMFileCache - } - if link.Link.Expiration != nil { - extractCache.Set(key, link, cache.WithEx[*extractLink](*link.Link.Expiration)) + if ol.link.Expiration != nil { + extractCache.SetWithTTL(key, ol, *ol.link.Expiration) + } else { + extractCache.SetWithExpirable(key, ol, &ol.link.SyncClosers) } - link.AddIfCloser(forget) - return link, nil + return ol, nil } - if storage.Config().OnlyLinkMFile { - link, err := fn() + for { + ol, err, _ := extractG.Do(key, fn) if err != nil { return nil, nil, err } - return link.Link, link.Obj, nil - } - - forget = utils.CloseFunc(func() error { - if forget != nil { - forget = nil - linkG.Forget(key) - } - return nil - }) - link, err, _ := extractG.Do(key, fn) - for err == nil && !link.AcquireReference() { - link, err, _ = extractG.Do(key, fn) - } - if err == errLinkMFileCache { - if linkM != nil { - return linkM.Link, linkM.Obj, nil + if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil } - forget = nil - link, err = fn() } - - if err != nil { - return nil, nil, err - } - return link.Link, link.Obj, nil } -func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*extractLink, error) { +func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*objWithLink, error) { storageAr, ok := storage.(driver.ArchiveReader) if !ok { return nil, errs.DriverExtractNotSupported @@ -438,7 +450,7 @@ func driverExtract(ctx context.Context, storage driver.Driver, path string, args return nil, errors.WithStack(errs.NotFile) } link, err := storageAr.Extract(ctx, archiveFile, args) - return &extractLink{Link: link, Obj: extracted}, err + return &objWithLink{link: link, obj: extracted}, err } type streamWithParent struct { @@ -480,7 +492,7 @@ func InternalExtract(ctx context.Context, storage driver.Driver, path string, ar func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) @@ -493,26 +505,56 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD return errors.WithMessage(err, "failed to get dst dir") } + var newObjs []model.Obj switch s := storage.(type) { case driver.ArchiveDecompressResult: - var newObjs []model.Obj newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) if err == nil { if len(newObjs) > 0 { - for _, newObj := range newObjs { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { + for _, newObj := range newObjs { + cache.UpdateObject(newObj.GetName(), newObj) + } + } } } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) + Cache.DeleteDirectory(storage, dstDirPath) } } case driver.ArchiveDecompress: err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) + Cache.DeleteDirectory(storage, dstDirPath) } default: return errs.NotImplement } + if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() { + onlyList := false + targetPath := dstDirPath + if len(newObjs) == 1 && newObjs[0].IsDir() { + targetPath = stdpath.Join(dstDirPath, newObjs[0].GetName()) + } else if len(newObjs) == 1 && !newObjs[0].IsDir() { + onlyList = true + } else if args.PutIntoNewDir { + targetPath = stdpath.Join(dstDirPath, strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName()))) + } else if innerBase := stdpath.Base(args.InnerPath); innerBase != "." && innerBase != "/" { + targetPath = stdpath.Join(dstDirPath, innerBase) + dstObj, e := Get(ctx, storage, targetPath) + onlyList = e != nil || !dstObj.IsDir() + } + if onlyList { + go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true}) + } else { + var limiter *rate.Limiter + if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil { + if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 { + limiter = rate.NewLimiter(rate.Limit(f), 1) + } + } + go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil) + } + } return errors.WithStack(err) } diff --git a/internal/op/cache.go b/internal/op/cache.go new file mode 100644 index 000000000..d8d32a74b --- /dev/null +++ b/internal/op/cache.go @@ -0,0 +1,228 @@ +package op + +import ( + stdpath "path" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/cache" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type CacheManager struct { + dirCache *cache.KeyedCache[*directoryCache] // Cache for directory listings + linkCache *cache.TypedCache[*objWithLink] // Cache for file links + userCache *cache.KeyedCache[*model.User] // Cache for user data + settingCache *cache.KeyedCache[any] // Cache for settings + detailCache *cache.KeyedCache[*model.StorageDetails] // Cache for storage details +} + +func NewCacheManager() *CacheManager { + return &CacheManager{ + dirCache: cache.NewKeyedCache[*directoryCache](time.Minute * 5), + linkCache: cache.NewTypedCache[*objWithLink](time.Minute * 30), + userCache: cache.NewKeyedCache[*model.User](time.Hour), + settingCache: cache.NewKeyedCache[any](time.Hour), + detailCache: cache.NewKeyedCache[*model.StorageDetails](time.Minute * 30), + } +} + +// global instance +var Cache = NewCacheManager() + +func Key(storage driver.Driver, path string) string { + return utils.GetFullPath(storage.GetStorage().MountPath, path) +} + +// recursively delete directory and its children from dirCache +func (cm *CacheManager) DeleteDirectoryTree(storage driver.Driver, dirPath string) { + if storage.Config().NoCache { + return + } + cm.deleteDirectoryTree(Key(storage, dirPath)) +} +func (cm *CacheManager) deleteDirectoryTree(key string) { + if dirCache, exists := cm.dirCache.Pop(key); exists { + for _, obj := range dirCache.objs { + if obj.IsDir() { + cm.deleteDirectoryTree(stdpath.Join(key, obj.GetName())) + } else { + cm.linkCache.DeleteKey(stdpath.Join(key, obj.GetName())) + } + } + } +} + +// remove directory from dirCache +func (cm *CacheManager) DeleteDirectory(storage driver.Driver, dirPath string) { + if storage.Config().NoCache { + return + } + cm.dirCache.Delete(Key(storage, dirPath)) +} + +// remove object from dirCache. +// if it's a directory, remove all its children from dirCache too. +// if it's a file, remove its link from linkCache. +func (cm *CacheManager) removeDirectoryObject(storage driver.Driver, dirPath string, obj model.Obj) { + key := Key(storage, dirPath) + if !obj.IsDir() { + cm.linkCache.DeleteKey(stdpath.Join(key, obj.GetName())) + } + + if storage.Config().NoCache { + return + } + if cache, exist := cm.dirCache.Get(key); exist { + if obj.IsDir() { + cm.deleteDirectoryTree(stdpath.Join(key, obj.GetName())) + } + cache.RemoveObject(obj.GetName()) + } +} + +// cache user data +func (cm *CacheManager) SetUser(username string, user *model.User) { + cm.userCache.Set(username, user) +} + +// cached user data +func (cm *CacheManager) GetUser(username string) (*model.User, bool) { + return cm.userCache.Get(username) +} + +// remove user data from cache +func (cm *CacheManager) DeleteUser(username string) { + cm.userCache.Delete(username) +} + +// caches setting +func (cm *CacheManager) SetSetting(key string, setting *model.SettingItem) { + cm.settingCache.Set(key, setting) +} + +// cached setting +func (cm *CacheManager) GetSetting(key string) (*model.SettingItem, bool) { + if data, exists := cm.settingCache.Get(key); exists { + if setting, ok := data.(*model.SettingItem); ok { + return setting, true + } + } + return nil, false +} + +// cache setting groups +func (cm *CacheManager) SetSettingGroup(key string, settings []model.SettingItem) { + cm.settingCache.Set(key, settings) +} + +// cached setting group +func (cm *CacheManager) GetSettingGroup(key string) ([]model.SettingItem, bool) { + if data, exists := cm.settingCache.Get(key); exists { + if settings, ok := data.([]model.SettingItem); ok { + return settings, true + } + } + return nil, false +} + +func (cm *CacheManager) SetStorageDetails(storage driver.Driver, details *model.StorageDetails) { + if storage.Config().NoCache { + return + } + expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) + cm.detailCache.SetWithTTL(utils.GetActualMountPath(storage.GetStorage().MountPath), details, expiration) +} + +func (cm *CacheManager) GetStorageDetails(storage driver.Driver) (*model.StorageDetails, bool) { + return cm.detailCache.Get(utils.GetActualMountPath(storage.GetStorage().MountPath)) +} + +func (cm *CacheManager) InvalidateStorageDetails(storage driver.Driver) { + cm.detailCache.Delete(utils.GetActualMountPath(storage.GetStorage().MountPath)) +} + +// clears all caches +func (cm *CacheManager) ClearAll() { + cm.dirCache.Clear() + cm.linkCache.Clear() + cm.userCache.Clear() + cm.settingCache.Clear() + cm.detailCache.Clear() +} + +type directoryCache struct { + objs []model.Obj + sorted []model.Obj + mu sync.RWMutex + + dirtyFlags uint8 +} + +const ( + dirtyRemove uint8 = 1 << iota // 对象删除：刷新 sorted 副本，但不需要 full sort/extract + dirtyUpdate // 对象更新：需要执行 full sort + extract +) + +func newDirectoryCache(objs []model.Obj) *directoryCache { + sorted := make([]model.Obj, len(objs)) + copy(sorted, objs) + return &directoryCache{ + objs: objs, + sorted: sorted, + } +} + +func (dc *directoryCache) RemoveObject(name string) { + dc.mu.Lock() + defer dc.mu.Unlock() + for i, obj := range dc.objs { + if obj.GetName() == name { + dc.objs = append(dc.objs[:i], dc.objs[i+1:]...) + dc.dirtyFlags |= dirtyRemove + break + } + } +} + +func (dc *directoryCache) UpdateObject(oldName string, newObj model.Obj) { + dc.mu.Lock() + defer dc.mu.Unlock() + if oldName != "" { + for i, obj := range dc.objs { + if obj.GetName() == oldName { + dc.objs[i] = newObj + dc.dirtyFlags |= dirtyUpdate + return + } + } + } + dc.objs = append(dc.objs, newObj) + dc.dirtyFlags |= dirtyUpdate +} + +func (dc *directoryCache) GetSortedObjects(meta driver.Meta) []model.Obj { + dc.mu.RLock() + if dc.dirtyFlags == 0 { + dc.mu.RUnlock() + return dc.sorted + } + dc.mu.RUnlock() + dc.mu.Lock() + defer dc.mu.Unlock() + + sorted := make([]model.Obj, len(dc.objs)) + copy(sorted, dc.objs) + dc.sorted = sorted + if dc.dirtyFlags&dirtyUpdate != 0 { + storage := meta.GetStorage() + if meta.Config().LocalSort { + model.SortFiles(sorted, storage.OrderBy, storage.OrderDirection) + } + model.ExtractFolder(sorted, storage.ExtractFolder) + } + dc.dirtyFlags = 0 + return sorted +} diff --git a/internal/op/driver.go b/internal/op/driver.go index f25b3a6b1..5b79b0aed 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -80,6 +80,13 @@ func getMainItems(config driver.Config) []driver.Item { Required: true, Help: "The cache expiration time for this storage", }) + items = append(items, driver.Item{ + Name: "custom_cache_policies", + Type: conf.TypeText, + Default: "", + Required: false, + Help: "The cache expiration rules for this storage", + }) } if config.MustProxy() { items = append(items, driver.Item{ @@ -90,17 +97,32 @@ func getMainItems(config driver.Config) []driver.Item { Required: true, }) } else { - items = append(items, []driver.Item{{ - Name: "web_proxy", - Type: conf.TypeBool, - }, { - Name: "webdav_policy", - Type: conf.TypeSelect, - Options: "302_redirect,use_proxy_url,native_proxy", - Default: "302_redirect", - Required: true, - }, - }...) + if config.DefaultProxy() { + items = append(items, []driver.Item{{ + Name: "web_proxy", + Type: conf.TypeBool, + Default: "true", + }, { + Name: "webdav_policy", + Type: conf.TypeSelect, + Options: "302_redirect,use_proxy_url,native_proxy", + Default: "native_proxy", + Required: true, + }, + }...) + } else { + items = append(items, []driver.Item{{ + Name: "web_proxy", + Type: conf.TypeBool, + }, { + Name: "webdav_policy", + Type: conf.TypeSelect, + Options: "302_redirect,use_proxy_url,native_proxy", + Default: "302_redirect", + Required: true, + }, + }...) + } if config.ProxyRangeOption { item := driver.Item{ Name: "proxy_range", diff --git a/internal/op/fs.go b/internal/op/fs.go index 99f6c84af..5116bbef5 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -2,242 +2,212 @@ package op import ( "context" - stderrors "errors" stdpath "path" - "slices" + "strconv" "strings" "time" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" - "github.com/OpenListTeam/OpenList/v4/pkg/generic_sync" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/go-cache" + "github.com/bmatcuk/doublestar/v4" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) -// In order to facilitate adding some other things before and after file op - -var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) var listG singleflight.Group[[]model.Obj] -func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) { - key := Key(storage, path) - objs, ok := listCache.Get(key) - if ok { - for i, obj := range objs { - if obj.GetName() == newObj.GetName() { - objs = slices.Delete(objs, i, i+1) - break - } - } - for i, obj := range objs { - if obj.GetName() == oldObj.GetName() { - objs[i] = newObj - break - } - } - listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } -} - -func delCacheObj(storage driver.Driver, path string, obj model.Obj) { - key := Key(storage, path) - objs, ok := listCache.Get(key) - if ok { - for i, oldObj := range objs { - if oldObj.GetName() == obj.GetName() { - objs = append(objs[:i], objs[i+1:]...) - break - } - } - listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } -} - -var addSortDebounceMap generic_sync.MapOf[string, func(func())] - -func addCacheObj(storage driver.Driver, path string, newObj model.Obj) { - key := Key(storage, path) - objs, ok := listCache.Get(key) - if ok { - for i, obj := range objs { - if obj.GetName() == newObj.GetName() { - objs[i] = newObj - return - } - } - - // Simple separation of files and folders - if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() { - objs = append(objs, newObj) - } else { - objs = append([]model.Obj{newObj}, objs...) - } - - if storage.Config().LocalSort { - debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute)) - log.Debug("addCacheObj: wait start sort") - debounce(func() { - log.Debug("addCacheObj: start sort") - model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) - addSortDebounceMap.Delete(key) - }) - } - - listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } -} - -func ClearCache(storage driver.Driver, path string) { - objs, ok := listCache.Get(Key(storage, path)) - if ok { - for _, obj := range objs { - if obj.IsDir() { - ClearCache(storage, stdpath.Join(path, obj.GetName())) - } - } - } - listCache.Del(Key(storage, path)) -} - -func DeleteCache(storage driver.Driver, path string) { - listCache.Del(Key(storage, path)) -} - -func Key(storage driver.Driver, path string) string { - return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path)) -} - // List files in storage, not contains virtual file func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs) ([]model.Obj, error) { + return list(ctx, storage, path, args, nil) +} + +func list(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, resultValidator func([]model.Obj) error) ([]model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) log.Debugf("op.List %s", path) key := Key(storage, path) if !args.Refresh { - if files, ok := listCache.Get(key); ok { + if dirCache, exists := Cache.dirCache.Get(key); exists { log.Debugf("use cache when list %s", path) - return files, nil + objs := dirCache.GetSortedObjects(storage) + if resultValidator != nil { + if err := resultValidator(objs); err == nil { + return objs, nil + } + } else { + return objs, nil + } } } - dir, err := GetUnwrap(ctx, storage, path) - if err != nil { - return nil, errors.WithMessage(err, "failed get dir") - } - log.Debugf("list dir: %+v", dir) - if !dir.IsDir() { - return nil, errors.WithStack(errs.NotFolder) - } + objs, err, _ := listG.Do(key, func() ([]model.Obj, error) { + dir, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, errors.WithMessage(err, "failed get dir") + } + log.Debugf("list dir: %+v", dir) + if !dir.IsDir() { + return nil, errors.WithStack(errs.NotFolder) + } files, err := storage.List(ctx, dir, args) if err != nil { return nil, errors.Wrapf(err, "failed to list objs") } - // set path - for _, f := range files { - if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" { - s.SetPath(stdpath.Join(dir.GetPath(), f.GetName())) - } - } // warp obj name - model.WrapObjsName(files) - // call hooks - go func(reqPath string, files []model.Obj) { - HandleObjsUpdateHook(reqPath, files) - }(utils.GetFullPath(storage.GetStorage().MountPath, path), files) - + wrapObjsName(storage, files) // sort objs if storage.Config().LocalSort { model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) } model.ExtractFolder(files, storage.GetStorage().ExtractFolder) + if !args.SkipHook { + // call hooks + go func(reqPath string, files []model.Obj) { + HandleObjsUpdateHook(context.WithoutCancel(ctx), reqPath, files) + }(utils.GetFullPath(storage.GetStorage().MountPath, path), files) + } + if !storage.Config().NoCache { if len(files) > 0 { log.Debugf("set cache: %s => %+v", key, files) - listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + + ttl := storage.GetStorage().CacheExpiration + + customCachePolicies := storage.GetStorage().CustomCachePolicies + if len(customCachePolicies) > 0 { + configPolicies := strings.Split(customCachePolicies, "\n") + for _, configPolicy := range configPolicies { + pattern, ttlstr, ok := strings.Cut(strings.TrimSpace(configPolicy), ":") + if !ok { + log.Warnf("Malformed custom cache policy entry: %s in storage %s for path %s. Expected format: pattern:ttl", configPolicy, storage.GetStorage().MountPath, path) + continue + } + if match, err1 := doublestar.Match(pattern, path); err1 != nil { + log.Warnf("Invalid glob pattern in custom cache policy: %s, error: %v", pattern, err1) + continue + } else if !match { + continue + } + + if configTtl, err1 := strconv.ParseInt(ttlstr, 10, 64); err1 == nil { + ttl = int(configTtl) + break + } + } + } + + duration := time.Minute * time.Duration(ttl) + Cache.dirCache.SetWithTTL(key, newDirectoryCache(files), duration) } else { log.Debugf("del cache: %s", key) - listCache.Del(key) + Cache.deleteDirectoryTree(key) } } return files, nil }) - return objs, err + if err != nil { + return nil, err + } + if resultValidator != nil { + if err := resultValidator(objs); err != nil { + return nil, err + } + } + return objs, nil } // Get object from list of files -func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { +func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj ...bool) (model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) + } path = utils.FixAndCleanPath(path) log.Debugf("op.Get %s", path) + // is root folder + if path == "/" { + if getRooter, ok := storage.(driver.GetRooter); ok { + rootObj, err := getRooter.GetRoot(ctx) + if err != nil { + return nil, errors.WithMessage(err, "failed get root obj") + } + return rootObj, nil + } + switch r := storage.(type) { + case driver.IRootId: + return &model.Object{ + ID: r.GetRootId(), + Name: RootName, + Modified: storage.GetStorage().Modified, + IsFolder: true, + Mask: model.Locked, + }, nil + case driver.IRootPath: + return &model.Object{ + Path: r.GetRootPath(), + Name: RootName, + Modified: storage.GetStorage().Modified, + Mask: model.Locked, + IsFolder: true, + }, nil + } + return nil, errors.New("please implement GetRooter or IRootPath or IRootId interface") + } + + // try get from cache first + dir, name := stdpath.Split(path) + dirCache, dirCacheExists := Cache.dirCache.Get(Key(storage, dir)) + refreshList := false + excludeTemp := utils.IsBool(excludeTempObj...) + if dirCacheExists { + files := dirCache.GetSortedObjects(storage) + for _, f := range files { + if f.GetName() == name { + if excludeTemp && model.ObjHasMask(f, model.Temp) { + refreshList = true + break + } + return f, nil + } + } + } + // get the obj directly without list so that we can reduce the io if g, ok := storage.(driver.Getter); ok { obj, err := g.Get(ctx, path) if err == nil { - return model.WrapObjName(obj), nil + return obj, nil } if !errs.IsNotImplementError(err) && !errs.IsNotSupportError(err) { return nil, errors.WithMessage(err, "failed to get obj") } } - // is root folder - if utils.PathEqual(path, "/") { - var rootObj model.Obj - if getRooter, ok := storage.(driver.GetRooter); ok { - obj, err := getRooter.GetRoot(ctx) - if err != nil { - return nil, errors.WithMessage(err, "failed get root obj") - } - rootObj = obj - } else { - switch r := storage.GetAddition().(type) { - case driver.IRootId: - rootObj = &model.Object{ - ID: r.GetRootId(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, - } - case driver.IRootPath: - rootObj = &model.Object{ - Path: r.GetRootPath(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, + if !dirCacheExists || refreshList { + var obj model.Obj + list(ctx, storage, dir, model.ListArgs{Refresh: refreshList}, func(objs []model.Obj) error { + for _, f := range objs { + if f.GetName() == name { + if excludeTemp && model.ObjHasMask(f, model.Temp) { + return errs.ObjectNotFound + } + obj = f + return nil } - default: - return nil, errors.Errorf("please implement IRootPath or IRootId or GetRooter method") } - } - if rootObj == nil { - return nil, errors.Errorf("please implement IRootPath or IRootId or GetRooter method") - } - return &model.ObjWrapName{ - Name: RootName, - Obj: rootObj, - }, nil - } - - // not root folder - dir, name := stdpath.Split(path) - files, err := List(ctx, storage, dir, model.ListArgs{}) - if err != nil { - return nil, errors.WithMessage(err, "failed get parent list") - } - for _, f := range files { - if f.GetName() == name { - return f, nil + return nil + }) + if obj != nil { + return obj, nil } } log.Debugf("cant find obj with name: %s", name) @@ -245,313 +215,368 @@ func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, er } func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { - obj, err := Get(ctx, storage, path) + obj, err := Get(ctx, storage, path, true) if err != nil { return nil, err } - return model.UnwrapObj(obj), err + return model.UnwrapObjName(obj), err } -var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16)) -var linkG = singleflight.Group[*model.Link]{Remember: true} -var errLinkMFileCache = stderrors.New("ErrLinkMFileCache") +var linkG = singleflight.Group[*objWithLink]{} // Link get link, if is an url. should have an expiry time func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - var ( - file model.Obj - err error - ) - // use cache directly - dir, name := stdpath.Split(stdpath.Join(storage.GetStorage().MountPath, path)) - if cacheFiles, ok := listCache.Get(strings.TrimSuffix(dir, "/")); ok { - for _, f := range cacheFiles { - if f.GetName() == name { - file = model.UnwrapObj(f) - break - } - } - } else { - if g, ok := storage.(driver.GetObjInfo); ok { - file, err = g.GetObjInfo(ctx, path) - } else { - file, err = GetUnwrap(ctx, storage, path) - } + return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } - if file == nil { - if err != nil { - return nil, nil, errors.WithMessage(err, "failed to get file") - } - return nil, nil, errors.WithStack(errs.ObjectNotFound) + + mode := storage.Config().LinkCacheMode + if mode == -1 { + mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path) } - if file.IsDir() { - return nil, nil, errors.WithStack(errs.NotFile) + typeKey := args.Type + if mode&driver.LinkCacheIP != 0 { + typeKey += "/" + args.IP } - - key := stdpath.Join(Key(storage, path), args.Type) - if link, ok := linkCache.Get(key); ok { - return link, file, nil + if mode&driver.LinkCacheUA != 0 { + typeKey += "/" + args.Header.Get("User-Agent") } + key := Key(storage, path) + if ol, exists := Cache.linkCache.GetType(key, typeKey); exists { + if ol.link.Expiration != nil || + ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil + } + } + + fn := func() (*objWithLink, error) { + file, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, errors.WithMessage(err, "failed to get file") + } + if file.IsDir() { + return nil, errors.WithStack(errs.NotFile) + } - var forget any - var linkM *model.Link - fn := func() (*model.Link, error) { link, err := storage.Link(ctx, file, args) if err != nil { return nil, errors.Wrapf(err, "failed get link") } - if link.MFile != nil && forget != nil { - linkM = link - return nil, errLinkMFileCache - } + ol := &objWithLink{link: link, obj: file} if link.Expiration != nil { - linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration)) + Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) + } else { + Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) } - link.AddIfCloser(forget) - return link, nil + return ol, nil } - - if storage.Config().OnlyLinkMFile { - link, err := fn() + for { + ol, err, _ := linkG.Do(key+"/"+typeKey, fn) if err != nil { return nil, nil, err } - return link, file, err - } - - forget = utils.CloseFunc(func() error { - if forget != nil { - forget = nil - linkG.Forget(key) + if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil } - return nil - }) - link, err, _ := linkG.Do(key, fn) - for err == nil && !link.AcquireReference() { - link, err, _ = linkG.Do(key, fn) } - - if err == errLinkMFileCache { - if linkM != nil { - return linkM, file, nil - } - forget = nil - link, err = fn() - } - - if err != nil { - return nil, nil, err - } - return link, file, nil } // Other api -func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) { +func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (any, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) + } + o, ok := storage.(driver.Other) + if !ok { + return nil, errs.NotImplement + } obj, err := GetUnwrap(ctx, storage, args.Path) if err != nil { return nil, errors.WithMessagef(err, "failed to get obj") } - if o, ok := storage.(driver.Other); ok { - return o.Other(ctx, model.OtherArgs{ - Obj: obj, - Method: args.Method, - Data: args.Data, - }) - } else { - return nil, errs.NotImplement - } + return o.Other(ctx, model.OtherArgs{ + Obj: obj, + Method: args.Method, + Data: args.Data, + }) } -var mkdirG singleflight.Group[interface{}] +var mkdirG singleflight.Group[any] -func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error { +func MakeDir(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) key := Key(storage, path) - _, err, _ := mkdirG.Do(key, func() (interface{}, error) { + _, err, _ := mkdirG.Do(key, func() (any, error) { // check if dir exists - f, err := GetUnwrap(ctx, storage, path) - if err != nil { - if errs.IsObjectNotFound(err) { - parentPath, dirName := stdpath.Split(path) - err = MakeDir(ctx, storage, parentPath) - if err != nil { - return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) - } - parentDir, err := GetUnwrap(ctx, storage, parentPath) - // this should not happen - if err != nil { - return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) - } - - switch s := storage.(type) { - case driver.MkdirResult: - var newObj model.Obj - newObj, err = s.MakeDir(ctx, parentDir, dirName) - if err == nil { - if newObj != nil { - addCacheObj(storage, parentPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, parentPath) - } - } - case driver.Mkdir: - err = s.MakeDir(ctx, parentDir, dirName) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, parentPath) - } - default: - return nil, errs.NotImplement - } - return nil, errors.WithStack(err) + f, err := Get(ctx, storage, path) + if err == nil { + if f.IsDir() { + return nil, nil } + return nil, errors.New("file exists") + } + if !errs.IsObjectNotFound(err) { return nil, errors.WithMessage(err, "failed to check if dir exists") } - // dir exists - if f.IsDir() { + parentPath, dirName := stdpath.Split(path) + if err = MakeDir(ctx, storage, parentPath); err != nil { + return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) + } + parentDir, err := GetUnwrap(ctx, storage, parentPath) + // this should not happen + if err != nil { + return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } + if model.ObjHasMask(parentDir, model.NoWrite) { + return nil, errors.WithStack(errs.PermissionDenied) + } + + var newObj model.Obj + switch s := storage.(type) { + case driver.MkdirResult: + newObj, err = s.MakeDir(ctx, parentDir, dirName) + case driver.Mkdir: + err = s.MakeDir(ctx, parentDir, dirName) + default: + return nil, errs.NotImplement + } + if err != nil { + return nil, errors.WithStack(err) + } + if storage.Config().NoCache { return nil, nil } - // dir to make is a file - return nil, errors.New("file exists") + if dirCache, exist := Cache.dirCache.Get(Key(storage, parentPath)); exist { + if newObj == nil { + t := time.Now() + newObj = &model.Object{ + Name: dirName, + IsFolder: true, + Modified: t, + Ctime: t, + Mask: model.Temp, + } + } + dirCache.UpdateObject("", wrapObjName(storage, newObj)) + } + return nil, nil }) return err } -func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { +func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) + if utils.PathEqual(srcPath, "/") { + return errors.New("move root folder is not allowed") + } + srcDirPath := stdpath.Dir(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) - srcRawObj, err := Get(ctx, storage, srcPath) + if dstDirPath == srcDirPath { + return errors.New("move in place") + } + srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { return errors.WithMessage(err, "failed to get src object") } - srcObj := model.UnwrapObj(srcRawObj) + if model.ObjHasMask(srcRawObj, model.NoMove) { + return errors.WithStack(errs.PermissionDenied) + } + srcObj := model.UnwrapObjName(srcRawObj) dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } - srcDirPath := stdpath.Dir(srcPath) + if model.ObjHasMask(dstDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } + var newObj model.Obj switch s := storage.(type) { case driver.MoveResult: - var newObj model.Obj newObj, err = s.Move(ctx, srcObj, dstDir) - if err == nil { - delCacheObj(storage, srcDirPath, srcRawObj) - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } - } case driver.Move: err = s.Move(ctx, srcObj, dstDir) - if err == nil { - delCacheObj(storage, srcDirPath, srcRawObj) - if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) + default: + err = errs.NotImplement + } + if err != nil { + return errors.WithStack(err) + } + + srcKey := Key(storage, srcDirPath) + dstKey := Key(storage, dstDirPath) + if !srcRawObj.IsDir() { + Cache.linkCache.DeleteKey(stdpath.Join(srcKey, srcRawObj.GetName())) + Cache.linkCache.DeleteKey(stdpath.Join(dstKey, srcRawObj.GetName())) + } + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(srcKey); exist { + if srcRawObj.IsDir() { + Cache.deleteDirectoryTree(stdpath.Join(srcKey, srcRawObj.GetName())) } + cache.RemoveObject(srcRawObj.GetName()) + } + if cache, exist := Cache.dirCache.Get(dstKey); exist { + if newObj == nil { + newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp} + } else { + newObj = wrapObjName(storage, newObj) + } + cache.UpdateObject(srcRawObj.GetName(), newObj) } - default: - return errs.NotImplement } - return errors.WithStack(err) + + if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { + return nil + } + if !srcObj.IsDir() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } else { + go objsUpdateHook(context.WithoutCancel(ctx), storage, stdpath.Join(dstDirPath, srcObj.GetName()), true) + } + return nil } -func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error { +func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) - srcRawObj, err := Get(ctx, storage, srcPath) + if utils.PathEqual(srcPath, "/") { + return errors.New("rename root folder is not allowed") + } + srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { return errors.WithMessage(err, "failed to get src object") } - srcObj := model.UnwrapObj(srcRawObj) - srcDirPath := stdpath.Dir(srcPath) + if model.ObjHasMask(srcRawObj, model.NoRename) { + return errors.WithStack(errs.PermissionDenied) + } + srcObj := model.UnwrapObjName(srcRawObj) + var newObj model.Obj switch s := storage.(type) { case driver.RenameResult: - var newObj model.Obj newObj, err = s.Rename(ctx, srcObj, dstName) - if err == nil { - if newObj != nil { - updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, srcDirPath) - if srcRawObj.IsDir() { - ClearCache(storage, srcPath) - } - } - } case driver.Rename: err = s.Rename(ctx, srcObj, dstName) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, srcDirPath) + default: + return errs.NotImplement + } + if err != nil { + return errors.WithStack(err) + } + + dirKey := Key(storage, stdpath.Dir(srcPath)) + if !srcRawObj.IsDir() { + Cache.linkCache.DeleteKey(stdpath.Join(dirKey, srcRawObj.GetName())) + Cache.linkCache.DeleteKey(stdpath.Join(dirKey, dstName)) + } + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(dirKey); exist { if srcRawObj.IsDir() { - ClearCache(storage, srcPath) + Cache.deleteDirectoryTree(stdpath.Join(dirKey, srcRawObj.GetName())) + } + if newObj == nil { + newObj = &model.ObjWrapMask{Obj: &model.ObjWrapName{Name: dstName, Obj: srcObj}, Mask: model.Temp} } + newObj = wrapObjName(storage, newObj) + cache.UpdateObject(srcRawObj.GetName(), newObj) } - default: - return errs.NotImplement } - return errors.WithStack(err) + + if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { + return nil + } + dstDirPath := stdpath.Dir(srcPath) + if !srcObj.IsDir() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } else { + go objsUpdateHook(context.WithoutCancel(ctx), storage, stdpath.Join(dstDirPath, srcObj.GetName()), true) + } + return nil } // Copy Just copy file[s] in a storage -func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { +func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) - srcObj, err := GetUnwrap(ctx, storage, srcPath) + if dstDirPath == stdpath.Dir(srcPath) { + return errors.New("copy in place") + } + srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { return errors.WithMessage(err, "failed to get src object") } + // if model.ObjHasMask(srcRawObj, model.NoCopy) { + // return errors.WithStack(errs.PermissionDenied) + // } + srcObj := model.UnwrapObjName(srcRawObj) dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } + if model.ObjHasMask(dstDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } + var newObj model.Obj switch s := storage.(type) { case driver.CopyResult: - var newObj model.Obj newObj, err = s.Copy(ctx, srcObj, dstDir) - if err == nil { - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } - } case driver.Copy: err = s.Copy(ctx, srcObj, dstDir) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } default: - return errs.NotImplement + err = errs.NotImplement } - return errors.WithStack(err) + if err != nil { + return errors.WithStack(err) + } + + dstKey := Key(storage, dstDirPath) + if !srcRawObj.IsDir() { + Cache.linkCache.DeleteKey(stdpath.Join(dstKey, srcRawObj.GetName())) + } + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(dstKey); exist { + if newObj == nil { + newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp} + } else { + newObj = wrapObjName(storage, newObj) + } + cache.UpdateObject(srcRawObj.GetName(), newObj) + } + } + + if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { + return nil + } + if !srcObj.IsDir() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } else { + go objsUpdateHook(context.WithoutCancel(ctx), storage, stdpath.Join(dstDirPath, srcObj.GetName()), true) + } + return nil } func Remove(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } + path = utils.FixAndCleanPath(path) if utils.PathEqual(path, "/") { - return errors.New("delete root folder is not allowed, please goto the manage page to delete the storage instead") + return errors.New("delete root folder is not allowed") } - path = utils.FixAndCleanPath(path) - rawObj, err := Get(ctx, storage, path) + rawObj, err := Get(ctx, storage, path, true) if err != nil { // if object not found, it's ok if errs.IsObjectNotFound(err) { @@ -560,17 +585,16 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { } return errors.WithMessage(err, "failed to get object") } + if model.ObjHasMask(rawObj, model.NoRemove) { + return errors.WithStack(errs.PermissionDenied) + } dirPath := stdpath.Dir(path) switch s := storage.(type) { case driver.Remove: - err = s.Remove(ctx, model.UnwrapObj(rawObj)) + err = s.Remove(ctx, model.UnwrapObjName(rawObj)) if err == nil { - delCacheObj(storage, dirPath, rawObj) - // clear folder cache recursively - if rawObj.IsDir() { - ClearCache(storage, path) - } + Cache.removeDirectoryObject(storage, dirPath, rawObj) } default: return errs.NotImplement @@ -578,21 +602,20 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { return errors.WithStack(err) } -func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress, lazyCache ...bool) error { - close := file.Close +func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error { defer func() { - if err := close(); err != nil { + if err := file.Close(); err != nil { log.Errorf("failed to close file streamer, %v", err) } }() if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } // UrlTree PUT - if storage.GetStorage().Driver == "UrlTree" { + if storage.Config().OnlyIndices { var link string dstDirPath, link = urlTreeSplitLineFormPath(stdpath.Join(dstDirPath, file.GetName())) - file = &stream.FileStream{Obj: &model.Object{Name: link}} + file = &stream.FileStream{Obj: &model.Object{Name: link}, Closers: utils.Closers{file}} } // if file exist and size = 0, delete it dstDirPath = utils.FixAndCleanPath(dstDirPath) @@ -625,6 +648,9 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod if err != nil { return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) } + if model.ObjHasMask(parentDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } // if up is nil, set a default to prevent panic if up == nil { up = func(p float64) {} @@ -635,25 +661,38 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod log.Warnf("file size < 0, try to get full size from cache") file.CacheFullAndWriter(nil, nil) } + + var newObj model.Obj switch s := storage.(type) { case driver.PutResult: - var newObj model.Obj newObj, err = s.Put(ctx, parentDir, file, up) - if err == nil { - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } - } case driver.Put: err = s.Put(ctx, parentDir, file, up) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } default: return errs.NotImplement } + if err == nil { + Cache.linkCache.DeleteKey(Key(storage, dstPath)) + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { + if newObj == nil { + newObj = &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + Modified: file.ModTime(), + Ctime: file.CreateTime(), + Mask: model.Temp, + } + } + newObj = wrapObjName(storage, newObj) + cache.UpdateObject(newObj.GetName(), newObj) + } + } + + if ctx.Value(conf.SkipHookKey) == nil && needHandleObjsUpdateHook() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } + } log.Debugf("put file [%s] done", file.GetName()) if storage.Config().NoOverwriteUpload && fi != nil && fi.GetSize() > 0 { if err != nil { @@ -664,54 +703,163 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod } } else { // upload success, remove old obj - err := Remove(ctx, storage, tempPath) - if err != nil { - return err - } else { - key := Key(storage, stdpath.Join(dstDirPath, file.GetName())) - linkCache.Del(key) - } + err = Remove(ctx, storage, tempPath) } } return errors.WithStack(err) } -func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string, lazyCache ...bool) error { +func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } dstDirPath = utils.FixAndCleanPath(dstDirPath) - _, err := GetUnwrap(ctx, storage, stdpath.Join(dstDirPath, dstName)) - if err == nil { - return errors.New("obj already exists") + dstPath := stdpath.Join(dstDirPath, dstName) + + if _, err := Get(ctx, storage, dstPath); err == nil { + return errors.WithStack(errs.ObjectAlreadyExists) } - err = MakeDir(ctx, storage, dstDirPath) + err := MakeDir(ctx, storage, dstDirPath) if err != nil { - return errors.WithMessagef(err, "failed to put url") + return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) } dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { - return errors.WithMessagef(err, "failed to put url") + return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) + } + if model.ObjHasMask(dstDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) } + var newObj model.Obj switch s := storage.(type) { case driver.PutURLResult: - var newObj model.Obj newObj, err = s.PutURL(ctx, dstDir, dstName, url) - if err == nil { - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } - } case driver.PutURL: err = s.PutURL(ctx, dstDir, dstName, url) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } default: - return errs.NotImplement + return errors.WithStack(errs.NotImplement) + } + if err == nil { + Cache.linkCache.DeleteKey(Key(storage, dstPath)) + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { + if newObj == nil { + t := time.Now() + newObj = &model.Object{ + Name: dstName, + Modified: t, + Ctime: t, + Mask: model.Temp, + } + } + newObj = wrapObjName(storage, newObj) + cache.UpdateObject(newObj.GetName(), newObj) + } + + if ctx.Value(conf.SkipHookKey) == nil && needHandleObjsUpdateHook() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } + } } log.Debugf("put url [%s](%s) done", dstName, url) return errors.WithStack(err) } + +func GetDirectUploadTools(storage driver.Driver) []string { + du, ok := storage.(driver.DirectUploader) + if !ok { + return nil + } + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil + } + return du.GetDirectUploadTools() +} + +func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver, dstDirPath, dstName string, fileSize int64) (any, error) { + du, ok := storage.(driver.DirectUploader) + if !ok { + return nil, errors.WithStack(errs.NotImplement) + } + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) + } + dstDirPath = utils.FixAndCleanPath(dstDirPath) + dstPath := stdpath.Join(dstDirPath, dstName) + _, err := Get(ctx, storage, dstPath) + if err == nil { + return nil, errors.WithStack(errs.ObjectAlreadyExists) + } + err = MakeDir(ctx, storage, dstDirPath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) + } + info, err := du.GetDirectUploadInfo(ctx, tool, dstDir, dstName, fileSize) + if err != nil { + return nil, errors.WithStack(err) + } + return info, nil +} + +func objsUpdateHook(ctx context.Context, storage driver.Driver, dirPath string, recursive bool) { + files, err := List(ctx, storage, dirPath, model.ListArgs{SkipHook: true}) + if err != nil { + return + } + if !recursive { + HandleObjsUpdateHook(ctx, utils.GetFullPath(storage.GetStorage().MountPath, dirPath), files) + return + } + var limiter *rate.Limiter + if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil { + if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 { + limiter = rate.NewLimiter(rate.Limit(f), 1) + } + } + recursivelyObjsUpdateHook(ctx, storage, dirPath, files, limiter) +} +func recursivelyObjsUpdateHook(ctx context.Context, storage driver.Driver, dirPath string, files []model.Obj, limiter *rate.Limiter) { + HandleObjsUpdateHook(ctx, utils.GetFullPath(storage.GetStorage().MountPath, dirPath), files) + for _, f := range files { + if utils.IsCanceled(ctx) { + return + } + if !f.IsDir() { + continue + } + dstPath := stdpath.Join(dirPath, f.GetName()) + if limiter != nil { + if err := limiter.Wait(ctx); err != nil { + return + } + } + files, err := List(ctx, storage, dstPath, model.ListArgs{SkipHook: true}) + if err == nil { + recursivelyObjsUpdateHook(ctx, storage, dstPath, files, limiter) + } + } +} + +func needHandleObjsUpdateHook() bool { + if len(objsUpdateHooks) < 1 { + return false + } + needHandle, _ := GetSettingItemByKey(conf.HandleHookAfterWriting) + return needHandle != nil && (needHandle.Value == "true" || needHandle.Value == "1") +} + +func wrapObjsName(storage driver.Driver, objs []model.Obj) { + if _, ok := storage.(driver.Getter); !ok { + model.WrapObjsName(objs) + } +} +func wrapObjName(storage driver.Driver, obj model.Obj) model.Obj { + if _, ok := storage.(driver.Getter); !ok { + return model.WrapObjName(obj) + } + return obj +} diff --git a/internal/op/hook.go b/internal/op/hook.go index 22a069985..5cf01730d 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -1,6 +1,7 @@ package op import ( + "context" "regexp" "strings" @@ -13,7 +14,7 @@ import ( ) // Obj -type ObjsUpdateHook = func(parent string, objs []model.Obj) +type ObjsUpdateHook = func(ctx context.Context, parent string, objs []model.Obj) var ( objsUpdateHooks = make([]ObjsUpdateHook, 0) @@ -23,9 +24,9 @@ func RegisterObjsUpdateHook(hook ObjsUpdateHook) { objsUpdateHooks = append(objsUpdateHooks, hook) } -func HandleObjsUpdateHook(parent string, objs []model.Obj) { +func HandleObjsUpdateHook(ctx context.Context, parent string, objs []model.Obj) { for _, hook := range objsUpdateHooks { - hook(parent, objs) + hook(ctx, parent, objs) } } diff --git a/internal/op/recursive_list.go b/internal/op/recursive_list.go new file mode 100644 index 000000000..de8a27b2b --- /dev/null +++ b/internal/op/recursive_list.go @@ -0,0 +1,125 @@ +package op + +import ( + "context" + stdpath "path" + "sync" + "sync/atomic" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" +) + +var ( + ManualScanCancel = atomic.Pointer[context.CancelFunc]{} + ScannedCount = atomic.Uint64{} +) + +func ManualScanRunning() bool { + return ManualScanCancel.Load() != nil +} + +func BeginManualScan(rawPath string, limit float64) error { + rawPath = utils.FixAndCleanPath(rawPath) + ctx, cancel := context.WithCancel(context.Background()) + if !ManualScanCancel.CompareAndSwap(nil, &cancel) { + cancel() + return errors.New("manual scan is running, please try later") + } + ScannedCount.Store(0) + go func() { + defer func() { (*ManualScanCancel.Swap(nil))() }() + err := RecursivelyList(ctx, rawPath, rate.Limit(limit), &ScannedCount) + if err != nil { + log.Errorf("failed recursively list: %v", err) + } + }() + return nil +} + +func StopManualScan() { + c := ManualScanCancel.Load() + if c != nil { + (*c)() + } +} + +func RecursivelyList(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64) error { + storage, actualPath, err := GetStorageAndActualPath(rawPath) + if err != nil && !errors.Is(err, errs.StorageNotFound) { + return err + } else if err == nil { + var limiter *rate.Limiter + if limit > .0 { + limiter = rate.NewLimiter(limit, 1) + } + RecursivelyListStorage(ctx, storage, actualPath, limiter, counter) + } else { + var wg sync.WaitGroup + recursivelyListVirtual(ctx, rawPath, limit, counter, &wg) + wg.Wait() + } + return nil +} + +func recursivelyListVirtual(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64, wg *sync.WaitGroup) { + objs := GetStorageVirtualFilesByPath(rawPath) + if counter != nil { + counter.Add(uint64(len(objs))) + } + for _, obj := range objs { + if utils.IsCanceled(ctx) { + return + } + nextPath := stdpath.Join(rawPath, obj.GetName()) + storage, actualPath, err := GetStorageAndActualPath(nextPath) + if err != nil && !errors.Is(err, errs.StorageNotFound) { + log.Errorf("error recursively list: failed get storage [%s]: %v", nextPath, err) + } else if err == nil { + var limiter *rate.Limiter + if limit > .0 { + limiter = rate.NewLimiter(limit, 1) + } + wg.Add(1) + go func() { + defer wg.Done() + RecursivelyListStorage(ctx, storage, actualPath, limiter, counter) + }() + } else { + recursivelyListVirtual(ctx, nextPath, limit, counter, wg) + } + } +} + +func RecursivelyListStorage(ctx context.Context, storage driver.Driver, actualPath string, limiter *rate.Limiter, counter *atomic.Uint64) { + objs, err := List(ctx, storage, actualPath, model.ListArgs{Refresh: true}) + if err != nil { + if !errors.Is(err, context.Canceled) { + log.Errorf("error recursively list: failed list (%s)[%s]: %v", storage.GetStorage().MountPath, actualPath, err) + } + return + } + if counter != nil { + counter.Add(uint64(len(objs))) + } + for _, obj := range objs { + if utils.IsCanceled(ctx) { + return + } + if !obj.IsDir() { + continue + } + if limiter != nil { + if err = limiter.Wait(ctx); err != nil { + return + } + } + nextPath := stdpath.Join(actualPath, obj.GetName()) + RecursivelyListStorage(ctx, storage, nextPath, limiter, counter) + } +} diff --git a/internal/op/setting.go b/internal/op/setting.go index cbfa083ff..bda4daaf6 100644 --- a/internal/op/setting.go +++ b/internal/op/setting.go @@ -5,26 +5,21 @@ import ( "sort" "strconv" "strings" - "time" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/go-cache" "github.com/pkg/errors" ) -var settingCache = cache.NewMemCache(cache.WithShards[*model.SettingItem](4)) var settingG singleflight.Group[*model.SettingItem] var settingCacheF = func(item *model.SettingItem) { - settingCache.Set(item.Key, item, cache.WithEx[*model.SettingItem](time.Hour)) + Cache.SetSetting(item.Key, item) } -var settingGroupCache = cache.NewMemCache(cache.WithShards[[]model.SettingItem](4)) var settingGroupG singleflight.Group[[]model.SettingItem] -var settingGroupCacheF = func(key string, item []model.SettingItem) { - settingGroupCache.Set(key, item, cache.WithEx[[]model.SettingItem](time.Hour)) +var settingGroupCacheF = func(key string, items []model.SettingItem) { + Cache.SetSettingGroup(key, items) } var settingChangingCallbacks = make([]func(), 0) @@ -34,8 +29,7 @@ func RegisterSettingChangingCallback(f func()) { } func SettingCacheUpdate() { - settingCache.Clear() - settingGroupCache.Clear() + Cache.ClearAll() for _, cb := range settingChangingCallbacks { cb() } @@ -60,7 +54,7 @@ func GetSettingsMap() map[string]string { } func GetSettingItems() ([]model.SettingItem, error) { - if items, ok := settingGroupCache.Get("ALL_SETTING_ITEMS"); ok { + if items, exists := Cache.GetSettingGroup("ALL_SETTING_ITEMS"); exists { return items, nil } items, err, _ := settingGroupG.Do("ALL_SETTING_ITEMS", func() ([]model.SettingItem, error) { @@ -75,7 +69,7 @@ func GetSettingItems() ([]model.SettingItem, error) { } func GetPublicSettingItems() ([]model.SettingItem, error) { - if items, ok := settingGroupCache.Get("ALL_PUBLIC_SETTING_ITEMS"); ok { + if items, exists := Cache.GetSettingGroup("ALL_PUBLIC_SETTING_ITEMS"); exists { return items, nil } items, err, _ := settingGroupG.Do("ALL_PUBLIC_SETTING_ITEMS", func() ([]model.SettingItem, error) { @@ -90,7 +84,7 @@ func GetPublicSettingItems() ([]model.SettingItem, error) { } func GetSettingItemByKey(key string) (*model.SettingItem, error) { - if item, ok := settingCache.Get(key); ok { + if item, exists := Cache.GetSetting(key); exists { return item, nil } @@ -118,8 +112,8 @@ func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) { } func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) { - key := strconv.Itoa(group) - if items, ok := settingGroupCache.Get(key); ok { + key := fmt.Sprintf("GROUP_%d", group) + if items, exists := Cache.GetSettingGroup(key); exists { return items, nil } items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) { @@ -135,11 +129,14 @@ func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) { func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) { sort.Ints(groups) - key := strings.Join(utils.MustSliceConvert(groups, func(i int) string { - return strconv.Itoa(i) - }), ",") - if items, ok := settingGroupCache.Get(key); ok { + keyParts := make([]string, 0, len(groups)) + for _, g := range groups { + keyParts = append(keyParts, strconv.Itoa(g)) + } + key := "GROUPS_" + strings.Join(keyParts, "_") + + if items, exists := Cache.GetSettingGroup(key); exists { return items, nil } items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) { @@ -165,10 +162,10 @@ func SaveSettingItems(items []model.SettingItem) error { } } err := db.SaveSettingItems(items) - if err != nil { + if err != nil { return fmt.Errorf("failed save setting: %+v", err) } - SettingCacheUpdate() + SettingCacheUpdate() return nil } diff --git a/internal/op/sharing.go b/internal/op/sharing.go index dbff1ba89..9db51c59d 100644 --- a/internal/op/sharing.go +++ b/internal/op/sharing.go @@ -137,3 +137,7 @@ func DeleteSharing(sid string) error { sharingCache.Del(sid) return db.DeleteSharingById(sid) } + +func DeleteSharingsByCreatorId(creatorId uint) error { + return db.DeleteSharingsByCreatorId(creatorId) +} diff --git a/internal/op/storage.go b/internal/op/storage.go index 53784802c..da4c84e31 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -15,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/generic_sync" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -239,6 +240,8 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) + Cache.DeleteDirectoryTree(storageDriver, "/") + Cache.InvalidateStorageDetails(storageDriver) } if err != nil { return errors.WithMessage(err, "failed get storage driver") @@ -259,6 +262,7 @@ func DeleteStorageById(ctx context.Context, id uint) error { if err != nil { return errors.WithMessage(err, "failed get storage") } + var dropErr error = nil if !storage.Disabled { storageDriver, err := GetStorageByMountPath(storage.MountPath) if err != nil { @@ -266,17 +270,19 @@ func DeleteStorageById(ctx context.Context, id uint) error { } // drop the storage in the driver if err := storageDriver.Drop(ctx); err != nil { - return errors.Wrapf(err, "failed drop storage") + dropErr = errors.Wrapf(err, "failed drop storage") } // delete the storage in the memory storagesMap.Delete(storage.MountPath) + Cache.DeleteDirectoryTree(storageDriver, "/") + Cache.InvalidateStorageDetails(storageDriver) go callStorageHooks("del", storageDriver) } // delete the storage in the database if err := db.DeleteStorageById(id); err != nil { return errors.WithMessage(err, "failed delete storage in database") } - return nil + return dropErr } // MustSaveDriverStorage call from specific driver @@ -335,38 +341,41 @@ func getStoragesByPath(path string) []driver.Driver { // for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av // GetStorageVirtualFilesByPath(/a) => b,c,d func GetStorageVirtualFilesByPath(prefix string) []model.Obj { - return getStorageVirtualFilesByPath(prefix, func(_ driver.Driver, obj model.Obj) model.Obj { - return obj - }) + return getStorageVirtualFilesByPath(prefix, nil, "") } -func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, hideDetails ...bool) []model.Obj { - if utils.IsBool(hideDetails...) { - return GetStorageVirtualFilesByPath(prefix) +func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, hideDetails, refresh bool, filterByName string) []model.Obj { + if hideDetails { + return getStorageVirtualFilesByPath(prefix, nil, filterByName) } return getStorageVirtualFilesByPath(prefix, func(d driver.Driver, obj model.Obj) model.Obj { + if _, ok := obj.(*model.ObjStorageDetails); ok { + return obj + } ret := &model.ObjStorageDetails{ - Obj: obj, - StorageDetailsWithName: model.StorageDetailsWithName{ - StorageDetails: nil, - DriverName: d.Config().Name, - }, + Obj: obj, + StorageDetails: nil, } - timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - details, err := GetStorageDetails(timeoutCtx, d) - if err != nil { - if !errors.Is(err, errs.NotImplement) { - log.Errorf("failed get %s storage details: %+v", d.GetStorage().MountPath, err) + resultChan := make(chan *model.StorageDetails, 1) + go func(dri driver.Driver) { + details, err := GetStorageDetails(ctx, dri, refresh) + if err != nil { + if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) { + log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, err) + } } - return ret + resultChan <- details + }(d) + select { + case r := <-resultChan: + ret.StorageDetails = r + case <-time.After(time.Second): } - ret.StorageDetails = details return ret - }) + }, filterByName) } -func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver, model.Obj) model.Obj) []model.Obj { +func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver, model.Obj) model.Obj, filterByName string) []model.Obj { files := make([]model.Obj, 0) storages := storagesMap.Values() sort.Slice(storages, func(i, j int) bool { @@ -376,45 +385,60 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver return storages[i].GetStorage().Order < storages[j].GetStorage().Order }) - prefix = utils.FixAndCleanPath(prefix) + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } set := make(map[string]int) var wg sync.WaitGroup for _, v := range storages { - mountPath := utils.GetActualMountPath(v.GetStorage().MountPath) // Exclude prefix itself and non prefix - if len(prefix) >= len(mountPath) || !utils.IsSubPath(prefix, mountPath) { + p, found := strings.CutPrefix(utils.GetActualMountPath(v.GetStorage().MountPath), prefix) + if !found || p == "" { + continue + } + name, _, found := strings.Cut(p, "/") + if filterByName != "" && name != filterByName { continue } - names := strings.SplitN(strings.TrimPrefix(mountPath[len(prefix):], "/"), "/", 2) - idx, ok := set[names[0]] - if !ok { - set[names[0]] = len(files) - obj := &model.Object{ - Name: names[0], - Size: 0, - Modified: v.GetStorage().Modified, - IsFolder: true, + + if idx, ok := set[name]; ok { + if !found { + files[idx].(*model.Object).Mask = model.Locked | model.Virtual + if rootCallback != nil { + wg.Add(1) + go func() { + defer wg.Done() + files[idx] = rootCallback(v, files[idx]) + }() + } } - if len(names) == 1 { - idx = len(files) - files = append(files, obj) + continue + } + set[name] = len(files) + obj := &model.Object{ + Name: name, + Modified: v.GetStorage().Modified, + IsFolder: true, + } + if !found { + idx := len(files) + obj.Mask = model.Locked | model.Virtual + files = append(files, obj) + if rootCallback != nil { wg.Add(1) go func() { defer wg.Done() files[idx] = rootCallback(v, files[idx]) }() - } else { - files = append(files, obj) } - } else if len(names) == 1 { - wg.Add(1) - go func() { - defer wg.Done() - files[idx] = rootCallback(v, files[idx]) - }() + } else { + obj.Mask = model.ReadOnly | model.Virtual + files = append(files, obj) } } - wg.Wait() + if rootCallback != nil { + wg.Wait() + } return files } @@ -439,10 +463,28 @@ func GetBalancedStorage(path string) driver.Driver { } } -func GetStorageDetails(ctx context.Context, storage driver.Driver) (*model.StorageDetails, error) { +var detailsG singleflight.Group[*model.StorageDetails] + +func GetStorageDetails(ctx context.Context, storage driver.Driver, refresh ...bool) (*model.StorageDetails, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) + } wd, ok := storage.(driver.WithDetails) if !ok { return nil, errs.NotImplement } - return wd.GetDetails(ctx) + if !utils.IsBool(refresh...) { + if ret, ok := Cache.GetStorageDetails(storage); ok { + return ret, nil + } + } + details, err, _ := detailsG.Do(storage.GetStorage().MountPath, func() (*model.StorageDetails, error) { + ret, err := wd.GetDetails(ctx) + if err != nil { + return nil, err + } + Cache.SetStorageDetails(storage, ret) + return ret, nil + }) + return details, err } diff --git a/internal/op/user.go b/internal/op/user.go index 3f526625b..dc2a0ce3e 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -1,17 +1,14 @@ package op import ( - "time" - "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/go-cache" + "github.com/pkg/errors" ) -var userCache = cache.NewMemCache(cache.WithShards[*model.User](2)) var userG singleflight.Group[*model.User] var guestUser *model.User var adminUser *model.User @@ -46,7 +43,7 @@ func GetUserByName(username string) (*model.User, error) { if username == "" { return nil, errs.EmptyUsername } - if user, ok := userCache.Get(username); ok { + if user, exists := Cache.GetUser(username); exists { return user, nil } user, err, _ := userG.Do(username, func() (*model.User, error) { @@ -54,7 +51,7 @@ func GetUserByName(username string) (*model.User, error) { if err != nil { return nil, err } - userCache.Set(username, _user, cache.WithEx[*model.User](time.Hour)) + Cache.SetUser(username, _user) return _user, nil }) return user, err @@ -81,7 +78,10 @@ func DeleteUserById(id uint) error { if old.IsAdmin() || old.IsGuest() { return errs.DeleteAdminOrGuest } - userCache.Del(old.Username) + Cache.DeleteUser(old.Username) + if err := DeleteSharingsByCreatorId(id); err != nil { + return errors.WithMessage(err, "failed to delete user's sharings") + } return db.DeleteUserById(id) } @@ -96,7 +96,7 @@ func UpdateUser(u *model.User) error { if u.IsGuest() { guestUser = nil } - userCache.Del(old.Username) + Cache.DeleteUser(old.Username) u.BasePath = utils.FixAndCleanPath(u.BasePath) return db.UpdateUser(u) } @@ -125,6 +125,6 @@ func DelUserCache(username string) error { if user.IsGuest() { guestUser = nil } - userCache.Del(username) + Cache.DeleteUser(username) return nil } diff --git a/internal/search/build.go b/internal/search/build.go index c5c74e094..bbed1f853 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -199,14 +199,13 @@ func Config(ctx context.Context) searcher.Config { return instance.Config() } -func Update(parent string, objs []model.Obj) { +func Update(ctx context.Context, parent string, objs []model.Obj) { if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running() { return } if isIgnorePath(parent) { return } - ctx := context.Background() // only update when index have built progress, err := Progress() if err != nil { @@ -216,6 +215,16 @@ func Update(parent string, objs []model.Obj) { if !progress.IsDone { return } + + // Use task queue for Meilisearch to avoid race conditions with async indexing + if msInstance, ok := instance.(interface { + EnqueueUpdate(parent string, objs []model.Obj) + }); ok { + // Enqueue task for async processing (diff calculation happens at consumption time) + msInstance.EnqueueUpdate(parent, objs) + return + } + nodes, err := instance.Get(ctx, parent) if err != nil { log.Errorf("update search index error while get nodes: %+v", err) @@ -242,27 +251,23 @@ func Update(parent string, objs []model.Obj) { } } } + // collect files and folders to add in batch + var toAddObjs []ObjWithParent for i := range objs { if toAdd.Contains(objs[i].GetName()) { - if !objs[i].IsDir() { - log.Debugf("add index: %s", path.Join(parent, objs[i].GetName())) - err = Index(ctx, parent, objs[i]) - if err != nil { - log.Errorf("update search index error while index new node: %+v", err) - return - } - } else { - // build index if it's a folder - dir := path.Join(parent, objs[i].GetName()) - err = BuildIndex(ctx, - []string{dir}, - conf.SlicesMap[conf.IgnorePaths], - setting.GetInt(conf.MaxIndexDepth, 20)-strings.Count(dir, "/"), false) - if err != nil { - log.Errorf("update search index error while build index: %+v", err) - return - } - } + log.Debugf("add index: %s", path.Join(parent, objs[i].GetName())) + toAddObjs = append(toAddObjs, ObjWithParent{ + Parent: parent, + Obj: objs[i], + }) + } + } + // batch index all files and folders at once + if len(toAddObjs) > 0 { + err = BatchIndex(ctx, toAddObjs) + if err != nil { + log.Errorf("update search index error while batch index new nodes: %+v", err) + return } } } diff --git a/internal/search/meilisearch/init.go b/internal/search/meilisearch/init.go index ae736fd97..3c379b790 100644 --- a/internal/search/meilisearch/init.go +++ b/internal/search/meilisearch/init.go @@ -91,6 +91,11 @@ func init() { return nil, err } } + + // Initialize and start task queue manager + m.taskQueue = NewTaskQueueManager(&m) + m.taskQueue.Start() + return &m, nil }) } diff --git a/internal/search/meilisearch/search.go b/internal/search/meilisearch/search.go index fbb8a3f85..75283228c 100644 --- a/internal/search/meilisearch/search.go +++ b/internal/search/meilisearch/search.go @@ -33,6 +33,7 @@ type Meilisearch struct { IndexUid string FilterableAttributes []string SearchableAttributes []string + taskQueue *TaskQueueManager } func (m *Meilisearch) Config() searcher.Config { @@ -82,14 +83,17 @@ func (m *Meilisearch) Index(ctx context.Context, node model.SearchNode) error { } func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { - documents, _ := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { + documents, err := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { parentHash := hashPath(src.Parent) nodePath := path.Join(src.Parent, src.Name) nodePathHash := hashPath(nodePath) parentPaths := utils.GetPathHierarchy(src.Parent) - parentPathHashes, _ := utils.SliceConvert(parentPaths, func(parentPath string) (string, error) { + parentPathHashes, err := utils.SliceConvert(parentPaths, func(parentPath string) (string, error) { return hashPath(parentPath), nil }) + if err != nil { + return nil, err + } return &searchDocument{ ID: nodePathHash, @@ -98,9 +102,12 @@ func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) SearchNode: src, }, nil }) + if err != nil { + return err + } // max up to 10,000 documents per batch to reduce error rate while uploading over the Internet - _, err := m.Client.Index(m.IndexUid).AddDocumentsInBatchesWithContext(ctx, documents, 10000) + _, err = m.Client.Index(m.IndexUid).AddDocumentsInBatchesWithContext(ctx, documents, 10000) if err != nil { return err } @@ -203,6 +210,9 @@ func (m *Meilisearch) Del(ctx context.Context, prefix string) error { } func (m *Meilisearch) Release(ctx context.Context) error { + if m.taskQueue != nil { + m.taskQueue.Stop() + } return nil } @@ -219,3 +229,115 @@ func (m *Meilisearch) getTaskStatus(ctx context.Context, taskUID int64) (meilise } return forTask.Status, nil } + +// EnqueueUpdate enqueues an update task to the task queue +func (m *Meilisearch) EnqueueUpdate(parent string, objs []model.Obj) { + if m.taskQueue == nil { + return + } + + m.taskQueue.Enqueue(parent, objs) +} + +// batchIndexWithTaskUID indexes documents and returns all taskUIDs +func (m *Meilisearch) batchIndexWithTaskUID(ctx context.Context, nodes []model.SearchNode) ([]int64, error) { + if len(nodes) == 0 { + return nil, nil + } + + documents, err := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { + parentHash := hashPath(src.Parent) + nodePath := path.Join(src.Parent, src.Name) + nodePathHash := hashPath(nodePath) + parentPaths := utils.GetPathHierarchy(src.Parent) + parentPathHashes, err := utils.SliceConvert(parentPaths, func(parentPath string) (string, error) { + return hashPath(parentPath), nil + }) + if err != nil { + return nil, err + } + + return &searchDocument{ + ID: nodePathHash, + ParentHash: parentHash, + ParentPathHashes: parentPathHashes, + SearchNode: src, + }, nil + }) + if err != nil { + return nil, err + } + + // max up to 10,000 documents per batch to reduce error rate while uploading over the Internet + tasks, err := m.Client.Index(m.IndexUid).AddDocumentsInBatchesWithContext(ctx, documents, 10000) + if err != nil { + return nil, err + } + + // Return all task UIDs + taskUIDs := make([]int64, 0, len(tasks)) + for _, task := range tasks { + taskUIDs = append(taskUIDs, task.TaskUID) + } + return taskUIDs, nil +} + +// batchDeleteWithTaskUID deletes documents and returns all taskUIDs +func (m *Meilisearch) batchDeleteWithTaskUID(ctx context.Context, paths []string) ([]int64, error) { + if len(paths) == 0 { + return nil, nil + } + + // Deduplicate paths first + pathSet := make(map[string]struct{}) + uniquePaths := make([]string, 0, len(paths)) + for _, p := range paths { + p = utils.FixAndCleanPath(p) + if _, exists := pathSet[p]; !exists { + pathSet[p] = struct{}{} + uniquePaths = append(uniquePaths, p) + } + } + + const batchSize = 100 // max paths per batch to avoid filter length limits + var taskUIDs []int64 + + // Process in batches to avoid filter length limits + for i := 0; i < len(uniquePaths); i += batchSize { + end := i + batchSize + if end > len(uniquePaths) { + end = len(uniquePaths) + } + batch := uniquePaths[i:end] + + // Build combined filter to delete all children in one request + // Format: parent_path_hashes = 'hash1' OR parent_path_hashes = 'hash2' OR ... + var filters []string + for _, p := range batch { + pathHash := hashPath(p) + filters = append(filters, fmt.Sprintf("parent_path_hashes = '%s'", pathHash)) + } + if len(filters) > 0 { + combinedFilter := strings.Join(filters, " OR ") + // Delete all children for all paths in one request + task, err := m.Client.Index(m.IndexUid).DeleteDocumentsByFilterWithContext(ctx, combinedFilter) + if err != nil { + return nil, err + } + taskUIDs = append(taskUIDs, task.TaskUID) + } + + // Convert paths to document IDs and batch delete + documentIDs := make([]string, 0, len(batch)) + for _, p := range batch { + documentIDs = append(documentIDs, hashPath(p)) + } + // Use batch delete API + task, err := m.Client.Index(m.IndexUid).DeleteDocumentsWithContext(ctx, documentIDs) + if err != nil { + return nil, err + } + taskUIDs = append(taskUIDs, task.TaskUID) + } + return taskUIDs, nil +} diff --git a/internal/search/meilisearch/task_queue.go b/internal/search/meilisearch/task_queue.go new file mode 100644 index 000000000..c5384e6e5 --- /dev/null +++ b/internal/search/meilisearch/task_queue.go @@ -0,0 +1,265 @@ +package meilisearch + +import ( + "context" + "path" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + mapset "github.com/deckarep/golang-set/v2" + log "github.com/sirupsen/logrus" +) + +// QueuedTask represents a task in the queue +type QueuedTask struct { + Parent string + Objs []model.Obj // current file system state + Depth int // path depth for sorting + EnqueueAt time.Time // enqueue time +} + +// TaskQueueManager manages the task queue for async index operations +type TaskQueueManager struct { + queue map[string]*QueuedTask // parent -> task + pendingTasks map[string][]int64 // parent -> all submitted taskUIDs + mu sync.RWMutex + ticker *time.Ticker + stopCh chan struct{} + m *Meilisearch + consuming atomic.Bool // flag to prevent concurrent consumption +} + +// NewTaskQueueManager creates a new task queue manager +func NewTaskQueueManager(m *Meilisearch) *TaskQueueManager { + return &TaskQueueManager{ + queue: make(map[string]*QueuedTask), + pendingTasks: make(map[string][]int64), + stopCh: make(chan struct{}), + m: m, + } +} + +// calculateDepth calculates the depth of a path +func calculateDepth(path string) int { + if path == "/" { + return 0 + } + return strings.Count(strings.Trim(path, "/"), "/") + 1 +} + +// Enqueue enqueues a task with current file system state +func (tqm *TaskQueueManager) Enqueue(parent string, objs []model.Obj) { + tqm.mu.Lock() + defer tqm.mu.Unlock() + + // deduplicate: overwrite existing task with the same parent + tqm.queue[parent] = &QueuedTask{ + Parent: parent, + Objs: objs, + Depth: calculateDepth(parent), + EnqueueAt: time.Now(), + } + log.Debugf("enqueued update task for parent: %s, depth: %d, objs: %d", parent, calculateDepth(parent), len(objs)) +} + +// Start starts the task queue consumer +func (tqm *TaskQueueManager) Start() { + tqm.ticker = time.NewTicker(30 * time.Second) + go func() { + for { + select { + case <-tqm.ticker.C: + tqm.consume() + case <-tqm.stopCh: + log.Info("task queue manager stopped") + return + } + } + }() + log.Info("task queue manager started, will consume every 30 seconds") +} + +// Stop stops the task queue consumer +func (tqm *TaskQueueManager) Stop() { + if tqm.ticker != nil { + tqm.ticker.Stop() + } + close(tqm.stopCh) +} + +// consume processes all tasks in the queue +func (tqm *TaskQueueManager) consume() { + // Prevent concurrent consumption + if !tqm.consuming.CompareAndSwap(false, true) { + log.Warn("previous consume still running, skip this round") + return + } + defer tqm.consuming.Store(false) + + tqm.mu.Lock() + + // extract all tasks + tasks := make([]*QueuedTask, 0, len(tqm.queue)) + for _, task := range tqm.queue { + tasks = append(tasks, task) + } + + // clear queue + tqm.queue = make(map[string]*QueuedTask) + + tqm.mu.Unlock() + + if len(tasks) == 0 { + return + } + + log.Infof("consuming task queue: %d tasks", len(tasks)) + + // sort tasks: shallow paths first, then by enqueue time + sort.Slice(tasks, func(i, j int) bool { + if tasks[i].Depth != tasks[j].Depth { + return tasks[i].Depth < tasks[j].Depth + } + return tasks[i].EnqueueAt.Before(tasks[j].EnqueueAt) + }) + + ctx := context.Background() + + // execute tasks in order + for _, task := range tasks { + // Check if there are pending tasks for this parent + tqm.mu.RLock() + pendingTaskUIDs, hasPending := tqm.pendingTasks[task.Parent] + tqm.mu.RUnlock() + + if hasPending && len(pendingTaskUIDs) > 0 { + // Check all pending task statuses + allCompleted := true + for _, taskUID := range pendingTaskUIDs { + taskStatus, err := tqm.m.getTaskStatus(ctx, taskUID) + if err != nil { + log.Errorf("failed to get task status for parent %s (taskUID: %d): %v", task.Parent, taskUID, err) + // If we can't get status, assume it's done and continue checking + continue + } + + // Check if task is still running + if taskStatus == "enqueued" || taskStatus == "processing" { + log.Warnf("skipping task for parent %s: previous task %d still %s", task.Parent, taskUID, taskStatus) + allCompleted = false + break // No need to check remaining tasks + } + } + + if !allCompleted { + // Re-enqueue the task if not already in queue (avoid overwriting newer snapshots) + tqm.mu.Lock() + if _, exists := tqm.queue[task.Parent]; !exists { + tqm.queue[task.Parent] = task + log.Debugf("re-enqueued skipped task for parent %s due to pending tasks", task.Parent) + } else { + log.Debugf("skipped task for parent %s not re-enqueued (newer task already in queue)", task.Parent) + } + tqm.mu.Unlock() + continue // Skip this task, some previous tasks are still running + } + + // All tasks are in terminal state, remove from pending + log.Debugf("all previous tasks for parent %s are completed, proceeding with new task", task.Parent) + tqm.mu.Lock() + delete(tqm.pendingTasks, task.Parent) + tqm.mu.Unlock() + } + + // Execute the task + tqm.executeTask(ctx, task) + } + + log.Infof("task queue consumption completed") +} + +// executeTask executes a single task +func (tqm *TaskQueueManager) executeTask(ctx context.Context, task *QueuedTask) { + parent := task.Parent + currentObjs := task.Objs + + // Query index to get old state + nodes, err := tqm.m.Get(ctx, parent) + if err != nil { + log.Errorf("failed to get indexed nodes for parent %s: %v", parent, err) + return + } + + // Calculate diff based on current index state + now := mapset.NewSet[string]() + for i := range currentObjs { + now.Add(currentObjs[i].GetName()) + } + old := mapset.NewSet[string]() + for i := range nodes { + old.Add(nodes[i].Name) + } + + toDelete := old.Difference(now) + toAdd := now.Difference(old) + + // Collect paths to delete + var pathsToDelete []string + for i := range nodes { + if toDelete.Contains(nodes[i].Name) && !op.HasStorage(path.Join(parent, nodes[i].Name)) { + pathsToDelete = append(pathsToDelete, path.Join(parent, nodes[i].Name)) + } + } + + var allTaskUIDs []int64 + + // Execute delete first + if len(pathsToDelete) > 0 { + log.Debugf("executing delete for parent %s: %d paths", parent, len(pathsToDelete)) + taskUIDs, err := tqm.m.batchDeleteWithTaskUID(ctx, pathsToDelete) + if err != nil { + log.Errorf("failed to batch delete for parent %s: %v", parent, err) + // Continue to add even if delete fails + } else { + allTaskUIDs = append(allTaskUIDs, taskUIDs...) + } + } + + // Collect objects to add + var nodesToAdd []model.SearchNode + for i := range currentObjs { + if toAdd.Contains(currentObjs[i].GetName()) { + log.Debugf("will add index: %s", path.Join(parent, currentObjs[i].GetName())) + nodesToAdd = append(nodesToAdd, model.SearchNode{ + Parent: parent, + Name: currentObjs[i].GetName(), + IsDir: currentObjs[i].IsDir(), + Size: currentObjs[i].GetSize(), + }) + } + } + + // Execute add + if len(nodesToAdd) > 0 { + log.Debugf("executing add for parent %s: %d nodes", parent, len(nodesToAdd)) + taskUIDs, err := tqm.m.batchIndexWithTaskUID(ctx, nodesToAdd) + if err != nil { + log.Errorf("failed to batch index for parent %s: %v", parent, err) + } else { + allTaskUIDs = append(allTaskUIDs, taskUIDs...) + } + } + + // Record all task UIDs for this parent + if len(allTaskUIDs) > 0 { + tqm.mu.Lock() + tqm.pendingTasks[parent] = allTaskUIDs + tqm.mu.Unlock() + log.Debugf("recorded %d taskUIDs for parent %s", len(allTaskUIDs), parent) + } +} diff --git a/internal/search/meilisearch/utils.go b/internal/search/meilisearch/utils.go index 9849c209e..3bed9b0f5 100644 --- a/internal/search/meilisearch/utils.go +++ b/internal/search/meilisearch/utils.go @@ -1,7 +1,6 @@ package meilisearch import ( - "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -13,16 +12,16 @@ func hashPath(path string) string { } func buildSearchDocumentFromResults(results map[string]any) *searchDocument { - searchNode := model.SearchNode{} - document := &searchDocument{ - SearchNode: searchNode, - } + document := &searchDocument{} // use assertion test to avoid panic - searchNode.Parent, _ = results["parent"].(string) - searchNode.Name, _ = results["name"].(string) - searchNode.IsDir, _ = results["is_dir"].(bool) - searchNode.Size, _ = results["size"].(int64) + document.SearchNode.Parent, _ = results["parent"].(string) + document.SearchNode.Name, _ = results["name"].(string) + document.SearchNode.IsDir, _ = results["is_dir"].(bool) + // JSON numbers are typically float64, not int64 + if size, ok := results["size"].(float64); ok { + document.SearchNode.Size = int64(size) + } document.ID, _ = results["id"].(string) document.ParentHash, _ = results["parent_hash"].(string) diff --git a/internal/setting/setting.go b/internal/setting/setting.go index 7ebd5e92a..32968ee74 100644 --- a/internal/setting/setting.go +++ b/internal/setting/setting.go @@ -28,3 +28,11 @@ func GetInt(key string, defaultVal int) int { func GetBool(key string) bool { return GetStr(key) == "true" || GetStr(key) == "1" } + +func GetFloat(key string, defaultVal float64) float64 { + f, err := strconv.ParseFloat(GetStr(key), 64) + if err != nil { + return defaultVal + } + return f +} diff --git a/internal/stream/limit.go b/internal/stream/limit.go index ee32840d5..8715c7365 100644 --- a/internal/stream/limit.go +++ b/internal/stream/limit.go @@ -7,7 +7,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" "golang.org/x/time/rate" ) @@ -42,17 +41,14 @@ type RateLimitReader struct { } func (r *RateLimitReader) Read(p []byte) (n int, err error) { - if r.Ctx != nil && utils.IsCanceled(r.Ctx) { - return 0, r.Ctx.Err() + if err = r.Ctx.Err(); err != nil { + return 0, err } n, err = r.Reader.Read(p) if err != nil { return } if r.Limiter != nil { - if r.Ctx == nil { - r.Ctx = context.Background() - } err = r.Limiter.WaitN(r.Ctx, n) } return @@ -72,17 +68,14 @@ type RateLimitWriter struct { } func (w *RateLimitWriter) Write(p []byte) (n int, err error) { - if w.Ctx != nil && utils.IsCanceled(w.Ctx) { - return 0, w.Ctx.Err() + if err = w.Ctx.Err(); err != nil { + return 0, err } n, err = w.Writer.Write(p) if err != nil { return } if w.Limiter != nil { - if w.Ctx == nil { - w.Ctx = context.Background() - } err = w.Limiter.WaitN(w.Ctx, n) } return @@ -102,34 +95,28 @@ type RateLimitFile struct { } func (r *RateLimitFile) Read(p []byte) (n int, err error) { - if r.Ctx != nil && utils.IsCanceled(r.Ctx) { - return 0, r.Ctx.Err() + if err = r.Ctx.Err(); err != nil { + return 0, err } n, err = r.File.Read(p) if err != nil { return } if r.Limiter != nil { - if r.Ctx == nil { - r.Ctx = context.Background() - } err = r.Limiter.WaitN(r.Ctx, n) } return } func (r *RateLimitFile) ReadAt(p []byte, off int64) (n int, err error) { - if r.Ctx != nil && utils.IsCanceled(r.Ctx) { - return 0, r.Ctx.Err() + if err = r.Ctx.Err(); err != nil { + return 0, err } n, err = r.File.ReadAt(p, off) if err != nil { return } if r.Limiter != nil { - if r.Ctx == nil { - r.Ctx = context.Background() - } err = r.Limiter.WaitN(r.Ctx, n) } return @@ -145,16 +132,16 @@ func (r *RateLimitFile) Close() error { type RateLimitRangeReaderFunc RangeReaderFunc func (f RateLimitRangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + if ServerDownloadLimit == nil { + return f(ctx, httpRange) + } rc, err := f(ctx, httpRange) if err != nil { return nil, err } - if ServerDownloadLimit != nil { - rc = &RateLimitReader{ - Ctx: ctx, - Reader: rc, - Limiter: ServerDownloadLimit, - } - } - return rc, nil + return &RateLimitReader{ + Ctx: ctx, + Reader: rc, + Limiter: ServerDownloadLimit, + }, nil } diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 8d2f504fd..4c8238100 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -10,7 +10,6 @@ import ( "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" @@ -28,10 +27,8 @@ type FileStream struct { ForceStreamUpload bool Exist model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it utils.Closers - - tmpFile model.File //if present, tmpFile has full content, it will be deleted at last - peekBuff *buffer.Reader size int64 + peekBuff *buffer.Reader oriReader io.Reader // the original reader, used for caching } @@ -39,12 +36,6 @@ func (f *FileStream) GetSize() int64 { if f.size > 0 { return f.size } - if file, ok := f.tmpFile.(*os.File); ok { - info, err := file.Stat() - if err == nil { - return info.Size() - } - } return f.Obj.GetSize() } @@ -63,24 +54,10 @@ func (f *FileStream) IsForceStreamUpload() bool { func (f *FileStream) Close() error { if f.peekBuff != nil { f.peekBuff.Reset() + f.oriReader = nil f.peekBuff = nil } - - var err1, err2 error - err1 = f.Closers.Close() - if errors.Is(err1, os.ErrClosed) { - err1 = nil - } - if file, ok := f.tmpFile.(*os.File); ok { - err2 = os.RemoveAll(file.Name()) - if err2 != nil { - err2 = errs.NewErr(err2, "failed to remove tmpFile [%s]", file.Name()) - } else { - f.tmpFile = nil - } - } - - return errors.Join(err1, err2) + return f.Closers.Close() } func (f *FileStream) GetExist() model.Obj { @@ -94,28 +71,29 @@ func (f *FileStream) SetExist(obj model.Obj) { // It's not thread-safe! func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writer) (model.File, error) { if cache := f.GetFile(); cache != nil { + _, err := cache.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } if writer == nil { return cache, nil } - _, err := cache.Seek(0, io.SeekStart) - if err == nil { - reader := f.Reader - if up != nil { - cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) - *up = model.UpdateProgressWithRange(*up, 50, 100) - reader = &ReaderUpdatingProgress{ - Reader: &SimpleReaderWithSize{ - Reader: reader, - Size: f.GetSize(), - }, - UpdateProgress: cacheProgress, - } - } - _, err = utils.CopyWithBuffer(writer, reader) - if err == nil { - _, err = cache.Seek(0, io.SeekStart) + reader := f.Reader + if up != nil { + cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) + *up = model.UpdateProgressWithRange(*up, 50, 100) + reader = &ReaderUpdatingProgress{ + Reader: &SimpleReaderWithSize{ + Reader: reader, + Size: f.GetSize(), + }, + UpdateProgress: cacheProgress, } } + _, err = utils.CopyWithBuffer(writer, reader) + if err == nil { + _, err = cache.Seek(0, io.SeekStart) + } if err != nil { return nil, err } @@ -123,21 +101,20 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ } reader := f.Reader - if up != nil { - cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) - *up = model.UpdateProgressWithRange(*up, 50, 100) - reader = &ReaderUpdatingProgress{ - Reader: &SimpleReaderWithSize{ - Reader: reader, - Size: f.GetSize(), - }, - UpdateProgress: cacheProgress, + if f.peekBuff != nil { + f.peekBuff.Seek(0, io.SeekStart) + if writer != nil { + _, err := utils.CopyWithBuffer(writer, f.peekBuff) + if err != nil { + return nil, err + } + f.peekBuff.Seek(0, io.SeekStart) } + reader = f.oriReader } if writer != nil { reader = io.TeeReader(reader, writer) } - if f.GetSize() < 0 { if f.peekBuff == nil { f.peekBuff = &buffer.Reader{} @@ -174,7 +151,6 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ } } } - tmpF, err := utils.CreateTempFile(reader, 0) if err != nil { return nil, err @@ -191,22 +167,42 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ return peekF, nil } - f.Reader = reader + if up != nil { + cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) + *up = model.UpdateProgressWithRange(*up, 50, 100) + size := f.GetSize() + if f.peekBuff != nil { + peekSize := f.peekBuff.Size() + cacheProgress(float64(peekSize) / float64(size) * 100) + size -= peekSize + } + reader = &ReaderUpdatingProgress{ + Reader: &SimpleReaderWithSize{ + Reader: reader, + Size: size, + }, + UpdateProgress: cacheProgress, + } + } + + if f.peekBuff != nil { + f.oriReader = reader + } else { + f.Reader = reader + } return f.cache(f.GetSize()) } func (f *FileStream) GetFile() model.File { - if f.tmpFile != nil { - return f.tmpFile - } if file, ok := f.Reader.(model.File); ok { return file } return nil } -// RangeRead have to cache all data first since only Reader is provided. -// It's not thread-safe! +// 从流读取指定范围的一块数据,并且不消耗流。 +// 当读取的边界超过内部设置大小后会缓存整个流。 +// 流未缓存时线程不完全 func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { if httpRange.Length < 0 || httpRange.Start+httpRange.Length > f.GetSize() { httpRange.Length = f.GetSize() - httpRange.Start @@ -215,12 +211,7 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { return io.NewSectionReader(f.GetFile(), httpRange.Start, httpRange.Length), nil } - size := httpRange.Start + httpRange.Length - if f.peekBuff != nil && size <= int64(f.peekBuff.Size()) { - return io.NewSectionReader(f.peekBuff, httpRange.Start, httpRange.Length), nil - } - - cache, err := f.cache(size) + cache, err := f.cache(httpRange.Start + httpRange.Length) if err != nil { return nil, err } @@ -232,14 +223,30 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { // 使用bytes.Buffer作为io.CopyBuffer的写入对象，CopyBuffer会调用Buffer.ReadFrom // 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大 +// 确保指定大小的数据被缓存 func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { if maxCacheSize > int64(conf.MaxBufferLimit) { - tmpF, err := utils.CreateTempFile(f.Reader, f.GetSize()) + size := f.GetSize() + reader := f.Reader + if f.peekBuff != nil { + size -= f.peekBuff.Size() + reader = f.oriReader + } + tmpF, err := utils.CreateTempFile(reader, size) if err != nil { return nil, err } - f.Add(tmpF) - f.tmpFile = tmpF + f.Add(utils.CloseFunc(func() error { + return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) + })) + if f.peekBuff != nil { + peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) + if err != nil { + return nil, err + } + f.Reader = peekF + return peekF, nil + } f.Reader = tmpF return tmpF, nil } @@ -247,8 +254,12 @@ func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { if f.peekBuff == nil { f.peekBuff = &buffer.Reader{} f.oriReader = f.Reader + f.Reader = io.MultiReader(f.peekBuff, f.oriReader) + } + bufSize := maxCacheSize - f.peekBuff.Size() + if bufSize <= 0 { + return f.peekBuff, nil } - bufSize := maxCacheSize - int64(f.peekBuff.Size()) var buf []byte if conf.MmapThreshold > 0 && bufSize >= int64(conf.MmapThreshold) { m, err := mmap.Alloc(int(bufSize)) @@ -267,37 +278,24 @@ func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", bufSize, n, err) } f.peekBuff.Append(buf) - if int64(f.peekBuff.Size()) >= f.GetSize() { + if f.peekBuff.Size() >= f.GetSize() { f.Reader = f.peekBuff - f.oriReader = nil - } else { - f.Reader = io.MultiReader(f.peekBuff, f.oriReader) } return f.peekBuff, nil } -func (f *FileStream) SetTmpFile(file model.File) { - f.AddIfCloser(file) - f.tmpFile = file - f.Reader = file -} - var _ model.FileStreamer = (*SeekableStream)(nil) var _ model.FileStreamer = (*FileStream)(nil) -//var _ seekableStream = (*FileStream)(nil) - -// for most internal stream, which is either RangeReadCloser or MFile -// Any functionality implemented based on SeekableStream should implement a Close method, -// whose only purpose is to close the SeekableStream object. If such functionality has -// additional resources that need to be closed, they should be added to the Closer property of -// the SeekableStream object and be closed together when the SeekableStream object is closed. type SeekableStream struct { *FileStream // should have one of belows to support rangeRead - rangeReadCloser model.RangeReadCloserIF + rangeReader model.RangeReaderIF } +// NewSeekableStream create a SeekableStream from FileStream and Link +// if FileStream.Reader is not nil, use it directly +// else create RangeReader from Link func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error) { if len(fs.Mimetype) == 0 { fs.Mimetype = utils.GetMimeType(fs.Obj.GetName()) @@ -317,30 +315,31 @@ func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error if err != nil { return nil, err } - rrc := &model.RangeReadCloser{ - RangeReader: rr, - } if _, ok := rr.(*model.FileRangeReader); ok { - fs.Reader, err = rrc.RangeRead(fs.Ctx, http_range.Range{Length: -1}) + var rc io.ReadCloser + rc, err = rr.RangeRead(fs.Ctx, http_range.Range{Length: -1}) if err != nil { return nil, err } + fs.Reader = rc + fs.Add(rc) } fs.size = size fs.Add(link) - fs.Add(rrc) - return &SeekableStream{FileStream: fs, rangeReadCloser: rrc}, nil + return &SeekableStream{FileStream: fs, rangeReader: rr}, nil } return nil, fmt.Errorf("illegal seekableStream") } -// RangeRead is not thread-safe, pls use it in single thread only. +// 如果使用缓存或者rangeReader读取指定范围的数据，是线程安全的 +// 其他特性继承自FileStream.RangeRead func (ss *SeekableStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { - if ss.GetFile() == nil && ss.rangeReadCloser != nil { - rc, err := ss.rangeReadCloser.RangeRead(ss.Ctx, httpRange) + if ss.GetFile() == nil && ss.rangeReader != nil { + rc, err := ss.rangeReader.RangeRead(ss.Ctx, httpRange) if err != nil { return nil, err } + ss.Add(rc) return rc, nil } return ss.FileStream.RangeRead(httpRange) @@ -356,13 +355,14 @@ func (ss *SeekableStream) Read(p []byte) (n int, err error) { func (ss *SeekableStream) generateReader() error { if ss.Reader == nil { - if ss.rangeReadCloser == nil { + if ss.rangeReader == nil { return fmt.Errorf("illegal seekableStream") } - rc, err := ss.rangeReadCloser.RangeRead(ss.Ctx, http_range.Range{Length: -1}) + rc, err := ss.rangeReader.RangeRead(ss.Ctx, http_range.Range{Length: -1}) if err != nil { return err } + ss.Add(rc) ss.Reader = rc } return nil @@ -456,7 +456,7 @@ func (r *headCache) Close() error { } func (r *RangeReadReadAtSeeker) InitHeadCache() { - if r.ss.GetFile() == nil && r.masterOff == 0 { + if r.masterOff == 0 { value, _ := r.readerMap.LoadAndDelete(int64(0)) r.headCache = &headCache{reader: value.(io.Reader)} r.ss.Closers.Add(r.headCache) @@ -464,12 +464,12 @@ func (r *RangeReadReadAtSeeker) InitHeadCache() { } func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (model.File, error) { - if ss.GetFile() != nil { - _, err := ss.GetFile().Seek(offset, io.SeekStart) + if cache := ss.GetFile(); cache != nil { + _, err := cache.Seek(offset, io.SeekStart) if err != nil { return nil, err } - return ss.GetFile(), nil + return cache, nil } r := &RangeReadReadAtSeeker{ ss: ss, @@ -479,10 +479,11 @@ func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (mode if offset < 0 || offset > ss.GetSize() { return nil, errors.New("offset out of range") } - _, err := r.getReaderAtOffset(offset) + reader, err := r.getReaderAtOffset(offset) if err != nil { return nil, err } + r.readerMap.Store(int64(offset), reader) } else { r.readerMap.Store(int64(offset), ss) } @@ -502,39 +503,41 @@ func NewMultiReaderAt(ss []*SeekableStream) (readerutil.SizeReaderAt, error) { } func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (io.Reader, error) { - var rr io.Reader - var cur int64 = -1 - r.readerMap.Range(func(key, value any) bool { - k := key.(int64) - if off == k { - cur = k - rr = value.(io.Reader) - return false + for { + var cur int64 = -1 + r.readerMap.Range(func(key, value any) bool { + k := key.(int64) + if off == k { + cur = k + return false + } + if off > k && off-k <= 4*utils.MB && k > cur { + cur = k + } + return true + }) + if cur < 0 { + break } - if off > k && off-k <= 4*utils.MB && (rr == nil || k < cur) { - rr = value.(io.Reader) - cur = k + v, ok := r.readerMap.LoadAndDelete(int64(cur)) + if !ok { + continue + } + rr := v.(io.Reader) + if off == int64(cur) { + // logrus.Debugf("getReaderAtOffset match_%d", off) + return rr, nil } - return true - }) - if cur >= 0 { - r.readerMap.Delete(int64(cur)) - } - if off == int64(cur) { - // logrus.Debugf("getReaderAtOffset match_%d", off) - return rr, nil - } - - if rr != nil { n, _ := utils.CopyWithBufferN(io.Discard, rr, off-cur) cur += n if cur == off { // logrus.Debugf("getReaderAtOffset old_%d", off) return rr, nil } + break } - // logrus.Debugf("getReaderAtOffset new_%d", off) + // logrus.Debugf("getReaderAtOffset new_%d", off) reader, err := r.ss.RangeRead(http_range.Range{Start: off, Length: -1}) if err != nil { return nil, err diff --git a/internal/stream/stream_test.go b/internal/stream/stream_test.go index 52c2abee0..9a81e7d41 100644 --- a/internal/stream/stream_test.go +++ b/internal/stream/stream_test.go @@ -7,13 +7,12 @@ import ( "io" "testing" - "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func TestFileStream_RangeRead(t *testing.T) { - conf.MaxBufferLimit = 16 * 1024 * 1024 type args struct { httpRange http_range.Range } @@ -73,16 +72,38 @@ func TestFileStream_RangeRead(t *testing.T) { } }) } - t.Run("after", func(t *testing.T) { - if f.GetFile() == nil { - t.Error("not cached") - } - buf2 := make([]byte, len(buf)) - if _, err := io.ReadFull(f, buf2); err != nil { - t.Errorf("FileStream.Read() error = %v", err) - } - if !bytes.Equal(buf, buf2) { - t.Errorf("FileStream.Read() = %s, want %s", buf2, buf) - } - }) + if f.GetFile() == nil { + t.Error("not cached") + } + buf2 := make([]byte, len(buf)) + if _, err := io.ReadFull(f, buf2); err != nil { + t.Errorf("FileStream.Read() error = %v", err) + } + if !bytes.Equal(buf, buf2) { + t.Errorf("FileStream.Read() = %s, want %s", buf2, buf) + } +} + +func TestFileStream_With_PreHash(t *testing.T) { + buf := []byte("github.com/OpenListTeam/OpenList") + f := &FileStream{ + Obj: &model.Object{ + Size: int64(len(buf)), + }, + Reader: io.NopCloser(bytes.NewReader(buf)), + } + + const hashSize int64 = 20 + reader, _ := f.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + preHash, _ := utils.HashReader(utils.SHA1, reader) + if preHash == "" { + t.Error("preHash is empty") + } + tmpF, fullHash, _ := CacheFullAndHash(f, nil, utils.SHA1) + fmt.Println(fullHash) + fileFullHash, _ := utils.HashFile(utils.SHA1, tmpF) + fmt.Println(fileFullHash) + if fullHash != fileFullHash { + t.Errorf("fullHash and fileFullHash should match: fullHash=%s fileFullHash=%s", fullHash, fileFullHash) + } } diff --git a/internal/stream/util.go b/internal/stream/util.go index cba29e789..6aa3dda5d 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -11,6 +11,7 @@ import ( "os" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" @@ -27,47 +28,61 @@ func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Ran } func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { - if link.MFile != nil { - return GetRangeReaderFromMFile(size, link.MFile), nil + if link.RangeReader != nil { + if link.Concurrency < 1 && link.PartSize < 1 { + return link.RangeReader, nil + } + down := net.NewDownloader(func(d *net.Downloader) { + d.Concurrency = link.Concurrency + d.PartSize = link.PartSize + d.HttpClient = net.GetRangeReaderHttpRequestFunc(link.RangeReader) + }) + rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return down.Download(ctx, &net.HttpRequestParams{ + Range: httpRange, + Size: size, + }) + } + // RangeReader只能在驱动限速 + return RangeReaderFunc(rangeReader), nil + } + + if len(link.URL) == 0 { + return nil, errors.New("invalid link: must have at least one of URL or RangeReader") } + if link.Concurrency > 0 || link.PartSize > 0 { down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize - }) - var rangeReader RangeReaderFunc = func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { - var req *net.HttpRequestParams - if link.RangeReader != nil { - req = &net.HttpRequestParams{ - Range: httpRange, - Size: size, + d.HttpClient = func(ctx context.Context, params *net.HttpRequestParams) (*http.Response, error) { + if ServerDownloadLimit == nil { + return net.DefaultHttpRequestFunc(ctx, params) } - } else { - requestHeader, _ := ctx.Value(conf.RequestHeaderKey).(http.Header) - header := net.ProcessHeader(requestHeader, link.Header) - req = &net.HttpRequestParams{ - Range: httpRange, - Size: size, - URL: link.URL, - HeaderRef: header, + resp, err := net.DefaultHttpRequestFunc(ctx, params) + if err == nil && resp.Body != nil { + resp.Body = &RateLimitReader{ + Ctx: ctx, + Reader: resp.Body, + Limiter: ServerDownloadLimit, + } } + return resp, err } - return down.Download(ctx, req) - } - if link.RangeReader != nil { - down.HttpClient = net.GetRangeReaderHttpRequestFunc(link.RangeReader) - return rangeReader, nil + }) + rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + requestHeader, _ := ctx.Value(conf.RequestHeaderKey).(http.Header) + header := net.ProcessHeader(requestHeader, link.Header) + return down.Download(ctx, &net.HttpRequestParams{ + Range: httpRange, + Size: size, + URL: link.URL, + HeaderRef: header, + }) } - return RateLimitRangeReaderFunc(rangeReader), nil - } - - if link.RangeReader != nil { - return link.RangeReader, nil + return RangeReaderFunc(rangeReader), nil } - if len(link.URL) == 0 { - return nil, errors.New("invalid link: must have at least one of MFile, URL, or RangeReader") - } rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { if httpRange.Length < 0 || httpRange.Start+httpRange.Length > size { httpRange.Length = size - httpRange.Start @@ -78,12 +93,20 @@ func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, response, err := net.RequestHttp(ctx, "GET", header, link.URL) if err != nil { - if _, ok := errors.Unwrap(err).(net.HttpStatusCodeError); ok { + if _, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { return nil, err } return nil, fmt.Errorf("http request failure, err:%w", err) } - if httpRange.Start == 0 && (httpRange.Length == -1 || httpRange.Length == size) || response.StatusCode == http.StatusPartialContent || + if ServerDownloadLimit != nil { + response.Body = &RateLimitReader{ + Ctx: ctx, + Reader: response.Body, + Limiter: ServerDownloadLimit, + } + } + if httpRange.Start == 0 && httpRange.Length == size || + response.StatusCode == http.StatusPartialContent || checkContentRange(&response.Header, httpRange.Start) { return response.Body, nil } else if response.StatusCode == http.StatusOK { @@ -96,11 +119,10 @@ func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, } return response.Body, nil } - return RateLimitRangeReaderFunc(rangeReader), nil + return RangeReaderFunc(rangeReader), nil } -// RangeReaderIF.RangeRead返回的io.ReadCloser保留file的签名。 -func GetRangeReaderFromMFile(size int64, file model.File) model.RangeReaderIF { +func GetRangeReaderFromMFile(size int64, file model.File) *model.FileRangeReader { return &model.FileRangeReader{ RangeReaderIF: RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length @@ -172,7 +194,7 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode return nil, err } - if f.Truncate((file.GetSize()+int64(maxBufferSize-1))/int64(maxBufferSize)*int64(maxBufferSize)) != nil { + if f.Truncate(file.GetSize()) != nil { // fallback to full cache _, _ = f.Close(), os.Remove(f.Name()) cache, err := file.CacheFullAndWriter(up, nil) @@ -182,11 +204,11 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode return &cachedSectionReader{cache}, nil } - ss := &fileSectionReader{Reader: file, temp: f} + ss := &fileSectionReader{file: file, temp: f} ss.bufPool = &pool.Pool[*offsetWriterWithBase]{ New: func() *offsetWriterWithBase { - base := ss.fileOff - ss.fileOff += int64(maxBufferSize) + base := ss.tempOffset + ss.tempOffset += int64(maxBufferSize) return &offsetWriterWithBase{io.NewOffsetWriter(ss.temp, base), base} }, } @@ -203,7 +225,7 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode New: func() []byte { buf, err := mmap.Alloc(maxBufferSize) if err == nil { - ss.file.Add(utils.CloseFunc(func() error { + file.Add(utils.CloseFunc(func() error { return mmap.Free(buf) })) } else { @@ -240,11 +262,11 @@ func (s *cachedSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker func (*cachedSectionReader) FreeSectionReader(sr io.ReadSeeker) {} type fileSectionReader struct { - io.Reader - off int64 - temp *os.File - fileOff int64 - bufPool *pool.Pool[*offsetWriterWithBase] + file model.FileStreamer + fileOffset int64 + temp *os.File + tempOffset int64 + bufPool *pool.Pool[*offsetWriterWithBase] } type offsetWriterWithBase struct { @@ -254,14 +276,14 @@ type offsetWriterWithBase struct { // 线程不安全 func (ss *fileSectionReader) DiscardSection(off int64, length int64) error { - if off != ss.off { - return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.off) + if off != ss.fileOffset { + return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } - _, err := utils.CopyWithBufferN(io.Discard, ss.Reader, length) + n, err := utils.CopyWithBufferN(io.Discard, ss.file, length) + ss.fileOffset += n if err != nil { - return fmt.Errorf("failed to skip data: (expect =%d) %w", length, err) + return fmt.Errorf("failed to skip data: (expect =%d, actual =%d) %w", length, n, err) } - ss.off += length return nil } @@ -270,17 +292,18 @@ type fileBufferSectionReader struct { fileBuf *offsetWriterWithBase } +// 线程不安全 func (ss *fileSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { - if off != ss.off { - return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.off) + if off != ss.fileOffset { + return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } fileBuf := ss.bufPool.Get() _, _ = fileBuf.Seek(0, io.SeekStart) - n, err := utils.CopyWithBufferN(fileBuf, ss.Reader, length) + n, err := utils.CopyWithBufferN(fileBuf, ss.file, length) + ss.fileOffset += n if err != nil { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) } - ss.off += length return &fileBufferSectionReader{io.NewSectionReader(ss.temp, fileBuf.base, length), fileBuf}, nil } @@ -293,21 +316,21 @@ func (ss *fileSectionReader) FreeSectionReader(rs io.ReadSeeker) { } type directSectionReader struct { - file model.FileStreamer - off int64 - bufPool *pool.Pool[[]byte] + file model.FileStreamer + fileOffset int64 + bufPool *pool.Pool[[]byte] } // 线程不安全 func (ss *directSectionReader) DiscardSection(off int64, length int64) error { - if off != ss.off { - return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.off) + if off != ss.fileOffset { + return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } - _, err := utils.CopyWithBufferN(io.Discard, ss.file, length) + n, err := utils.CopyWithBufferN(io.Discard, ss.file, length) + ss.fileOffset += n if err != nil { - return fmt.Errorf("failed to skip data: (expect =%d) %w", length, err) + return fmt.Errorf("failed to skip data: (expect =%d, actual =%d) %w", length, n, err) } - ss.off += length return nil } @@ -318,16 +341,16 @@ type bufferSectionReader struct { // 线程不安全 func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { - if off != ss.off { - return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.off) + if off != ss.fileOffset { + return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } tempBuf := ss.bufPool.Get() buf := tempBuf[:length] n, err := io.ReadFull(ss.file, buf) + ss.fileOffset += int64(n) if int64(n) != length { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) } - ss.off += int64(n) return &bufferSectionReader{bytes.NewReader(buf), buf}, nil } func (ss *directSectionReader) FreeSectionReader(rs io.ReadSeeker) { diff --git a/internal/task/base.go b/internal/task/base.go index 8976ed90b..f969e6c27 100644 --- a/internal/task/base.go +++ b/internal/task/base.go @@ -14,7 +14,7 @@ type TaskExtension struct { Creator *model.User startTime *time.Time endTime *time.Time - totalBytes int64 + TotalBytes int64 ApiUrl string } @@ -58,25 +58,25 @@ func (t *TaskExtension) ClearEndTime() { } func (t *TaskExtension) SetTotalBytes(totalBytes int64) { - t.totalBytes = totalBytes + t.TotalBytes = totalBytes } func (t *TaskExtension) GetTotalBytes() int64 { - return t.totalBytes + return t.TotalBytes } -func (t *TaskExtension) ReinitCtx() error { +func (t *TaskExtension) SetRetry(retry int, maxRetry int) { + t.Base.SetRetry(retry, maxRetry) + if retry > 0 || !conf.Conf.Tasks.AllowRetryCanceled || t.Ctx() == nil { + return + } select { case <-t.Ctx().Done(): - if !conf.Conf.Tasks.AllowRetryCanceled { - return t.Ctx().Err() - } ctx, cancel := context.WithCancel(context.Background()) t.SetCtx(ctx) t.SetCancelFunc(cancel) default: } - return nil } type TaskExtensionInfo interface { diff --git a/internal/task_group/group.go b/internal/task_group/group.go index 3cf78c8a2..edd51fe74 100644 --- a/internal/task_group/group.go +++ b/internal/task_group/group.go @@ -1,12 +1,13 @@ package task_group import ( + "context" "sync" "github.com/sirupsen/logrus" ) -type OnCompletionFunc func(groupID string, payloads ...any) +type OnCompletionFunc func(ctx context.Context, groupID string, payloads ...any) type TaskGroupCoordinator struct { name string mu sync.Mutex @@ -53,7 +54,7 @@ func (tgc *TaskGroupCoordinator) AppendPayload(groupID string, payload any) { tgc.groupPayloads[groupID] = append(tgc.groupPayloads[groupID], payload) } -func (tgc *TaskGroupCoordinator) Done(groupID string, success bool) { +func (tgc *TaskGroupCoordinator) Done(ctx context.Context, groupID string, success bool) { tgc.mu.Lock() defer tgc.mu.Unlock() state, ok := tgc.groupStates[groupID] @@ -71,7 +72,7 @@ func (tgc *TaskGroupCoordinator) Done(groupID string, success bool) { if tgc.onCompletion != nil && state.hasSuccess { logrus.Debugf("OnCompletion:%s", groupID) tgc.mu.Unlock() - tgc.onCompletion(groupID, payloads...) + tgc.onCompletion(ctx, groupID, payloads...) tgc.mu.Lock() } return diff --git a/internal/task_group/transfer.go b/internal/task_group/transfer.go index 3cef57865..0e9661b45 100644 --- a/internal/task_group/transfer.go +++ b/internal/task_group/transfer.go @@ -5,46 +5,66 @@ import ( "fmt" "path" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) type SrcPathToRemove string // ActualPath -type DstPathToRefresh string +type DstPathToHook string -func RefreshAndRemove(dstPath string, payloads ...any) { +func HookAndRemove(ctx context.Context, dstPath string, payloads ...any) { dstStorage, dstActualPath, err := op.GetStorageAndActualPath(dstPath) if err != nil { log.Error(errors.WithMessage(err, "failed get dst storage")) return } - _, dstNeedRefresh := dstStorage.(driver.Put) - dstNeedRefresh = dstNeedRefresh && !dstStorage.Config().NoCache - if dstNeedRefresh { - op.DeleteCache(dstStorage, dstActualPath) + dstNeedHandleHook := setting.GetBool(conf.HandleHookAfterWriting) + dstHandleHookLimit := setting.GetFloat(conf.HandleHookRateLimit, .0) + var listLimiter *rate.Limiter + if dstNeedHandleHook && dstHandleHookLimit > .0 { + listLimiter = rate.NewLimiter(rate.Limit(dstHandleHookLimit), 1) + } + hookedPaths := make(map[string]struct{}) + handleHook := func(actualPath string) { + if _, ok := hookedPaths[actualPath]; ok { + return + } + if listLimiter != nil { + _ = listLimiter.Wait(ctx) + } + files, e := op.List(ctx, dstStorage, actualPath, model.ListArgs{SkipHook: true}) + if e != nil { + log.Errorf("failed handle objs update hook: %v", e) + } else { + op.HandleObjsUpdateHook(ctx, utils.GetFullPath(dstStorage.GetStorage().MountPath, actualPath), files) + hookedPaths[actualPath] = struct{}{} + } + } + if dstNeedHandleHook { + handleHook(dstActualPath) } - var ctx context.Context for _, payload := range payloads { switch p := payload.(type) { - case DstPathToRefresh: - if dstNeedRefresh { - op.DeleteCache(dstStorage, string(p)) + case DstPathToHook: + if dstNeedHandleHook { + handleHook(string(p)) } case SrcPathToRemove: - if ctx == nil { - ctx = context.Background() - } srcStorage, srcActualPath, err := op.GetStorageAndActualPath(string(p)) if err != nil { log.Error(errors.WithMessage(err, "failed get src storage")) continue } - err = verifyAndRemove(ctx, srcStorage, dstStorage, srcActualPath, dstActualPath, dstNeedRefresh) + err = verifyAndRemove(ctx, srcStorage, dstStorage, srcActualPath, dstActualPath) if err != nil { log.Error(err) } @@ -52,14 +72,14 @@ func RefreshAndRemove(dstPath string, payloads ...any) { } } -func verifyAndRemove(ctx context.Context, srcStorage, dstStorage driver.Driver, srcPath, dstPath string, refresh bool) error { - srcObj, err := op.Get(ctx, srcStorage, srcPath) +func verifyAndRemove(ctx context.Context, srcStorage, dstStorage driver.Driver, srcPath, dstPath string) error { + srcObj, err := op.GetUnwrap(ctx, srcStorage, srcPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", path.Join(srcStorage.GetStorage().MountPath, srcPath)) } dstObjPath := path.Join(dstPath, srcObj.GetName()) - dstObj, err := op.Get(ctx, dstStorage, dstObjPath) + dstObj, err := op.GetUnwrap(ctx, dstStorage, dstObjPath) if err != nil { return errors.WithMessagef(err, "failed get dst [%s] file", path.Join(dstStorage.GetStorage().MountPath, dstObjPath)) } @@ -78,13 +98,10 @@ func verifyAndRemove(ctx context.Context, srcStorage, dstStorage driver.Driver, return errors.WithMessagef(err, "failed list src [%s] objs", path.Join(srcStorage.GetStorage().MountPath, srcPath)) } - if refresh { - op.DeleteCache(dstStorage, dstObjPath) - } hasErr := false for _, obj := range srcObjs { srcSubPath := path.Join(srcPath, obj.GetName()) - err := verifyAndRemove(ctx, srcStorage, dstStorage, srcSubPath, dstObjPath, refresh) + err := verifyAndRemove(ctx, srcStorage, dstStorage, srcSubPath, dstObjPath) if err != nil { log.Error(err) hasErr = true @@ -100,4 +117,4 @@ func verifyAndRemove(ctx context.Context, srcStorage, dstStorage driver.Driver, return nil } -var TransferCoordinator *TaskGroupCoordinator = NewTaskGroupCoordinator("RefreshAndRemove", RefreshAndRemove) +var TransferCoordinator *TaskGroupCoordinator = NewTaskGroupCoordinator("HookAndRemove", HookAndRemove) diff --git a/pkg/aria2/rpc/call.go b/pkg/aria2/rpc/call.go index a2af84617..e8f7d4d6e 100644 --- a/pkg/aria2/rpc/call.go +++ b/pkg/aria2/rpc/call.go @@ -69,14 +69,11 @@ func (h *httpCaller) setNotifier(ctx context.Context, u url.URL, notifier Notifi go func() { defer h.wg.Done() defer conn.Close() - select { - case <-ctx.Done(): - conn.SetWriteDeadline(time.Now().Add(time.Second)) - if err := conn.WriteMessage(websocket.CloseMessage, - websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")); err != nil { - log.Printf("sending websocket close message: %v", err) - } - return + <-ctx.Done() + conn.SetWriteDeadline(time.Now().Add(time.Second)) + if err := conn.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")); err != nil { + log.Printf("sending websocket close message: %v", err) } }() h.wg.Add(1) @@ -120,7 +117,7 @@ func (h *httpCaller) setNotifier(ctx context.Context, u url.URL, notifier Notifi return } -func (h httpCaller) Call(method string, params, reply interface{}) (err error) { +func (h *httpCaller) Call(method string, params, reply interface{}) (err error) { payload, err := EncodeClientRequest(method, params) if err != nil { return @@ -236,7 +233,7 @@ func (w *websocketCaller) Close() (err error) { return } -func (w websocketCaller) Call(method string, params, reply interface{}) (err error) { +func (w *websocketCaller) Call(method string, params, reply interface{}) (err error) { ctx, cancel := context.WithTimeout(context.Background(), w.timeout) defer cancel() select { @@ -251,11 +248,9 @@ func (w websocketCaller) Call(method string, params, reply interface{}) (err err return errors.New("sending channel blocking") } - select { - case <-ctx.Done(): - if err := ctx.Err(); err == context.DeadlineExceeded { - return err - } + <-ctx.Done() + if err := ctx.Err(); err == context.DeadlineExceeded { + return err } return } diff --git a/pkg/errgroup/errgroup.go b/pkg/errgroup/errgroup.go index d3c4feaff..7d72a1d8e 100644 --- a/pkg/errgroup/errgroup.go +++ b/pkg/errgroup/errgroup.go @@ -29,6 +29,7 @@ func NewGroupWithContext(ctx context.Context, limit int, retryOpts ...retry.Opti } // OrderedGroup +// 使得Lifecycle.Before是有序且线程安全 func NewOrderedGroupWithContext(ctx context.Context, limit int, retryOpts ...retry.Option) (*Group, context.Context) { group, ctx := NewGroupWithContext(ctx, limit, retryOpts...) group.startChan = make(chan token, 1) @@ -53,11 +54,11 @@ func (g *Group) Go(do func(ctx context.Context) error) { } type Lifecycle struct { - // Before在OrderedGroup是线程安全的。 + // Before在OrderedGroup是有序且线程安全的 // 只会被调用一次 - Before func(ctx context.Context) error + Before func(ctx context.Context) (err error) // 如果Before返回err就不调用Do - Do func(ctx context.Context) error + Do func(ctx context.Context) (err error) // 最后调用一次After After func(err error) } diff --git a/pkg/gowebdav/errors.go b/pkg/gowebdav/errors.go index bbf1e929e..de03e7b96 100644 --- a/pkg/gowebdav/errors.go +++ b/pkg/gowebdav/errors.go @@ -1,8 +1,8 @@ package gowebdav import ( - "fmt" "os" + "strconv" ) // StatusError implements error and wraps @@ -12,7 +12,7 @@ type StatusError struct { } func (se StatusError) Error() string { - return fmt.Sprintf("%d", se.Status) + return strconv.Itoa(se.Status) } // IsErrCode returns true if the given error diff --git a/pkg/singleflight/singleflight.go b/pkg/singleflight/singleflight.go index 3555d5bd0..483834785 100644 --- a/pkg/singleflight/singleflight.go +++ b/pkg/singleflight/singleflight.go @@ -73,9 +73,6 @@ type call[T any] struct { type Group[T any] struct { mu sync.Mutex // protects m m map[string]*call[T] // lazily initialized - - // Won't remember error - Remember bool } // Result holds the results of Do, so they can be passed @@ -159,7 +156,7 @@ func (g *Group[T]) doCall(c *call[T], key string, fn func() (T, error)) { g.mu.Lock() defer g.mu.Unlock() c.wg.Done() - if (!g.Remember || c.err != nil) && g.m[key] == c { + if g.m[key] == c { delete(g.m, key) } diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 1f819edf8..4fa58d397 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -185,3 +185,20 @@ const ( GB TB ) + +// IsSystemFile checks if a filename is a common system file that should be ignored +// Returns true for files like .DS_Store, desktop.ini, Thumbs.db, and Apple Double files (._*) +func IsSystemFile(filename string) bool { + // Common system files + switch filename { + case ".DS_Store", "desktop.ini", "Thumbs.db", "@eaDir": + return true + } + + // Apple Double files (._*) + if strings.HasPrefix(filename, "._") { + return true + } + + return false +} diff --git a/pkg/utils/file_test.go b/pkg/utils/file_test.go new file mode 100644 index 000000000..bcfff7934 --- /dev/null +++ b/pkg/utils/file_test.go @@ -0,0 +1,43 @@ +package utils + +import ( + "testing" +) + +func TestIsSystemFile(t *testing.T) { + testCases := []struct { + filename string + expected bool + }{ + // System files that should be filtered + {".DS_Store", true}, + {"desktop.ini", true}, + {"Thumbs.db", true}, + {"._test.txt", true}, + {"._", true}, + {"._somefile", true}, + {"._folder_name", true}, + {"@eaDir", true}, + + // Regular files that should not be filtered + {"test.txt", false}, + {"file.pdf", false}, + {"document.docx", false}, + {".gitignore", false}, + {".env", false}, + {"_underscore.txt", false}, + {"normal_file.txt", false}, + {"", false}, + {".hidden", false}, + {"..special", false}, + } + + for _, tc := range testCases { + t.Run(tc.filename, func(t *testing.T) { + result := IsSystemFile(tc.filename) + if result != tc.expected { + t.Errorf("IsSystemFile(%q) = %v, want %v", tc.filename, result, tc.expected) + } + }) + } +} diff --git a/pkg/utils/io.go b/pkg/utils/io.go index ff52f843e..f398161f6 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -187,51 +187,38 @@ func NewClosers(c ...io.Closer) Closers { return Closers(c) } -type SyncClosersIF interface { - ClosersIF - AcquireReference() bool -} - type SyncClosers struct { closers []io.Closer ref int32 } -var _ SyncClosersIF = (*SyncClosers)(nil) - +// if closed, return false func (c *SyncClosers) AcquireReference() bool { - for { - ref := atomic.LoadInt32(&c.ref) - if ref < 0 { - return false - } - newRef := ref + 1 - if atomic.CompareAndSwapInt32(&c.ref, ref, newRef) { - // log.Debugf("AcquireReference %p: %d", c, newRef) - return true - } + ref := atomic.AddInt32(&c.ref, 1) + if ref > 0 { + // log.Debugf("AcquireReference %p: %d", c, ref) + return true } + atomic.StoreInt32(&c.ref, closersClosed) + return false } -const closersClosed = math.MinInt16 +const closersClosed = math.MinInt32 func (c *SyncClosers) Close() error { - ref := atomic.AddInt32(&c.ref, -1) - if ref > 0 { - // log.Debugf("ReleaseReference %p: %d", c, ref) - return nil - } - - if ref < -1 { - atomic.StoreInt32(&c.ref, closersClosed) - return nil - } - - // Attempt to acquire FinalClose permission. - // At this point, ref must be 0 or -1. We try to atomically change it to the closersClosed state. - // Only the first successful goroutine gets the cleanup permission. - if !atomic.CompareAndSwapInt32(&c.ref, ref, closersClosed) { - return nil + for { + ref := atomic.LoadInt32(&c.ref) + if ref < 0 { + return nil + } + if ref > 1 { + if atomic.CompareAndSwapInt32(&c.ref, ref, ref-1) { + // log.Debugf("ReleaseReference %p: %d", c, ref) + return nil + } + } else if atomic.CompareAndSwapInt32(&c.ref, ref, closersClosed) { + break + } } // log.Debugf("FinalClose %p", c) @@ -264,6 +251,16 @@ func (c *SyncClosers) AddIfCloser(a any) { } } +var _ ClosersIF = (*SyncClosers)(nil) + +// 实现cache.Expirable接口 +func (c *SyncClosers) Expired() bool { + return atomic.LoadInt32(&c.ref) < 0 +} +func (c *SyncClosers) Length() int { + return len(c.closers) +} + func NewSyncClosers(c ...io.Closer) SyncClosers { return SyncClosers{closers: c} } diff --git a/pkg/utils/path.go b/pkg/utils/path.go index 2a104ff51..ec75501e2 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -44,11 +44,15 @@ func IsSubPath(path string, subPath string) bool { } func Ext(path string) string { + return strings.ToLower(SourceExt(path)) +} + +func SourceExt(path string) string { ext := stdpath.Ext(path) if len(ext) > 0 && ext[0] == '.' { ext = ext[1:] } - return strings.ToLower(ext) + return ext } func EncodePath(path string, all ...bool) string { diff --git a/server/common/ldap.go b/server/common/ldap.go new file mode 100644 index 000000000..9dc3b93fd --- /dev/null +++ b/server/common/ldap.go @@ -0,0 +1,107 @@ +package common + +import ( + "crypto/tls" + "fmt" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/ldap.v3" +) + +var ErrFailedLdapAuth = errors.New("failed to auth") + +func HandleLdapLogin(username, password string) error { + // Auth start + ldapServer := setting.GetStr(conf.LdapServer) + skipTlsVerify := setting.GetBool(conf.LdapSkipTlsVerify) + ldapManagerDN := setting.GetStr(conf.LdapManagerDN) + ldapManagerPassword := setting.GetStr(conf.LdapManagerPassword) + ldapUserSearchBase := setting.GetStr(conf.LdapUserSearchBase) + ldapUserSearchFilter := setting.GetStr(conf.LdapUserSearchFilter) // (uid=%s) + + // Connect to LdapServer + l, err := dial(ldapServer, skipTlsVerify) + if err != nil { + return errors.WithMessagef(err, "failed to connect to LDAP") + } + defer l.Close() + + // First bind with a read only user + if ldapManagerDN != "" && ldapManagerPassword != "" { + err = l.Bind(ldapManagerDN, ldapManagerPassword) + if err != nil { + return errors.WithMessagef(err, "failed to bind to LDAP") + } + } + + // Search for the given username + searchRequest := ldap.NewSearchRequest( + ldapUserSearchBase, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + fmt.Sprintf(ldapUserSearchFilter, ldap.EscapeFilter(username)), + []string{"dn"}, + nil, + ) + sr, err := l.Search(searchRequest) + if err != nil { + return errors.WithMessagef(err, "failed login ldap: LDAP search failed") + } + if len(sr.Entries) != 1 { + return errors.New("failed login ldap: user does not exist or too many entries returned") + } + userDN := sr.Entries[0].DN + + // Bind as the user to verify their password + err = l.Bind(userDN, password) + if err != nil { + return errors.WithMessagef(ErrFailedLdapAuth, "%v", err) + } + log.Infof("LDAP auth successful for %s", username) + // Auth finished + return nil +} + +func LdapRegister(username string) (*model.User, error) { + if username == "" { + return nil, errors.New("cannot get username from ldap provider") + } + user := &model.User{ + Username: username, + Password: "", + Authn: "[]", + Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), + BasePath: setting.GetStr(conf.LdapDefaultDir), + Role: 0, + Disabled: false, + AllowLdap: true, + } + user.SetPassword(random.String(16)) + if err := op.CreateUser(user); err != nil { + return nil, err + } + return user, nil +} + +func dial(ldapServer string, skipTlsVerify ...bool) (*ldap.Conn, error) { + tlsEnabled := false + if strings.HasPrefix(ldapServer, "ldaps://") { + tlsEnabled = true + ldapServer = strings.TrimPrefix(ldapServer, "ldaps://") + } else if strings.HasPrefix(ldapServer, "ldap://") { + ldapServer = strings.TrimPrefix(ldapServer, "ldap://") + } + + if tlsEnabled { + return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: utils.IsBool(skipTlsVerify...)}) + } else { + return ldap.Dial("tcp", ldapServer) + } +} diff --git a/server/common/proxy.go b/server/common/proxy.go index d352582eb..c7c975d25 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -18,11 +18,11 @@ import ( ) func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error { - if link.MFile != nil { - attachHeader(w, file, link) - http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile) - return nil - } + // if link.MFile != nil { + // attachHeader(w, file, link) + // http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile) + // return nil + // } if link.Concurrency > 0 || link.PartSize > 0 { attachHeader(w, file, link) @@ -101,7 +101,7 @@ func GetEtag(file model.Obj, size int64) string { } func ProxyRange(ctx context.Context, link *model.Link, size int64) *model.Link { - if link.MFile == nil && link.RangeReader == nil && !strings.HasPrefix(link.URL, GetApiUrl(ctx)+"/") { + if link.RangeReader == nil && !strings.HasPrefix(link.URL, GetApiUrl(ctx)+"/") { if link.ContentLength > 0 { size = link.ContentLength } diff --git a/server/ftp.go b/server/ftp.go index fb7d86c8e..d07a62dd9 100644 --- a/server/ftp.go +++ b/server/ftp.go @@ -19,6 +19,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/ftp" ftpserver "github.com/fclairamb/ftpserverlib" ) @@ -112,6 +113,12 @@ func (d *FtpMainDriver) ClientDisconnected(cc ftpserver.ClientContext) { } func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) { + ip := cc.RemoteAddr().String() + count, ok := model.LoginCache.Get(ip) + if ok && count >= model.DefaultMaxAuthRetries { + model.LoginCache.Expire(ip, model.DefaultLockDuration) + return nil, errors.New("Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.") + } var userObj *model.User var err error if user == "anonymous" || user == "guest" { @@ -121,17 +128,24 @@ func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) } } else { userObj, err = op.GetUserByName(user) - if err != nil { - return nil, err + if err == nil { + err = userObj.ValidateRawPassword(pass) + if err != nil && setting.GetBool(conf.LdapLoginEnabled) && userObj.AllowLdap { + err = common.HandleLdapLogin(user, pass) + } + } else if setting.GetBool(conf.LdapLoginEnabled) && model.CanFTPAccess(int32(setting.GetInt(conf.LdapDefaultPermission, 0))) { + userObj, err = tryLdapLoginAndRegister(user, pass) } - passHash := model.StaticHash(pass) - if err = userObj.ValidatePwdStaticHash(passHash); err != nil { + if err != nil { + model.LoginCache.Set(ip, count+1) return nil, err } } if userObj.Disabled || !userObj.CanFTPAccess() { + model.LoginCache.Set(ip, count+1) return nil, errors.New("user is not allowed to access via FTP") } + model.LoginCache.Del(ip) ctx := context.Background() ctx = context.WithValue(ctx, conf.UserKey, userObj) @@ -140,7 +154,7 @@ func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) } else { ctx = context.WithValue(ctx, conf.MetaPassKey, "") } - ctx = context.WithValue(ctx, conf.ClientIPKey, cc.RemoteAddr().String()) + ctx = context.WithValue(ctx, conf.ClientIPKey, ip) ctx = context.WithValue(ctx, conf.ProxyHeaderKey, d.proxyHeader) return ftp.NewAferoAdapter(ctx), nil } @@ -167,7 +181,7 @@ func lookupIP(host string) string { } ips, err := net.LookupIP(host) if err != nil || len(ips) == 0 { - utils.Log.Fatalf("given FTP public host is invalid, and the default value will be used: %v", err) + utils.Log.Errorf("given FTP public host is invalid, and the default value will be used: %v", err) return "" } for _, ip := range ips { @@ -272,7 +286,7 @@ func newPortMapper(str string) ftpserver.PasvPortGetter { break } if err != nil { - utils.Log.Fatalf("failed to convert FTP PASV port mapper %s: %v, the port mapper will be ignored.", mapper, err) + utils.Log.Errorf("failed to convert FTP PASV port mapper %s: %v, the port mapper will be ignored.", mapper, err) return nil } } diff --git a/server/ftp/afero.go b/server/ftp/afero.go index 8b7b467f7..02c956a50 100644 --- a/server/ftp/afero.go +++ b/server/ftp/afero.go @@ -104,7 +104,7 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve return nil, err } if (flags & os.O_EXCL) != 0 { - return nil, errors.New("file already exists") + return nil, errs.ObjectAlreadyExists } if (flags & os.O_WRONLY) != 0 { return nil, errors.New("cannot write to uploading file") @@ -122,7 +122,7 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve return nil, errs.ObjectNotFound } if (flags&os.O_EXCL) != 0 && exists { - return nil, errors.New("file already exists") + return nil, errs.ObjectAlreadyExists } if (flags & os.O_WRONLY) != 0 { if offset != 0 { diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index 34fb36c63..c549a1943 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -15,7 +15,9 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ftpserver "github.com/fclairamb/ftpserverlib" "github.com/pkg/errors" @@ -49,6 +51,11 @@ func OpenUpload(ctx context.Context, path string, trunc bool) (*FileUploadProxy, if err != nil { return nil, err } + // Check if system file should be ignored + _, name := stdpath.Split(path) + if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) { + return nil, errs.IgnoredSystemFile + } tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err @@ -150,6 +157,11 @@ func OpenUploadWithLength(ctx context.Context, path string, trunc bool, length i if err != nil { return nil, err } + // Check if system file should be ignored + _, name := stdpath.Split(path) + if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) { + return nil, errs.IgnoredSystemFile + } if trunc { _ = fs.Remove(ctx, path) } diff --git a/server/handles/archive.go b/server/handles/archive.go index 418de6f23..4fd405688 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -1,7 +1,6 @@ package handles import ( - "encoding/json" "fmt" "io" stdpath "path" @@ -229,30 +228,15 @@ func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { }) } -type StringOrArray []string - -func (s *StringOrArray) UnmarshalJSON(data []byte) error { - var value string - if err := json.Unmarshal(data, &value); err == nil { - *s = []string{value} - return nil - } - var sliceValue []string - if err := json.Unmarshal(data, &sliceValue); err != nil { - return err - } - *s = sliceValue - return nil -} - type ArchiveDecompressReq struct { - SrcDir string `json:"src_dir" form:"src_dir"` - DstDir string `json:"dst_dir" form:"dst_dir"` - Name StringOrArray `json:"name" form:"name"` - ArchivePass string `json:"archive_pass" form:"archive_pass"` - InnerPath string `json:"inner_path" form:"inner_path"` - CacheFull bool `json:"cache_full" form:"cache_full"` - PutIntoNewDir bool `json:"put_into_new_dir" form:"put_into_new_dir"` + SrcDir string `json:"src_dir" form:"src_dir"` + DstDir string `json:"dst_dir" form:"dst_dir"` + Names []string `json:"name" form:"name"` + ArchivePass string `json:"archive_pass" form:"archive_pass"` + InnerPath string `json:"inner_path" form:"inner_path"` + CacheFull bool `json:"cache_full" form:"cache_full"` + PutIntoNewDir bool `json:"put_into_new_dir" form:"put_into_new_dir"` + Overwrite bool `json:"overwrite" form:"overwrite"` } func FsArchiveDecompress(c *gin.Context) { @@ -266,8 +250,8 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcPaths := make([]string, 0, len(req.Name)) - for _, name := range req.Name { + srcPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) if err != nil { common.ErrorResp(c, err, 403) @@ -295,6 +279,7 @@ func FsArchiveDecompress(c *gin.Context) { }, CacheFull: req.CacheFull, PutIntoNewDir: req.PutIntoNewDir, + Overwrite: req.Overwrite, }) if e != nil { if errors.Is(e, errs.WrongArchivePassword) { diff --git a/server/handles/auth.go b/server/handles/auth.go index 35776ba6a..780069091 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -45,27 +45,28 @@ func loginHash(c *gin.Context, req *LoginReq) { ip := c.ClientIP() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { - common.ErrorStrResp(c, "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.", 429) + common.ErrorStrResp(c, model.TooManyAttempts, 429) model.LoginCache.Expire(ip, model.DefaultLockDuration) return } // check username user, err := op.GetUserByName(req.Username) if err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // check 2FA if user.OtpSecret != "" { if !totp.Validate(req.OtpCode, user.OtpSecret) { - common.ErrorStrResp(c, "Invalid 2FA code", 402) + // 402 - need opt + common.ErrorStrResp(c, model.Invalid2FACode, 402) model.LoginCache.Set(ip, count+1) return } @@ -73,7 +74,7 @@ func loginHash(c *gin.Context, req *LoginReq) { // generate token token, err := common.GenerateToken(user) if err != nil { - common.ErrorResp(c, err, 400, true) + common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, gin.H{"token": token}) @@ -107,7 +108,7 @@ func UpdateCurrent(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not update profile", 403) + common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } user.Username = req.Username @@ -125,7 +126,7 @@ func UpdateCurrent(c *gin.Context) { func Generate2FA(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } key, err := totp.Generate(totp.GenerateOpts{ @@ -164,11 +165,11 @@ func Verify2FA(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } if !totp.Validate(req.Code, req.Secret) { - common.ErrorStrResp(c, "Invalid 2FA code", 400) + common.ErrorStrResp(c, model.Invalid2FACode, 400) return } user.OtpSecret = req.Secret diff --git a/server/handles/direct_upload.go b/server/handles/direct_upload.go new file mode 100644 index 000000000..69cfd2fa8 --- /dev/null +++ b/server/handles/direct_upload.go @@ -0,0 +1,54 @@ +package handles + +import ( + "net/url" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +type FsGetDirectUploadInfoReq struct { + Path string `json:"path" form:"path"` + FileName string `json:"file_name" form:"file_name"` + FileSize int64 `json:"file_size" form:"file_size"` + Tool string `json:"tool" form:"tool"` +} + +// FsGetDirectUploadInfo returns the direct upload info if supported by the driver +// If the driver does not support direct upload, returns null for upload_info +func FsGetDirectUploadInfo(c *gin.Context) { + var req FsGetDirectUploadInfoReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + // Decode path + path, err := url.PathUnescape(req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + // Get user and join path + user := c.Request.Context().Value(conf.UserKey).(*model.User) + path, err = user.JoinPath(path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + overwrite := c.GetHeader("Overwrite") != "false" + if !overwrite { + if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, "file exists", 403) + return + } + } + directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, directUploadInfo) +} diff --git a/server/handles/down.go b/server/handles/down.go index 84ebdc44f..d4d634cbe 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -9,6 +9,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" @@ -147,7 +148,7 @@ func proxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) { if Writer.IsWritten() { log.Errorf("%s %s local proxy error: %+v", c.Request.Method, c.Request.URL.Path, err) } else { - if statusCode, ok := errors.Unwrap(err).(net.HttpStatusCodeError); ok { + if statusCode, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { common.ErrorPage(c, err, int(statusCode), true) } else { common.ErrorPage(c, err, 500, true) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index f45da69bb..62382a27c 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -6,18 +6,18 @@ import ( "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/task" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/generic" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type MkdirOrLinkReq struct { @@ -57,10 +57,12 @@ func FsMkdir(c *gin.Context) { } type MoveCopyReq struct { - SrcDir string `json:"src_dir"` - DstDir string `json:"dst_dir"` - Names []string `json:"names"` - Overwrite bool `json:"overwrite"` + SrcDir string `json:"src_dir"` + DstDir string `json:"dst_dir"` + Names []string `json:"names"` + Overwrite bool `json:"overwrite"` + SkipExisting bool `json:"skip_existing"` + Merge bool `json:"merge"` } func FsMove(c *gin.Context) { @@ -78,31 +80,44 @@ func FsMove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcDir, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) - return - } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - if !req.Overwrite { - for _, name := range req.Names { - if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { - common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + validPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + // ensure req.Names is not a relative path + srcPath := stdpath.Join(req.SrcDir, name) + srcPath, err = user.JoinPath(srcPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !req.Overwrite { + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) return } + if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { + if !req.SkipExisting { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + return + } else { + continue + } + } } + validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo - for i, name := range req.Names { - t, err := fs.Move(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + for i, p := range validPaths { + t, err := fs.Move(c.Request.Context(), p, dstDir, len(validPaths) > i+1) if t != nil { addedTasks = append(addedTasks, t) } @@ -140,31 +155,49 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcDir, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) - return - } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - if !req.Overwrite { - for _, name := range req.Names { - if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { - common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + validPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + // ensure req.Names is not a relative path + srcPath := stdpath.Join(req.SrcDir, name) + srcPath, err = user.JoinPath(srcPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !req.Overwrite { + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) return } + if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { + if !req.SkipExisting && !req.Merge { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + return + } else if !req.Merge || !res.IsDir() { + continue + } + } } + validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo - for i, name := range req.Names { - t, err := fs.Copy(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + for i, p := range validPaths { + var t task.TaskExtensionInfo + if req.Merge { + t, err = fs.Merge(c.Request.Context(), p, dstDir, len(validPaths) > i+1) + } else { + t, err = fs.Copy(c.Request.Context(), p, dstDir, len(validPaths) > i+1) + } if t != nil { addedTasks = append(addedTasks, t) } @@ -255,13 +288,25 @@ func FsRemove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - reqDir, err := user.JoinPath(req.Dir) - if err != nil { - common.ErrorResp(c, err, 403) - return + for i, name := range req.Names { + if strings.TrimSpace(utils.FixAndCleanPath(name)) == "/" { + log.Warnf("FsRemove: invalid item skipped: %s (parent directory: %s)\n", name, req.Dir) + req.Names[i] = "" + continue + } + // ensure req.Names is not a relative path + var err error + req.Names[i], err = user.JoinPath(stdpath.Join(req.Dir, name)) + if err != nil { + common.ErrorResp(c, err, 403) + return + } } - for _, name := range req.Names { - err := fs.Remove(c.Request.Context(), stdpath.Join(reqDir, name)) + for _, path := range req.Names { + if path == "" { + continue + } + err := fs.Remove(c.Request.Context(), path) if err != nil { common.ErrorResp(c, err, 500) return @@ -386,7 +431,7 @@ func Link(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if storage.Config().NoLinkURL || storage.Config().OnlyLinkMFile { + if storage.Config().NoLinkURL { common.SuccessResp(c, model.Link{ URL: fmt.Sprintf("%s/p%s?d&sign=%s", common.GetApiUrl(c), diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 6665094cd..886da9dc9 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -33,28 +33,27 @@ type DirReq struct { } type ObjResp struct { - Id string `json:"id"` - Path string `json:"path"` - Name string `json:"name"` - Size int64 `json:"size"` - IsDir bool `json:"is_dir"` - Modified time.Time `json:"modified"` - Created time.Time `json:"created"` - Sign string `json:"sign"` - Thumb string `json:"thumb"` - Type int `json:"type"` - HashInfoStr string `json:"hashinfo"` - HashInfo map[*utils.HashType]string `json:"hash_info"` - MountDetails *model.StorageDetailsWithName `json:"mount_details,omitempty"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + MountDetails *model.StorageDetails `json:"mount_details,omitempty"` } type FsListResp struct { - Content []ObjResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` + Content []ObjResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + Provider string `json:"provider"` + DirectUploadTools []string `json:"direct_upload_tools,omitempty"` } func FsListSplit(c *gin.Context) { @@ -109,17 +108,20 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { } total, objs := pagination(objs, &req.PageReq) provider := "unknown" - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) - if err == nil { - provider = storage.GetStorage().Driver + var directUploadTools []string + if user.CanWrite() { + if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { + directUploadTools = op.GetDirectUploadTools(storage) + } } common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), - Total: int64(total), - Readme: getReadme(meta, reqPath), - Header: getHeader(meta, reqPath), - Write: user.CanWrite() || common.CanWrite(meta, reqPath), - Provider: provider, + Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), + Total: int64(total), + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Write: user.CanWrite() || common.CanWrite(meta, reqPath), + Provider: provider, + DirectUploadTools: directUploadTools, }) } @@ -230,8 +232,6 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) resp = append(resp, ObjResp{ - Id: obj.GetID(), - Path: obj.GetPath(), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), @@ -361,8 +361,6 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { mountDetails, _ := model.GetStorageDetails(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ - Id: obj.GetID(), - Path: obj.GetPath(), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 71d9dbae7..0f46398cd 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -8,8 +8,10 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -28,6 +30,14 @@ func getLastModified(c *gin.Context) time.Time { return lastModified } +// shouldIgnoreSystemFile checks if the filename should be ignored based on settings +func shouldIgnoreSystemFile(filename string) bool { + if setting.GetBool(conf.IgnoreSystemFiles) { + return utils.IsSystemFile(filename) + } + return false +} + func FsStream(c *gin.Context) { defer func() { if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 { @@ -56,6 +66,11 @@ func FsStream(c *gin.Context) { } } dir, name := stdpath.Split(path) + // Check if system file should be ignored + if shouldIgnoreSystemFile(name) { + common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403) + return + } // 如果请求头 Content-Length 和 X-File-Size 都没有，则 size=-1，表示未知大小的流式上传 size := c.Request.ContentLength if size < 0 { @@ -97,7 +112,7 @@ func FsStream(c *gin.Context) { if asTask { t, err = fs.PutAsTask(c.Request.Context(), dir, s) } else { - err = fs.PutDirectly(c.Request.Context(), dir, s, true) + err = fs.PutDirectly(c.Request.Context(), dir, s) } if err != nil { common.ErrorResp(c, err, 500) @@ -160,6 +175,11 @@ func FsForm(c *gin.Context) { } defer f.Close() dir, name := stdpath.Split(path) + // Check if system file should be ignored + if shouldIgnoreSystemFile(name) { + common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403) + return + } h := make(map[*utils.HashType]string) if md5 := c.GetHeader("X-File-Md5"); md5 != "" { h[utils.MD5] = md5 @@ -192,7 +212,7 @@ func FsForm(c *gin.Context) { }{f} t, err = fs.PutAsTask(c.Request.Context(), dir, s) } else { - err = fs.PutDirectly(c.Request.Context(), dir, s, true) + err = fs.PutDirectly(c.Request.Context(), dir, s) } if err != nil { common.ErrorResp(c, err, 500) diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go index dff3aebe3..ba44615f7 100644 --- a/server/handles/ldap_login.go +++ b/server/handles/ldap_login.go @@ -1,21 +1,13 @@ package handles import ( - "crypto/tls" - "errors" - "fmt" - "strings" - "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" - "gopkg.in/ldap.v3" + "github.com/pkg/errors" ) func LoginLdap(c *gin.Context) { @@ -24,15 +16,16 @@ func LoginLdap(c *gin.Context) { common.ErrorResp(c, err, 400) return } - loginLdap(c, &req) -} - -func loginLdap(c *gin.Context, req *LoginReq) { enabled := setting.GetBool(conf.LdapLoginEnabled) if !enabled { common.ErrorStrResp(c, "ldap is not enabled", 403) return } + user, err := op.GetUserByName(req.Username) + if err == nil && !user.AllowLdap { + common.ErrorStrResp(c, "login via ldap is not allowed", 403) + return + } // check count of login ip := c.ClientIP() @@ -43,67 +36,19 @@ func loginLdap(c *gin.Context, req *LoginReq) { return } - // Auth start - ldapServer := setting.GetStr(conf.LdapServer) - ldapManagerDN := setting.GetStr(conf.LdapManagerDN) - ldapManagerPassword := setting.GetStr(conf.LdapManagerPassword) - ldapUserSearchBase := setting.GetStr(conf.LdapUserSearchBase) - ldapUserSearchFilter := setting.GetStr(conf.LdapUserSearchFilter) // (uid=%s) - - // Connect to LdapServer - l, err := dial(ldapServer) + err = common.HandleLdapLogin(req.Username, req.Password) if err != nil { - utils.Log.Errorf("failed to connect to LDAP: %v", err) - common.ErrorResp(c, err, 500) - return - } - - // First bind with a read only user - if ldapManagerDN != "" && ldapManagerPassword != "" { - err = l.Bind(ldapManagerDN, ldapManagerPassword) - if err != nil { - utils.Log.Errorf("Failed to bind to LDAP: %v", err) + if errors.Is(err, common.ErrFailedLdapAuth) { + model.LoginCache.Set(ip, count+1) + common.ErrorResp(c, err, 400) + } else { common.ErrorResp(c, err, 500) - return } - } - - // Search for the given username - searchRequest := ldap.NewSearchRequest( - ldapUserSearchBase, - ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, - fmt.Sprintf(ldapUserSearchFilter, req.Username), - []string{"dn"}, - nil, - ) - sr, err := l.Search(searchRequest) - if err != nil { - utils.Log.Errorf("LDAP search failed: %v", err) - common.ErrorResp(c, err, 500) - return - } - if len(sr.Entries) != 1 { - utils.Log.Errorf("User does not exist or too many entries returned") - common.ErrorResp(c, err, 500) return } - userDN := sr.Entries[0].DN - // Bind as the user to verify their password - err = l.Bind(userDN, req.Password) - if err != nil { - utils.Log.Errorf("Failed to auth. %v", err) - common.ErrorResp(c, err, 400) - model.LoginCache.Set(ip, count+1) - return - } else { - utils.Log.Infof("Auth successful username:%s", req.Username) - } - // Auth finished - - user, err := op.GetUserByName(req.Username) - if err != nil { - user, err = ladpRegister(req.Username) + if user == nil { + user, err = common.LdapRegister(req.Username) if err != nil { common.ErrorResp(c, err, 400) model.LoginCache.Set(ip, count+1) @@ -120,38 +65,3 @@ func loginLdap(c *gin.Context, req *LoginReq) { common.SuccessResp(c, gin.H{"token": token}) model.LoginCache.Del(ip) } - -func ladpRegister(username string) (*model.User, error) { - if username == "" { - return nil, errors.New("cannot get username from ldap provider") - } - user := &model.User{ - ID: 0, - Username: username, - Password: random.String(16), - Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), - BasePath: setting.GetStr(conf.LdapDefaultDir), - Role: 0, - Disabled: false, - } - if err := db.CreateUser(user); err != nil { - return nil, err - } - return user, nil -} - -func dial(ldapServer string) (*ldap.Conn, error) { - var tlsEnabled bool = false - if strings.HasPrefix(ldapServer, "ldaps://") { - tlsEnabled = true - ldapServer = strings.TrimPrefix(ldapServer, "ldaps://") - } else if strings.HasPrefix(ldapServer, "ldap://") { - ldapServer = strings.TrimPrefix(ldapServer, "ldap://") - } - - if tlsEnabled { - return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: true}) - } else { - return ldap.Dial("tcp", ldapServer) - } -} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index fb6527349..153b27293 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -2,9 +2,11 @@ package handles import ( "strings" - + _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" + _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" @@ -200,6 +202,96 @@ func Set115Open(c *gin.Context) { common.SuccessResp(c, "ok") } +type Set123PanReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func Set123Pan(c *gin.Context) { + var req Set123PanReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*_123.Pan123); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123Pan is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.Pan123TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("123Pan") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type Set123OpenReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` + CallbackUrl string `json:"callback_url" form:"callback_url"` +} + +func Set123Open(c *gin.Context) { + var req Set123OpenReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*_123_open.Open123); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123 Open is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.Pan123OpenTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.Pan123OpenOfflineDownloadCallbackUrl, Value: req.CallbackUrl, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("123 Open") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + type SetPikPakReq struct { TempDir string `json:"temp_dir" form:"temp_dir"` } @@ -413,7 +505,7 @@ func AddOfflineDownload(c *gin.Context) { if trimmedUrl == "" { continue } - + t, err := tool.AddURL(c, &tool.AddURLArgs{ URL: trimmedUrl, DstDirPath: reqPath, diff --git a/server/handles/scan.go b/server/handles/scan.go new file mode 100644 index 000000000..fc5e80f66 --- /dev/null +++ b/server/handles/scan.go @@ -0,0 +1,47 @@ +package handles + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +type ManualScanReq struct { + Path string `json:"path"` + Limit float64 `json:"limit"` +} + +func StartManualScan(c *gin.Context) { + var req ManualScanReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.BeginManualScan(req.Path, req.Limit); err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c) +} + +func StopManualScan(c *gin.Context) { + if !op.ManualScanRunning() { + common.ErrorStrResp(c, "manual scan is not running", 400) + return + } + op.StopManualScan() + common.SuccessResp(c) +} + +type ManualScanResp struct { + ObjCount uint64 `json:"obj_count"` + IsDone bool `json:"is_done"` +} + +func GetManualScanProgress(c *gin.Context) { + ret := ManualScanResp{ + ObjCount: op.ScannedCount.Load(), + IsDone: !op.ManualScanRunning(), + } + common.SuccessResp(c, ret) +} diff --git a/server/handles/sharing.go b/server/handles/sharing.go index 5a6ea265f..43f855afb 100644 --- a/server/handles/sharing.go +++ b/server/handles/sharing.go @@ -34,9 +34,9 @@ func SharingGet(c *gin.Context, req *FsGetReq) { return } _ = countAccess(c.ClientIP(), s) - fakePath := fmt.Sprintf("/%s/%s", sid, path) url := "" if !obj.IsDir() { + fakePath := fmt.Sprintf("/%s/%s", sid, path) url = fmt.Sprintf("%s/sd%s", common.GetApiUrl(c), utils.EncodePath(fakePath, true)) if s.Pwd != "" { url += "?pwd=" + s.Pwd @@ -45,8 +45,6 @@ func SharingGet(c *gin.Context, req *FsGetReq) { thumb, _ := model.GetThumb(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ - Id: "", - Path: fakePath, Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), @@ -80,14 +78,11 @@ func SharingList(c *gin.Context, req *ListReq) { return } _ = countAccess(c.ClientIP(), s) - fakePath := fmt.Sprintf("/%s/%s", sid, path) total, objs := pagination(objs, &req.PageReq) common.SuccessResp(c, FsListResp{ Content: utils.MustSliceConvert(objs, func(obj model.Obj) ObjResp { thumb, _ := model.GetThumb(obj) return ObjResp{ - Id: "", - Path: stdpath.Join(fakePath, obj.GetName()), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), @@ -408,7 +403,7 @@ func ListSharings(c *gin.Context) { }) } -type CreateSharingReq struct { +type UpdateSharingReq struct { Files []string `json:"files"` Expires *time.Time `json:"expires"` Pwd string `json:"pwd"` @@ -418,12 +413,9 @@ type CreateSharingReq struct { Readme string `json:"readme"` Header string `json:"header"` model.Sort -} - -type UpdateSharingReq struct { - ID string `json:"id"` - Accessed int `json:"accessed"` - CreateSharingReq + CreatorName string `json:"creator"` + Accessed int `json:"accessed"` + ID string `json:"id"` } func UpdateSharing(c *gin.Context) { @@ -436,24 +428,38 @@ func UpdateSharing(c *gin.Context) { common.ErrorStrResp(c, "must add at least 1 object", 400) return } - user := c.Request.Context().Value(conf.UserKey).(*model.User) - if !user.CanShare() { - common.ErrorStrResp(c, "permission denied", 403) - return + var user *model.User + var err error + reqUser := c.Request.Context().Value(conf.UserKey).(*model.User) + if reqUser.IsAdmin() && req.CreatorName != "" { + user, err = op.GetUserByName(req.CreatorName) + if err != nil { + common.ErrorStrResp(c, "no such a user", 400) + return + } + } else { + user = reqUser + if !user.CanShare() { + common.ErrorStrResp(c, "permission denied", 403) + return + } } for i, s := range req.Files { s = utils.FixAndCleanPath(s) req.Files[i] = s - if !user.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { + if !reqUser.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { common.ErrorStrResp(c, fmt.Sprintf("permission denied to share path [%s]", s), 500) return } } s, err := op.GetSharingById(req.ID) - if err != nil || (!user.IsAdmin() && s.CreatorId != user.ID) { + if err != nil || (!reqUser.IsAdmin() && s.CreatorId != user.ID) { common.ErrorStrResp(c, "sharing not found", 404) return } + if reqUser.IsAdmin() && req.CreatorName == "" { + user = s.Creator + } s.Files = req.Files s.Expires = req.Expires s.Pwd = req.Pwd @@ -464,6 +470,7 @@ func UpdateSharing(c *gin.Context) { s.Header = req.Header s.Readme = req.Readme s.Remark = req.Remark + s.Creator = user if err = op.UpdateSharing(s); err != nil { common.ErrorResp(c, err, 500) } else { @@ -476,7 +483,7 @@ func UpdateSharing(c *gin.Context) { } func CreateSharing(c *gin.Context) { - var req CreateSharingReq + var req UpdateSharingReq var err error if err = c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) @@ -486,24 +493,35 @@ func CreateSharing(c *gin.Context) { common.ErrorStrResp(c, "must add at least 1 object", 400) return } - user := c.Request.Context().Value(conf.UserKey).(*model.User) - if !user.CanShare() { - common.ErrorStrResp(c, "permission denied", 403) - return + var user *model.User + reqUser := c.Request.Context().Value(conf.UserKey).(*model.User) + if reqUser.IsAdmin() && req.CreatorName != "" { + user, err = op.GetUserByName(req.CreatorName) + if err != nil { + common.ErrorStrResp(c, "no such a user", 400) + return + } + } else { + user = reqUser + if !user.CanShare() || (!user.IsAdmin() && req.ID != "") { + common.ErrorStrResp(c, "permission denied", 403) + return + } } for i, s := range req.Files { s = utils.FixAndCleanPath(s) req.Files[i] = s - if !user.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { + if !reqUser.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { common.ErrorStrResp(c, fmt.Sprintf("permission denied to share path [%s]", s), 500) return } } s := &model.Sharing{ SharingDB: &model.SharingDB{ + ID: req.ID, Expires: req.Expires, Pwd: req.Pwd, - Accessed: 0, + Accessed: req.Accessed, MaxAccessed: req.MaxAccessed, Disabled: req.Disabled, Sort: req.Sort, diff --git a/server/handles/storage.go b/server/handles/storage.go index e802caad4..687c93551 100644 --- a/server/handles/storage.go +++ b/server/handles/storage.go @@ -4,7 +4,6 @@ import ( "context" "errors" "strconv" - "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -24,9 +23,15 @@ type StorageResp struct { MountDetails *model.StorageDetails `json:"mount_details,omitempty"` } -func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp { +type detailWithIndex struct { + idx int + val *model.StorageDetails +} + +func makeStorageResp(ctx *gin.Context, storages []model.Storage) []*StorageResp { ret := make([]*StorageResp, len(storages)) - var wg sync.WaitGroup + detailsChan := make(chan detailWithIndex, len(storages)) + workerCount := 0 for i, s := range storages { ret[i] = &StorageResp{ Storage: s, @@ -43,22 +48,26 @@ func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp { if !ok { continue } - wg.Add(1) - go func() { - defer wg.Done() - ctx, cancel := context.WithTimeout(c, time.Second*3) - defer cancel() - details, err := op.GetStorageDetails(ctx, d) - if err != nil { - if !errors.Is(err, errs.NotImplement) { - log.Errorf("failed get %s details: %+v", s.MountPath, err) + workerCount++ + go func(dri driver.Driver, idx int) { + details, e := op.GetStorageDetails(ctx, dri) + if e != nil { + if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { + log.Errorf("failed get %s details: %+v", dri.GetStorage().MountPath, e) } - return } - ret[i].MountDetails = details - }() + detailsChan <- detailWithIndex{idx: idx, val: details} + }(d, i) + } + for workerCount > 0 { + select { + case r := <-detailsChan: + ret[r.idx].MountDetails = r.val + workerCount-- + case <-time.After(time.Second * 3): + workerCount = 0 + } } - wg.Wait() return ret } @@ -175,7 +184,7 @@ func LoadAllStorages(c *gin.Context) { common.ErrorResp(c, err, 500, true) return } - conf.StoragesLoaded = false + conf.ResetStoragesLoadSignal() go func(storages []model.Storage) { for _, storage := range storages { storageDriver, err := op.GetStorageByMountPath(storage.MountPath) @@ -195,7 +204,7 @@ func LoadAllStorages(c *gin.Context) { log.Infof("success load storage: [%s], driver: [%s]", storage.MountPath, storage.Driver) } - conf.StoragesLoaded = true + conf.SendStoragesLoadedSignal() }(storages) common.SuccessResp(c) } diff --git a/server/middlewares/check.go b/server/middlewares/check.go index a1011de3b..c7203a490 100644 --- a/server/middlewares/check.go +++ b/server/middlewares/check.go @@ -22,9 +22,12 @@ func StoragesLoaded(c *gin.Context) { return } } - common.ErrorStrResp(c, "Loading storage, please wait", 500) - c.Abort() - return + select { + case <-conf.StoragesLoadSignal(): + case <-c.Request.Context().Done(): + c.Abort() + return + } } common.GinWithValue(c, conf.ApiUrlKey, common.GetApiUrlFromRequest(c.Request), diff --git a/server/middlewares/search.go b/server/middlewares/search.go index 7807a210b..dc89a08ad 100644 --- a/server/middlewares/search.go +++ b/server/middlewares/search.go @@ -11,7 +11,7 @@ import ( func SearchIndex(c *gin.Context) { mode := setting.GetStr(conf.SearchIndex) if mode == "none" { - common.ErrorResp(c, errs.SearchNotAvailable, 500) + common.ErrorResp(c, errs.SearchNotAvailable, 404) c.Abort() } else { c.Next() diff --git a/server/router.go b/server/router.go index 66f0539ba..57d1166ae 100644 --- a/server/router.go +++ b/server/router.go @@ -160,6 +160,8 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_transmission", handles.SetTransmission) setting.POST("/set_115", handles.Set115) setting.POST("/set_115_open", handles.Set115Open) + setting.POST("/set_123_pan", handles.Set123Pan) + setting.POST("/set_123_open", handles.Set123Open) setting.POST("/set_pikpak", handles.SetPikPak) setting.POST("/set_thunder", handles.SetThunder) setting.POST("/set_thunderx", handles.SetThunderX) @@ -178,6 +180,11 @@ func admin(g *gin.RouterGroup) { index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) + + scan := g.Group("/scan") + scan.POST("/start", handles.StartManualScan) + scan.POST("/stop", handles.StopManualScan) + scan.GET("/progress", handles.GetManualScanProgress) } func fsAndShare(g *gin.RouterGroup) { @@ -210,6 +217,8 @@ func _fs(g *gin.RouterGroup) { // g.POST("/add_transmission", handles.SetTransmission) g.POST("/add_offline_download", handles.AddOfflineDownload) g.POST("/archive/decompress", handles.FsArchiveDecompress) + // Direct upload (client-side upload to storage) + g.POST("/get_direct_upload_info", middlewares.FsUp, handles.FsGetDirectUploadInfo) } func _task(g *gin.RouterGroup) { diff --git a/server/s3/backend.go b/server/s3/backend.go index ac398699c..f00d2149c 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -19,6 +19,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -257,7 +258,7 @@ func (b *s3Backend) PutObject( if err != nil { if errs.IsObjectNotFound(err) && strings.Contains(objectName, "/") { log.Debugf("reqPath: %s not found and objectName contains /, need to makeDir", reqPath) - err = fs.MakeDir(ctx, reqPath, true) + err = fs.MakeDir(ctx, reqPath) if err != nil { return result, errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath) } @@ -280,12 +281,21 @@ func (b *s3Backend) PutObject( ti, _ = swift.FloatStringToTime(val) } + // If Modified is not set, use current time + if ti.IsZero() { + ti = time.Now() + } + obj := model.Object{ Name: path.Base(fp), Size: size, Modified: ti, Ctime: time.Now(), } + // Check if system file should be ignored + if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { + return result, errs.IgnoredSystemFile + } stream := &stream.FileStream{ Obj: &obj, Reader: input, diff --git a/server/sftp.go b/server/sftp.go index 055f79730..37dc9870d 100644 --- a/server/sftp.go +++ b/server/sftp.go @@ -11,6 +11,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/ftp" "github.com/OpenListTeam/OpenList/v4/server/sftp" "github.com/OpenListTeam/sftpd-openlist" @@ -92,17 +93,31 @@ func (d *SftpDriver) NoClientAuth(conn ssh.ConnMetadata) (*ssh.Permissions, erro } func (d *SftpDriver) PasswordAuth(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) { + ip := conn.RemoteAddr().String() + count, ok := model.LoginCache.Get(ip) + if ok && count >= model.DefaultMaxAuthRetries { + model.LoginCache.Expire(ip, model.DefaultLockDuration) + return nil, errors.New("Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.") + } + pass := string(password) userObj, err := op.GetUserByName(conn.User()) + if err == nil { + err = userObj.ValidateRawPassword(pass) + if err != nil && setting.GetBool(conf.LdapLoginEnabled) && userObj.AllowLdap { + err = common.HandleLdapLogin(conn.User(), pass) + } + } else if setting.GetBool(conf.LdapLoginEnabled) && model.CanFTPAccess(int32(setting.GetInt(conf.LdapDefaultPermission, 0))) { + userObj, err = tryLdapLoginAndRegister(conn.User(), pass) + } if err != nil { + model.LoginCache.Set(ip, count+1) return nil, err } if userObj.Disabled || !userObj.CanFTPAccess() { + model.LoginCache.Set(ip, count+1) return nil, errors.New("user is not allowed to access via SFTP") } - passHash := model.StaticHash(string(password)) - if err = userObj.ValidatePwdStaticHash(passHash); err != nil { - return nil, err - } + model.LoginCache.Del(ip) return nil, nil } diff --git a/server/sftp/hostkey.go b/server/sftp/hostkey.go index ffa25c665..6d857666a 100644 --- a/server/sftp/hostkey.go +++ b/server/sftp/hostkey.go @@ -24,7 +24,7 @@ func InitHostKey() { if !utils.Exists(sshPath) { err := utils.CreateNestedDirectory(sshPath) if err != nil { - utils.Log.Fatalf("failed to create ssh directory: %+v", err) + utils.Log.Errorf("failed to create ssh directory: %+v", err) return } } @@ -54,30 +54,30 @@ func LoadOrGenerateRSAHostKey(parentDir string) (ssh.Signer, bool) { _ = os.Remove(publicKeyPath) privateKey, err := rsa.GenerateKey(rand.Reader, 4096) if err != nil { - utils.Log.Fatalf("failed to generate RSA private key: %+v", err) + utils.Log.Errorf("failed to generate RSA private key: %+v", err) return nil, false } publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey) if err != nil { - utils.Log.Fatalf("failed to generate RSA public key: %+v", err) + utils.Log.Errorf("failed to generate RSA public key: %+v", err) return nil, false } ret, err := ssh.NewSignerFromKey(privateKey) if err != nil { - utils.Log.Fatalf("failed to generate RSA signer: %+v", err) + utils.Log.Errorf("failed to generate RSA signer: %+v", err) return nil, false } privateBytes := rsaEncodePrivateKey(privateKey) publicBytes := ssh.MarshalAuthorizedKey(publicKey) err = os.WriteFile(privateKeyPath, privateBytes, 0600) if err != nil { - utils.Log.Fatalf("failed to write RSA private key to file: %+v", err) + utils.Log.Errorf("failed to write RSA private key to file: %+v", err) return nil, false } err = os.WriteFile(publicKeyPath, publicBytes, 0644) if err != nil { _ = os.Remove(privateKeyPath) - utils.Log.Fatalf("failed to write RSA public key to file: %+v", err) + utils.Log.Errorf("failed to write RSA public key to file: %+v", err) return nil, false } return ret, true diff --git a/server/utils.go b/server/utils.go new file mode 100644 index 000000000..0e9d68809 --- /dev/null +++ b/server/utils.go @@ -0,0 +1,14 @@ +package server + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" +) + +func tryLdapLoginAndRegister(user, pass string) (*model.User, error) { + err := common.HandleLdapLogin(user, pass) + if err != nil { + return nil, err + } + return common.LdapRegister(user) +} diff --git a/server/webdav.go b/server/webdav.go index b2afe581d..789236b8b 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -6,14 +6,13 @@ import ( "path" "strings" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/middlewares" - - "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/server/webdav" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -94,8 +93,8 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - user, err := op.GetUserByName(username) - if err != nil || user.ValidateRawPassword(password) != nil { + user, ok := tryLogin(username, password) + if !ok { if c.Request.Method == "OPTIONS" { common.GinWithValue(c, conf.UserKey, guest) c.Next() @@ -146,3 +145,16 @@ func WebDAVAuth(c *gin.Context) { common.GinWithValue(c, conf.UserKey, user) c.Next() } + +func tryLogin(username, password string) (*model.User, bool) { + user, err := op.GetUserByName(username) + if err == nil { + err = user.ValidateRawPassword(password) + if err != nil && setting.GetBool(conf.LdapLoginEnabled) && user.AllowLdap { + err = common.HandleLdapLogin(username, password) + } + } else if setting.GetBool(conf.LdapLoginEnabled) && model.CanWebdavRead(int32(setting.GetInt(conf.LdapDefaultPermission, 0))) { + user, err = tryLdapLoginAndRegister(username, password) + } + return user, err == nil +} diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 0c4f0922c..504c5fc1d 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/net" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -272,7 +273,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta } err = common.Proxy(w, r, link, fi) if err != nil { - if statusCode, ok := errors.Unwrap(err).(net.HttpStatusCodeError); ok { + if statusCode, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { return int(statusCode), err } return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err) @@ -358,6 +359,10 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, Modified: h.getModTime(r), Ctime: h.getCreateTime(r), } + // Check if system file should be ignored + if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { + return http.StatusForbidden, errs.IgnoredSystemFile + } fsStream := &stream.FileStream{ Obj: &obj, Reader: r.Body,