From f0e53d18a8d71687e12f757fc268afc2c255fedb Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 26 Jan 2026 19:31:09 +0800 Subject: [PATCH 01/86] fix(drivers/alias): default sort & substitute link (#1917) * fix(drivers/alias): default sort & substitute link * fix * fix --- drivers/alias/driver.go | 48 +++++++++++++++++++++++++++++++---------- drivers/alias/util.go | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 64376957e..e1ba41eb6 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -229,6 +229,15 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ for _, obj := range objMap { objs = append(objs, obj) } + if d.OrderBy == "" { + sort := getAllSort(dirs) + if sort.OrderBy != "" { + model.SortFiles(objs, sort.OrderBy, sort.OrderDirection) + } + if d.ExtractFolder == "" && sort.ExtractFolder != "" { + model.ExtractFolder(objs, sort.ExtractFolder) + } + } return objs, nil } @@ -276,21 +285,38 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( }, nil } - reqPath := d.getBalancedPath(ctx, file) - link, fi, err := d.link(ctx, reqPath, args) + var link *model.Link + var fi model.Obj + var err error + files := file.(BalancedObjs) + if d.ReadConflictPolicy == RandomBalancedRP || d.ReadConflictPolicy == AllRWP { + rand.Shuffle(len(files), func(i, j int) { + files[i], files[j] = files[j], files[i] + }) + } + for _, f := range files { + if f == nil { + continue + } + link, fi, err = d.link(ctx, f.GetPath(), args) + if err == nil { + if link == nil { + // 重定向且需要通过代理 + return &model.Link{ + URL: fmt.Sprintf("%s/p%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(f.GetPath(), true), + sign.Sign(f.GetPath())), + }, nil + } + break + } + } if err != nil { return nil, err } - if link == nil { - // 重定向且需要通过代理 - return &model.Link{ - URL: fmt.Sprintf("%s/p%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)), - }, nil - } resultLink := *link // 复制一份，避免修改到原始link + resultLink.Expiration = nil resultLink.SyncClosers = utils.NewSyncClosers(link) if args.Redirect { return &resultLink, nil diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 7336b9ba9..8e5eb8a84 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -490,3 +490,43 @@ func (d *Alias) extract(ctx context.Context, reqPath string, args model.ArchiveI link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) return link, err } + +func getAllSort(dirs []model.Obj) model.Sort { + ret := model.Sort{} + noSort := false + noExtractFolder := false + for _, dir := range dirs { + if dir == nil { + continue + } + storage, err := fs.GetStorage(dir.GetPath(), &fs.GetStoragesArgs{}) + if err != nil { + continue + } + if !noSort && storage.GetStorage().OrderBy != "" { + if ret.OrderBy == "" { + ret.OrderBy = storage.GetStorage().OrderBy + ret.OrderDirection = storage.GetStorage().OrderDirection + if ret.OrderDirection == "" { + ret.OrderDirection = "asc" + } + } else if ret.OrderBy != storage.GetStorage().OrderBy || ret.OrderDirection != storage.GetStorage().OrderDirection { + ret.OrderBy = "" + ret.OrderDirection = "" + noSort = true + } + } + if !noExtractFolder && storage.GetStorage().ExtractFolder != "" { + if ret.ExtractFolder == "" { + ret.ExtractFolder = storage.GetStorage().ExtractFolder + } else if ret.ExtractFolder != storage.GetStorage().ExtractFolder { + ret.ExtractFolder = "" + noExtractFolder = true + } + } + if noSort && noExtractFolder { + break + } + } + return ret +} From 29fcf5904acff340e738b84d8be1d19b1fee8e0e Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Wed, 28 Jan 2026 19:30:52 +0800 Subject: [PATCH 02/86] fix(drivers/cloudreve_v4): add IsFolder attribute to Getter response (#2035) * fix(drivers/cloudreve_v4): add IsFolder attribute to Getter response Signed-off-by: MadDogOwner * refactor(drivers/cloudreve_v4): implement File.fileToObject method Signed-off-by: MadDogOwner * fix(drivers/cloudreve_v4): implement 404 not found for getter Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner --- drivers/cloudreve_v4/driver.go | 19 ++----------------- drivers/cloudreve_v4/types.go | 12 ++++++++++++ drivers/cloudreve_v4/util.go | 5 +++++ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/drivers/cloudreve_v4/driver.go b/drivers/cloudreve_v4/driver.go index a1d301635..cd5cf1b3b 100644 --- a/drivers/cloudreve_v4/driver.go +++ b/drivers/cloudreve_v4/driver.go @@ -129,15 +129,7 @@ func (d *CloudreveV4) List(ctx context.Context, dir model.Obj, args model.ListAr } } return &model.ObjThumb{ - Object: model.Object{ - ID: src.ID, - Path: src.Path, - Name: src.Name, - Size: src.Size, - Modified: src.UpdatedAt, - Ctime: src.CreatedAt, - IsFolder: src.Type == 1, - }, + Object: *fileToObject(&src), Thumbnail: thumb, }, nil }) @@ -151,14 +143,7 @@ func (d *CloudreveV4) Get(ctx context.Context, path string) (model.Obj, error) { if err != nil { return nil, err } - return &model.Object{ - ID: info.ID, - Path: info.Path, - Name: info.Name, - Size: info.Size, - Modified: info.UpdatedAt, - Ctime: info.CreatedAt, - }, nil + return fileToObject(&info), nil } func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { diff --git a/drivers/cloudreve_v4/types.go b/drivers/cloudreve_v4/types.go index 23335042f..b67cfc862 100644 --- a/drivers/cloudreve_v4/types.go +++ b/drivers/cloudreve_v4/types.go @@ -122,6 +122,18 @@ type File struct { PrimaryEntity string `json:"primary_entity"` } +func fileToObject(f *File) *model.Object { + return &model.Object{ + ID: f.ID, + Path: f.Path, + Name: f.Name, + Size: f.Size, + Modified: f.UpdatedAt, + Ctime: f.CreatedAt, + IsFolder: f.Type == 1, + } +} + type StoragePolicy struct { ID string `json:"id"` Name string `json:"name"` diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go index 853df9ad6..f8fe5f269 100644 --- a/drivers/cloudreve_v4/util.go +++ b/drivers/cloudreve_v4/util.go @@ -16,6 +16,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -30,6 +31,7 @@ import ( const ( CodeLoginRequired = http.StatusUnauthorized + CodePathNotExist = 40016 // Path not exist CodeCredentialInvalid = 40020 // Failed to issue token ) @@ -101,6 +103,9 @@ func (d *CloudreveV4) _request(method string, path string, callback base.ReqCall if r.Code == CodeCredentialInvalid { return ErrorIssueToken } + if r.Code == CodePathNotExist { + return errs.ObjectNotFound + } return fmt.Errorf("%d: %s", r.Code, r.Msg) } From 27732ccc88363b71faf837e68e0fc2f87feb792e Mon Sep 17 00:00:00 2001 From: mkitsdts <136291922+mkitsdts@users.noreply.github.com> Date: Thu, 29 Jan 2026 21:30:17 +0800 Subject: [PATCH 03/86] fix(drivers/quark): apply html escaping in quark (#2046) * fix(drivers/quark): apply html escaping in quark --- drivers/quark_uc/util.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index 48aabc48f..87798c6ef 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "fmt" + "html" "io" "net/http" "strconv" @@ -70,10 +71,10 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { page := 1 size := 100 query := map[string]string{ - "pdir_fid": parent, - "_size": strconv.Itoa(size), - "_fetch_total": "1", - "fetch_all_file": "1", + "pdir_fid": parent, + "_size": strconv.Itoa(size), + "_fetch_total": "1", + "fetch_all_file": "1", "fetch_risk_file_name": "1", } if d.OrderBy != "none" { @@ -89,6 +90,7 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { return nil, err } for _, file := range resp.Data.List { + file.FileName = html.UnescapeString(file.FileName) if d.OnlyListVideoFile { // 开启后 只列出视频文件和文件夹 if file.IsDir() || file.Category == 1 { From d685bbfa9adc3037dc31813615ae9b3fe6d46993 Mon Sep 17 00:00:00 2001 From: Hu Yuantao <130338111+datao2001@users.noreply.github.com> Date: Thu, 29 Jan 2026 21:48:16 +0800 Subject: [PATCH 04/86] fix(api/remove): add validation for empty items in delete file list (#1617) * fix(FsRemove): add validation for empty items in delete file list If Req.Names contains an empty string item, the whole directory will be removed. As a result we need add a simple guard to prevent such cases. Signed-off-by: huyuantao * fix(FsRemove): enhance validation to prevent unintended directory deletion 1. Use `utils.FixAndCleanPath` to correctly identify and block invalid names. 2. Change error handling from `return` to `continue`. Signed-off-by: huyuantao --------- Signed-off-by: huyuantao Co-authored-by: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> --- server/handles/fsmanage.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 8247fa8cb..2a1c5e5a7 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -282,6 +282,11 @@ func FsRemove(c *gin.Context) { return } for _, name := range req.Names { + // Skip invalid item names (empty string, whitespace, ".", "/","\t\t","..") to prevent accidental removal of current directory + if strings.TrimSpace(utils.FixAndCleanPath(name)) == "/" { + utils.Log.Warnf("FsRemove: invalid item skipped: %s (parent directory: %s)\n", name, reqDir) + continue + } err := fs.Remove(c.Request.Context(), stdpath.Join(reqDir, name)) if err != nil { common.ErrorResp(c, err, 500) From 7b78fed106382430c69ef351d43f5d09928fff14 Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Sat, 31 Jan 2026 16:50:32 +0800 Subject: [PATCH 05/86] Merge commit from fork Co-authored-by: KirCute <951206789@qq.com> --- server/handles/archive.go | 6 +- server/handles/fsmanage.go | 109 ++++++++++++++++++++++--------------- 2 files changed, 67 insertions(+), 48 deletions(-) diff --git a/server/handles/archive.go b/server/handles/archive.go index 56418de26..4fd405688 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -231,7 +231,7 @@ func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { type ArchiveDecompressReq struct { SrcDir string `json:"src_dir" form:"src_dir"` DstDir string `json:"dst_dir" form:"dst_dir"` - Name []string `json:"name" form:"name"` + Names []string `json:"name" form:"name"` ArchivePass string `json:"archive_pass" form:"archive_pass"` InnerPath string `json:"inner_path" form:"inner_path"` CacheFull bool `json:"cache_full" form:"cache_full"` @@ -250,8 +250,8 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcPaths := make([]string, 0, len(req.Name)) - for _, name := range req.Name { + srcPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) if err != nil { common.ErrorResp(c, err, 403) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 2a1c5e5a7..62382a27c 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -6,18 +6,18 @@ import ( "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/task" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/generic" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type MkdirOrLinkReq struct { @@ -80,36 +80,44 @@ func FsMove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcDir, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) - return - } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - var validNames []string - if !req.Overwrite { - for _, name := range req.Names { - if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil && !req.SkipExisting { - common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + validPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + // ensure req.Names is not a relative path + srcPath := stdpath.Join(req.SrcDir, name) + srcPath, err = user.JoinPath(srcPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !req.Overwrite { + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) return - } else if res == nil { - validNames = append(validNames, name) + } + if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { + if !req.SkipExisting { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + return + } else { + continue + } } } - } else { - validNames = req.Names + validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo - for i, name := range validNames { - t, err := fs.Move(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(validNames) > i+1) + for i, p := range validPaths { + t, err := fs.Move(c.Request.Context(), p, dstDir, len(validPaths) > i+1) if t != nil { addedTasks = append(addedTasks, t) } @@ -147,44 +155,48 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcDir, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) - return - } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - var validNames []string - if !req.Overwrite { - for _, name := range req.Names { - if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + validPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + // ensure req.Names is not a relative path + srcPath := stdpath.Join(req.SrcDir, name) + srcPath, err = user.JoinPath(srcPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !req.Overwrite { + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) + return + } + if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { if !req.SkipExisting && !req.Merge { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return - } else if req.Merge && res.IsDir() { - validNames = append(validNames, name) + } else if !req.Merge || !res.IsDir() { + continue } - } else { - validNames = append(validNames, name) } } - } else { - validNames = req.Names + validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo - for i, name := range validNames { + for i, p := range validPaths { var t task.TaskExtensionInfo if req.Merge { - t, err = fs.Merge(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(validNames) > i+1) + t, err = fs.Merge(c.Request.Context(), p, dstDir, len(validPaths) > i+1) } else { - t, err = fs.Copy(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(validNames) > i+1) + t, err = fs.Copy(c.Request.Context(), p, dstDir, len(validPaths) > i+1) } if t != nil { addedTasks = append(addedTasks, t) @@ -276,18 +288,25 @@ func FsRemove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - reqDir, err := user.JoinPath(req.Dir) - if err != nil { - common.ErrorResp(c, err, 403) - return - } - for _, name := range req.Names { - // Skip invalid item names (empty string, whitespace, ".", "/","\t\t","..") to prevent accidental removal of current directory + for i, name := range req.Names { if strings.TrimSpace(utils.FixAndCleanPath(name)) == "/" { - utils.Log.Warnf("FsRemove: invalid item skipped: %s (parent directory: %s)\n", name, reqDir) + log.Warnf("FsRemove: invalid item skipped: %s (parent directory: %s)\n", name, req.Dir) + req.Names[i] = "" + continue + } + // ensure req.Names is not a relative path + var err error + req.Names[i], err = user.JoinPath(stdpath.Join(req.Dir, name)) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + } + for _, path := range req.Names { + if path == "" { continue } - err := fs.Remove(c.Request.Context(), stdpath.Join(reqDir, name)) + err := fs.Remove(c.Request.Context(), path) if err != nil { common.ErrorResp(c, err, 500) return From e3c664f81d0584fbbdb86ffe6644be16259371c1 Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Sat, 31 Jan 2026 16:52:20 +0800 Subject: [PATCH 06/86] Merge commit from fork Co-authored-by: KirCute <951206789@qq.com> --- internal/bootstrap/config.go | 1 + internal/bootstrap/patch/all.go | 1 + internal/bootstrap/patch/v4_1_9/skip_tls.go | 32 +++++++++++++++++++++ internal/conf/config.go | 2 +- internal/conf/var.go | 5 ++-- 5 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 internal/bootstrap/patch/v4_1_9/skip_tls.go diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 116b4cd35..74e218f8f 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -54,6 +54,7 @@ func InitConfig() { } } configPath = filepath.Clean(configPath) + conf.ConfigPath = configPath log.Infof("reading config file: %s", configPath) if !utils.Exists(configPath) { log.Infof("config file not exists, creating default config file") diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go index c4a72a966..5d4c814dc 100644 --- a/internal/bootstrap/patch/all.go +++ b/internal/bootstrap/patch/all.go @@ -44,6 +44,7 @@ var UpgradePatches = []VersionPatches{ Version: "v4.1.9", Patches: []func(){ v4_1_9.EnableWebDavProxy, + v4_1_9.ResetSkipTlsVerify, }, }, } diff --git a/internal/bootstrap/patch/v4_1_9/skip_tls.go b/internal/bootstrap/patch/v4_1_9/skip_tls.go new file mode 100644 index 000000000..1d9858d95 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_9/skip_tls.go @@ -0,0 +1,32 @@ +package v4_1_9 + +import ( + "os" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +func ResetSkipTlsVerify() { + if !conf.Conf.TlsInsecureSkipVerify { + return + } + if !strings.HasPrefix(conf.Version, "v") { + return + } + + conf.Conf.TlsInsecureSkipVerify = false + + confBody, err := utils.Json.MarshalIndent(conf.Conf, "", " ") + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: marshal config error: %+v", err) + return + } + err = os.WriteFile(conf.ConfigPath, confBody, 0o777) + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: update config struct error: %+v", err) + return + } + utils.Log.Infof("[ResetSkipTlsVerify] succeeded to set tls_insecure_skip_verify to false") +} diff --git a/internal/conf/config.go b/internal/conf/config.go index c5ace8005..f347380d8 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -182,7 +182,7 @@ func DefaultConfig(dataDir string) *Config { MmapThreshold: 4, MaxConnections: 0, MaxConcurrency: 64, - TlsInsecureSkipVerify: true, + TlsInsecureSkipVerify: false, Tasks: TasksConfig{ Download: TaskConfig{ Workers: 5, diff --git a/internal/conf/var.go b/internal/conf/var.go index 9a02eca26..972f69997 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -15,8 +15,9 @@ var ( ) var ( - Conf *Config - URL *url.URL + Conf *Config + URL *url.URL + ConfigPath string ) var SlicesMap = make(map[string][]string) From f5421876548cd5462fd21714bb609141e09b27ba Mon Sep 17 00:00:00 2001 From: Chaloemchai Date: Sun, 1 Feb 2026 19:39:25 +0700 Subject: [PATCH 07/86] fix(drivers/teldrive): enhance file listing and upload functionality with pagination and random chunk naming (#2034) * fix(drivers/teldrive): enhance file listing and upload functionality with pagination and random chunk naming * fix(drivers/teldrive): optimize file listing by removing unnecessary mutex and restructuring data handling * Update drivers/teldrive/meta.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Chaloemchai --------- Signed-off-by: Chaloemchai Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- drivers/teldrive/driver.go | 52 ++++++++++++++++++++++++++--- drivers/teldrive/meta.go | 1 + drivers/teldrive/upload.go | 67 +++++++++++++++++++++++++++----------- 3 files changed, 96 insertions(+), 24 deletions(-) diff --git a/drivers/teldrive/driver.go b/drivers/teldrive/driver.go index 11ba0971e..d420eb4d0 100644 --- a/drivers/teldrive/driver.go +++ b/drivers/teldrive/driver.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "path" + "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -17,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" + "golang.org/x/sync/errgroup" ) type Teldrive struct { @@ -53,18 +55,58 @@ func (d *Teldrive) Drop(ctx context.Context) error { } func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - var listResp ListResp + var firstResp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "path": dir.GetPath(), - "limit": "1000", // overide default 500, TODO pagination + "limit": "500", + "page": "1", }) - }, &listResp) + }, &firstResp) + if err != nil { return nil, err } - return utils.SliceConvert(listResp.Items, func(src Object) (model.Obj, error) { + pagesData := make([][]Object, firstResp.Meta.TotalPages) + pagesData[0] = firstResp.Items + + if firstResp.Meta.TotalPages > 1 { + g, _ := errgroup.WithContext(ctx) + g.SetLimit(8) + + for i := 2; i <= firstResp.Meta.TotalPages; i++ { + page := i + g.Go(func() error { + var resp ListResp + err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "path": dir.GetPath(), + "limit": "500", + "page": strconv.Itoa(page), + }) + }, &resp) + + if err != nil { + return err + } + + pagesData[page-1] = resp.Items + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + } + + var allItems []Object + for _, items := range pagesData { + allItems = append(allItems, items...) + } + + return utils.SliceConvert(allItems, func(src Object) (model.Obj, error) { return &model.Object{ Path: path.Join(dir.GetPath(), src.Name), ID: src.ID, @@ -184,7 +226,7 @@ func (d *Teldrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStr } if totalParts <= 1 { - return d.doSingleUpload(ctx, dstDir, file, up, totalParts, chunkSize, fileId) + return d.doSingleUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) } return d.doMultiUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) diff --git a/drivers/teldrive/meta.go b/drivers/teldrive/meta.go index 23bae5f94..cc7a5dbf7 100644 --- a/drivers/teldrive/meta.go +++ b/drivers/teldrive/meta.go @@ -11,6 +11,7 @@ type Addition struct { Cookie string `json:"cookie" type:"string" required:"true" help:"access_token=xxx"` UseShareLink bool `json:"use_share_link" type:"bool" default:"false" help:"Create share link when getting link to support 302. If disabled, you need to enable web proxy."` ChunkSize int64 `json:"chunk_size" type:"number" default:"10" help:"Chunk size in MiB"` + RandomChunkName bool `json:"random_chunk_name" type:"bool" default:"true" help:"Random chunk name"` UploadConcurrency int64 `json:"upload_concurrency" type:"number" default:"4" help:"Concurrency upload requests"` } diff --git a/drivers/teldrive/upload.go b/drivers/teldrive/upload.go index 87cffa1ae..b94f5fc93 100644 --- a/drivers/teldrive/upload.go +++ b/drivers/teldrive/upload.go @@ -1,6 +1,8 @@ package teldrive import ( + "crypto/md5" + "encoding/hex" "fmt" "io" "net/http" @@ -16,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + "github.com/google/uuid" "github.com/pkg/errors" "golang.org/x/net/context" "golang.org/x/sync/errgroup" @@ -38,6 +41,11 @@ func (d *Teldrive) touch(name, path string) error { return nil } +func getMD5Hash(text string) string { + hash := md5.Sum([]byte(text)) + return hex.EncodeToString(hash[:]) +} + func (d *Teldrive) createFileOnUploadSuccess(name, id, path string, uploadedFileParts []FilePart, totalSize int64) error { remoteFileParts, err := d.getFilePart(id) if err != nil { @@ -101,12 +109,10 @@ func (d *Teldrive) getFilePart(fileId string) ([]FilePart, error) { return uploadedParts, nil } -func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback, resp interface{}) error { +func (d *Teldrive) singleUploadRequest(ctx context.Context, fileId string, callback base.ReqCallback, resp any) error { url := d.Address + "/api/uploads/" + fileId client := resty.New().SetTimeout(0) - ctx := context.Background() - req := client.R(). SetContext(ctx) req.SetHeader("Cookie", d.Cookie) @@ -135,16 +141,18 @@ func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback, } func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up model.UpdateProgress, - totalParts int, chunkSize int64, fileId string) error { + maxRetried, totalParts int, chunkSize int64, fileId string) error { totalSize := file.GetSize() var fileParts []FilePart var uploaded int64 = 0 - ss, err := stream.NewStreamSectionReader(file, int(totalSize), &up) + var partName string + chunkSize = min(totalSize, chunkSize) + ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up) if err != nil { return err } - + chunkCnt := 0 for uploaded < totalSize { if utils.IsCanceled(ctx) { return ctx.Err() @@ -154,6 +162,7 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo if err != nil { return err } + chunkCnt += 1 filePart := &FilePart{} if err := retry.Do(func() error { @@ -161,13 +170,19 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo return err } - if err := d.singleUploadRequest(fileId, func(req *resty.Request) { + if d.RandomChunkName { + partName = getMD5Hash(uuid.New().String()) + } else { + partName = file.GetName() + if totalParts > 1 { + partName = fmt.Sprintf("%s.part.%03d", file.GetName(), chunkCnt) + } + } + + if err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ - "partName": func() string { - digits := len(strconv.Itoa(totalParts)) - return file.GetName() + fmt.Sprintf(".%0*d", digits, 1) - }(), - "partNo": strconv.Itoa(1), + "partName": partName, + "partNo": strconv.Itoa(chunkCnt), "fileName": file.GetName(), } req.SetQueryParams(uploadParams) @@ -180,7 +195,7 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo return nil }, retry.Context(ctx), - retry.Attempts(3), + retry.Attempts(uint(maxRetried)), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)); err != nil { return err @@ -189,8 +204,11 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo if filePart.Name != "" { fileParts = append(fileParts, *filePart) uploaded += curChunkSize - up(float64(uploaded) / float64(totalSize)) + up(float64(uploaded) / float64(totalSize) * 100) ss.FreeSectionReader(rd) + } else { + // For common situation this code won't reach + return fmt.Errorf("[Teldrive] upload chunk %d failed: filePart Somehow missing", chunkCnt) } } @@ -318,6 +336,7 @@ func (d *Teldrive) doMultiUpload(ctx context.Context, dstDir model.Obj, file mod func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task chunkTask, totalParts, maxRetried int) (*FilePart, error) { filePart := &FilePart{} retryCount := 0 + var partName string defer task.ss.FreeSectionReader(task.reader) for { @@ -331,12 +350,22 @@ func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task ch return &existingPart, nil } - err := d.singleUploadRequest(fileId, func(req *resty.Request) { + if _, err := task.reader.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + if d.RandomChunkName { + partName = getMD5Hash(uuid.New().String()) + } else { + partName = task.fileName + if totalParts > 1 { + partName = fmt.Sprintf("%s.part.%03d", task.fileName, task.chunkIdx) + } + } + + err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ - "partName": func() string { - digits := len(strconv.Itoa(totalParts)) - return task.fileName + fmt.Sprintf(".%0*d", digits, task.chunkIdx) - }(), + "partName": partName, "partNo": strconv.Itoa(task.chunkIdx), "fileName": task.fileName, } From 5d9fc8359d537b0840b7a4c2e78f7b2bd2d16fa5 Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Mon, 2 Feb 2026 18:43:26 +0800 Subject: [PATCH 08/86] fix(115_share): adjust 115 share driver for official API update (#2068) * fix(115_share): add user agent support and update driver dependency * fix(115): fix download error * feat: add thumbnail support for 115 driver and 115 share - Add Thumb() method to FileObj in 115 driver to return thumbnail URL - Add ThumbURL field to FileObj struct in 115 share utility - Update 115driver dependency from v1.2.2 to v1.2.3 to support thumbnail functionality - Implement Thumb() method for 115 share FileObj to return thumbnail URL --- drivers/115/driver.go | 3 +- drivers/115/types.go | 4 +++ drivers/115/util.go | 62 ++----------------------------------- drivers/115_share/driver.go | 21 +++++++++++-- drivers/115_share/utils.go | 11 +++++-- go.mod | 2 +- go.sum | 4 +-- 7 files changed, 37 insertions(+), 70 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 162d835d0..d4f5741d0 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -68,8 +68,7 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, err } userAgent := args.Header.Get("User-Agent") - downloadInfo, err := d. - DownloadWithUA(file.(*FileObj).PickCode, userAgent) + downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } diff --git a/drivers/115/types.go b/drivers/115/types.go index 28a8ced30..3477ffed0 100644 --- a/drivers/115/types.go +++ b/drivers/115/types.go @@ -22,6 +22,10 @@ func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + type UploadResult struct { driver.BasicResp Data struct { diff --git a/drivers/115/util.go b/drivers/115/util.go index b000436b2..7ae375b75 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "net/url" "strconv" "strings" @@ -22,11 +21,9 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/aliyun/aliyun-oss-go-sdk/oss" - cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" - crypto "github.com/SheltonZhu/115driver/pkg/crypto/m115" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/pkg/errors" ) @@ -108,60 +105,6 @@ func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } -func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { - key := crypto.GenerateKey() - result := driver115.DownloadResp{} - params, err := utils.Json.Marshal(map[string]string{"pick_code": pickCode}) - if err != nil { - return nil, err - } - - data := crypto.Encode(params, key) - - bodyReader := strings.NewReader(url.Values{"data": []string{data}}.Encode()) - reqUrl := fmt.Sprintf("%s?t=%s", driver115.AndroidApiDownloadGetUrl, driver115.Now().String()) - req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Cookie", d.Cookie) - req.Header.Set("User-Agent", ua) - - resp, err := d.client.Client.GetClient().Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if err := utils.Json.Unmarshal(body, &result); err != nil { - return nil, err - } - - if err = result.Err(string(body)); err != nil { - return nil, err - } - - b, err := crypto.Decode(string(result.EncodedData), key) - if err != nil { - return nil, err - } - - downloadInfo := struct { - Url string `json:"url"` - }{} - if err := utils.Json.Unmarshal(b, &downloadInfo); err != nil { - return nil, err - } - - info := &driver115.DownloadInfo{} - info.PickCode = pickCode - info.Header = resp.Request.Header - info.Url.Url = downloadInfo.Url - return info, nil -} - func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { userID := strconv.FormatInt(c.client.UserID, 10) userIDMd5 := md5.Sum([]byte(userID)) @@ -309,7 +252,8 @@ func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSPar // UploadByMultipart upload by mutipart blocks func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.UploadOSSParams, fileSize int64, s model.FileStreamer, - dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption) (*UploadResult, error) { + dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption, +) (*UploadResult, error) { var ( chunks []oss.FileChunk parts []oss.UploadPart diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index 00fa623e6..fe8b7733a 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -3,6 +3,7 @@ package _115_share import ( "context" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -49,9 +50,16 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr if err := d.WaitLimit(ctx); err != nil { return nil, err } - + var ua string + // TODO: will use user agent from header + // if args.Header != nil { + // ua = args.Header.Get("User-Agent") + // } + if ua == "" { + ua = base.UserAgentNT + } files := make([]driver115.ShareFile, 0) - fileResp, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) + fileResp, err := d.client.GetShareSnapWithUA(ua, d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) if err != nil { return nil, err } @@ -77,7 +85,14 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if err := d.WaitLimit(ctx); err != nil { return nil, err } - downloadInfo, err := d.client.DownloadByShareCode(d.ShareCode, d.ReceiveCode, file.GetID()) + var ua string + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + downloadInfo, err := d.client.DownloadByShareCodeWithUA(ua, d.ShareCode, d.ReceiveCode, file.GetID()) if err != nil { return nil, err } diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go index 082d9d462..f9575d493 100644 --- a/drivers/115_share/utils.go +++ b/drivers/115_share/utils.go @@ -5,6 +5,7 @@ import ( "strconv" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" driver115 "github.com/SheltonZhu/115driver/pkg/driver" @@ -20,6 +21,7 @@ type FileObj struct { FileName string isDir bool FileID string + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -54,6 +56,10 @@ func (f *FileObj) GetPath() string { return "" } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + func transFunc(sf driver115.ShareFile) (model.Obj, error) { timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) if err != nil { @@ -74,15 +80,14 @@ func transFunc(sf driver115.ShareFile) (model.Obj, error) { FileName: string(sf.FileName), isDir: isDir, FileID: fileID, + ThumbURL: sf.ThumbURL, }, nil } -var UserAgent = driver115.UA115Browser - func (d *Pan115Share) login() error { var err error opts := []driver115.Option{ - driver115.UA(UserAgent), + driver115.UA(base.UserAgentNT), } d.client = driver115.New(opts...) if _, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, ""); err != nil { diff --git a/go.mod b/go.mod index e751bcfe0..ce2e753bf 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/OpenListTeam/wopan-sdk-go v0.1.5 github.com/ProtonMail/go-crypto v1.3.0 github.com/ProtonMail/gopenpgp/v2 v2.9.0 - github.com/SheltonZhu/115driver v1.1.1 + github.com/SheltonZhu/115driver v1.2.3 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/antchfx/htmlquery v1.3.5 github.com/antchfx/xpath v1.3.5 diff --git a/go.sum b/go.sum index 57964be4c..ef43501f1 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,8 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2 github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.1.1 h1:9EMhe2ZJflGiAaZbYInw2jqxTcqZNF+DtVDsEy70aFU= -github.com/SheltonZhu/115driver v1.1.1/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= +github.com/SheltonZhu/115driver v1.2.3 h1:94XMP/ey7VXIlpoBLIJHEoXu7N8YsELZlXVbxWcDDvk= +github.com/SheltonZhu/115driver v1.2.3/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= From 6861cb4c3b6f283facd77b4ebed90d12f5429e60 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Fri, 6 Feb 2026 21:52:06 +0800 Subject: [PATCH 09/86] chore(ci): add breaking change guideline to PR title check (#2087) * feat(ci): add PR title validation for breaking changes Updated regex to allow '!' for breaking changes in PR titles. Signed-off-by: MadDogOwner * chore(pr): Update PR template Add bilingual instructions for PR title formatting. Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner --- .github/PULL_REQUEST_TEMPLATE.md | 2 ++ .github/workflows/issue_pr_comment.yml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e74e59631..f1687eabf 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,11 +2,13 @@ Provide a general summary of your changes in the Title above. The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`. If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body. + For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`. --> ## Description / 描述 diff --git a/.github/workflows/issue_pr_comment.yml b/.github/workflows/issue_pr_comment.yml index c618485f0..f15acd424 100644 --- a/.github/workflows/issue_pr_comment.yml +++ b/.github/workflows/issue_pr_comment.yml @@ -47,12 +47,14 @@ jobs: with: script: | const title = context.payload.pull_request.title || ""; - const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\): /i.test(title); + const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\)!?: /i.test(title); if (!ok) { let comment = "⚠️ PR 标题需以 `feat(): `, `docs(): `, `fix(): `, `style(): `, `refactor(): `, `chore(): ` 其中之一开头，例如：`feat(component): 新增功能`。\n"; comment += "⚠️ The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`.\n\n"; comment += "如果跨多个组件，请使用主要组件作为前缀，并在标题中枚举、描述中说明。\n"; comment += "If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body.\n\n"; + comment += "如果是破坏性变更，请在类型后添加 `!`，例如 `feat(component)!: 破坏性变更`。\n"; + comment += "For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`.\n\n"; await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, From a121f861dcec9b7ef2fb4808e48456f50a567bab Mon Sep 17 00:00:00 2001 From: gdm257 Date: Sun, 8 Feb 2026 14:01:26 +0800 Subject: [PATCH 10/86] feat(drivers/123open): support sha1 reuse api (#2089) * feat(drivers/123open): support sha1 reuse api * fix(drivers/123open): fix typos --- drivers/123_open/driver.go | 16 ++++++++++++++++ drivers/123_open/types.go | 12 ++++++++++++ drivers/123_open/upload.go | 18 ++++++++++++++++++ drivers/123_open/util.go | 21 +++++++++++---------- 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 9608cedf9..e20140277 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -181,6 +181,22 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return nil, fmt.Errorf("parse parentFileID error: %v", err) } + + // 尝试 SHA1 秒传 + sha1Hash := file.GetHash().GetHash(utils.SHA1) + if len(sha1Hash) == utils.SHA1.Width { + resp, err := d.sha1Reuse(parentFileId, file.GetName(), sha1Hash, file.GetSize(), 2) + if err == nil && resp.Data.Reuse { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: resp.Data.FileID, + Type: 2, + SHA1: sha1Hash, + }, nil + } + } + // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) if len(etag) < utils.MD5.Width { diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go index 8745ff795..7d586c8b0 100644 --- a/drivers/123_open/types.go +++ b/drivers/123_open/types.go @@ -58,9 +58,13 @@ type File struct { Category int `json:"category"` Status int `json:"status"` Trashed int `json:"trashed"` + SHA1 string } func (f File) GetHash() utils.HashInfo { + if len(f.SHA1) == utils.SHA1.Width && len(f.Etag) != utils.MD5.Width { + return utils.NewHashInfo(utils.SHA1, f.SHA1) + } return utils.NewHashInfo(utils.MD5, f.Etag) } @@ -190,6 +194,14 @@ type UploadCompleteResp struct { } `json:"data"` } +type SHA1ReuseResp struct { + BaseResp + Data struct { + FileID int64 `json:"fileID"` + Reuse bool `json:"reuse"` + } `json:"data"` +} + type OfflineDownloadResp struct { BaseResp Data struct { diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go index 90cff90d7..0e03684e9 100644 --- a/drivers/123_open/upload.go +++ b/drivers/123_open/upload.go @@ -183,3 +183,21 @@ func (d *Open123) complete(preuploadID string) (*UploadCompleteResp, error) { } return &resp, nil } + +// SHA1 秒传 +func (d *Open123) sha1Reuse(parentFileID int64, filename string, sha1Hash string, size int64, duplicate int) (*SHA1ReuseResp, error) { + var resp SHA1ReuseResp + _, err := d.Request(UploadSHA1Reuse, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "parentFileID": parentFileID, + "filename": filename, + "sha1": strings.ToLower(sha1Hash), + "size": size, + "duplicate": duplicate, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} diff --git a/drivers/123_open/util.go b/drivers/123_open/util.go index 5d961d5c2..1b6eea2da 100644 --- a/drivers/123_open/util.go +++ b/drivers/123_open/util.go @@ -21,16 +21,17 @@ import ( var ( // 不同情况下获取的AccessTokenQPS限制不同 如下模块化易于拓展 Api = "https://open-api.123pan.com" - UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) - FileList = InitApiInfo(Api+"/api/v2/file/list", 3) - DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 5) - DirectLink = InitApiInfo(Api+"/api/v1/direct-link/url", 5) - Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) - Move = InitApiInfo(Api+"/api/v1/file/move", 1) - Rename = InitApiInfo(Api+"/api/v1/file/name", 1) - Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) - UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) - UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) + UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) + FileList = InitApiInfo(Api+"/api/v2/file/list", 3) + DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 5) + DirectLink = InitApiInfo(Api+"/api/v1/direct-link/url", 5) + Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) + Move = InitApiInfo(Api+"/api/v1/file/move", 1) + Rename = InitApiInfo(Api+"/api/v1/file/name", 1) + Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) + UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) + UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) + UploadSHA1Reuse = InitApiInfo(Api+"/upload/v2/file/sha1_reuse", 2) OfflineDownload = InitApiInfo(Api+"/api/v1/offline/download", 1) OfflineDownloadProcess = InitApiInfo(Api+"/api/v1/offline/download/process", 5) From 8431c1b1e3166f1804b194eb4e600d023a238514 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 8 Feb 2026 16:26:54 +0800 Subject: [PATCH 11/86] fix(deps): update go4.org digest to a507140 (#2095) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 34 ++++++++++++++++++---------------- go.sum | 29 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index ce2e753bf..18ff90740 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/OpenListTeam/OpenList/v4 -go 1.23.4 +go 1.24.0 + +toolchain go1.24.13 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 @@ -73,11 +75,11 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.6.0 github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 - golang.org/x/crypto v0.40.0 + golang.org/x/crypto v0.46.0 golang.org/x/image v0.29.0 - golang.org/x/net v0.42.0 - golang.org/x/oauth2 v0.30.0 - golang.org/x/time v0.12.0 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 + golang.org/x/time v0.14.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 @@ -87,7 +89,7 @@ require ( ) require ( - cloud.google.com/go/compute/metadata v0.7.0 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect @@ -124,7 +126,7 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.uber.org/mock v0.5.0 // indirect golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect - golang.org/x/mod v0.27.0 // indirect + golang.org/x/mod v0.30.0 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect ) @@ -159,7 +161,7 @@ require ( github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/yuin/goldmark v1.7.13 - go4.org v0.0.0-20230225012048-214862532bf5 + go4.org v0.0.0-20260112195520-a5071408f32f resty.dev/v3 v3.0.0-beta.2 // indirect ) @@ -285,14 +287,14 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.4.0 // indirect golang.org/x/arch v0.18.0 // indirect - golang.org/x/sync v0.16.0 - golang.org/x/sys v0.34.0 - golang.org/x/term v0.33.0 // indirect - golang.org/x/text v0.27.0 - golang.org/x/tools v0.35.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.40.0 + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 + golang.org/x/tools v0.39.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index ef43501f1..ae1dabfa4 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbf cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -752,6 +753,8 @@ go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= +go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= +go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= @@ -770,6 +773,8 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -807,6 +812,8 @@ golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -834,6 +841,8 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -841,6 +850,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -855,6 +866,8 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -889,6 +902,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -901,6 +916,8 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -917,10 +934,14 @@ golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -953,6 +974,8 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -988,6 +1011,8 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -997,10 +1022,14 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From a8d1c0ddbfc716e307c902f8c8dcffb72b39e1d0 Mon Sep 17 00:00:00 2001 From: jenfonro <799170122@qq.com> Date: Sun, 8 Feb 2026 21:15:00 +0800 Subject: [PATCH 12/86] =?UTF-8?q?fix(=E2=80=8Edrivers/quark=5Fuc=5Ftv)=20:?= =?UTF-8?q?=20Update=20error=20code=20judgment=20(#2080)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix error code * add ErrorInfo check --- drivers/quark_uc_tv/util.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/quark_uc_tv/util.go b/drivers/quark_uc_tv/util.go index d68a2f3c3..c0da6eb6b 100644 --- a/drivers/quark_uc_tv/util.go +++ b/drivers/quark_uc_tv/util.go @@ -8,6 +8,7 @@ import ( "errors" "net/http" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -70,8 +71,16 @@ func (d *QuarkUCTV) request(ctx context.Context, pathname string, method string, return nil, err } // 判断 是否需要 刷新 access_token - if e.Status == -1 && e.Errno == 10001 { - // token 过期 + errInfoLower := strings.ToLower(strings.TrimSpace(e.ErrorInfo)) + maybeTokenInvalid := + (e.Status == -1 && (e.Errno == 10001 || e.Errno == 11001)) || + (errInfoLower != "" && + (strings.Contains(errInfoLower, "access token") || + strings.Contains(errInfoLower, "access_token") || + strings.Contains(errInfoLower, "token无效") || + strings.Contains(errInfoLower, "token 无效"))) + if maybeTokenInvalid { + // token 过期 / 无效 err = d.getRefreshTokenByTV(ctx, d.Addition.RefreshToken, true) if err != nil { return nil, err From 0673a7430223da2ad284250eb2d1b79109b4411c Mon Sep 17 00:00:00 2001 From: LXY <767763591@qq.com> Date: Mon, 16 Feb 2026 22:57:43 +0800 Subject: [PATCH 13/86] chore(typo): fix typo in UpdateFileReq field name (#2133) Fix typo in UpdateFileReq field name ref: https://github.com/OpenListTeam/115-sdk-go/pull/3 Signed-off-by: LXY <767763591@qq.com> --- drivers/115_open/driver.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index c1a855749..ec76a6bc8 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -176,7 +176,7 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ FileID: srcObj.GetID(), - FileNma: newName, + FileName: newName, }) if err != nil { return nil, err diff --git a/go.mod b/go.mod index 18ff90740..c36ac1ca0 100644 --- a/go.mod +++ b/go.mod @@ -131,7 +131,7 @@ require ( ) require ( - github.com/OpenListTeam/115-sdk-go v0.2.2 + github.com/OpenListTeam/115-sdk-go v0.2.3 github.com/STARRY-S/zip v0.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/blevesearch/go-faiss v1.0.25 // indirect diff --git a/go.sum b/go.sum index ae1dabfa4..b9a4570bd 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,8 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/OpenListTeam/115-sdk-go v0.2.2 h1:JCrGHqQjBX3laOA6Hw4CuBovSg7g+FC5s0LEAYsRciU= -github.com/OpenListTeam/115-sdk-go v0.2.2/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/OpenListTeam/115-sdk-go v0.2.3 h1:nDNz0GxgliW+nT2Ds486k/rp/GgJj7Ngznc98ZBUwZo= +github.com/OpenListTeam/115-sdk-go v0.2.3/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= From e0ee73708082b784c0c3f6d84804eef7f0a6f606 Mon Sep 17 00:00:00 2001 From: mkitsdts <136291922+mkitsdts@users.noreply.github.com> Date: Thu, 19 Feb 2026 14:48:43 +0800 Subject: [PATCH 14/86] fix(driver/wps): fetch all files via multiple API invocations (#2139) fix(driver/wps): wps list all files in one request --- drivers/wps/types.go | 3 ++- drivers/wps/util.go | 34 ++++++++++++++++++++++------------ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/drivers/wps/types.go b/drivers/wps/types.go index 7ff6119dd..a04df3d11 100644 --- a/drivers/wps/types.go +++ b/drivers/wps/types.go @@ -41,7 +41,8 @@ type FileInfo struct { } type filesResp struct { - Files []FileInfo `json:"files"` + Files []FileInfo `json:"files"` + NextOffset int `json:"next_offset"` } type downloadResp struct { diff --git a/drivers/wps/util.go b/drivers/wps/util.go index 79be1ac4b..6f8f342da 100644 --- a/drivers/wps/util.go +++ b/drivers/wps/util.go @@ -274,19 +274,29 @@ func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { func (d *Wps) getFiles(ctx context.Context, groupID, parentID int64) ([]FileInfo, error) { var resp filesResp - url := fmt.Sprintf("%s/api/v5/groups/%d/files", d.driveHost()+d.drivePrefix(), groupID) - r, err := d.request(ctx). - SetQueryParam("parentid", strconv.FormatInt(parentID, 10)). - SetResult(&resp). - SetError(&resp). - Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) + var files []FileInfo + next_offset := 0 + for range 50 { + url := fmt.Sprintf("%s/api/v5/groups/%d/files", d.driveHost()+d.drivePrefix(), groupID) + r, err := d.request(ctx). + SetQueryParam("parentid", strconv.FormatInt(parentID, 10)). + SetQueryParam("offset", fmt.Sprint(next_offset)). + SetResult(&resp). + SetError(&resp). + Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + files = append(files, resp.Files...) + if resp.NextOffset == -1 { + break + } + next_offset = resp.NextOffset } - return resp.Files, nil + return files, nil } func parseTime(v int64) time.Time { From db0e2ec1038d2ef51a5a9dafa7c2b20b59f36cc8 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 19 Feb 2026 17:14:55 +0800 Subject: [PATCH 15/86] feat(security): add SECURITY.md (#2147) [skip ci] Add SECURITY.md Signed-off-by: MadDogOwner --- SECURITY.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..ff558d64e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,89 @@ +# Security Policy + +## Supported Versions + +Only the latest stable release receives security patches. We strongly recommend always keeping OpenList up to date. + +| Version | Supported | +| -------------------- | ------------------ | +| Latest stable (v4.x) | :white_check_mark: | +| Older versions | :x: | + +## Reporting a Vulnerability + +**Please do NOT report security vulnerabilities through public GitHub Issues.** + +If you discover a security vulnerability in OpenList, please report it responsibly by using one of the following channels: + +- **GitHub Private Security Advisory** (preferred): [Submit here](https://github.com/OpenListTeam/OpenList/security/advisories/new) +- **Telegram**: Contact a maintainer privately via [@OpenListTeam](https://t.me/OpenListTeam) + +When reporting, please include as much of the following as possible: + +- A description of the vulnerability and its potential impact +- The affected version(s) +- Step-by-step instructions to reproduce the issue +- Any proof-of-concept code or screenshots (if applicable) +- Suggested mitigation or fix (optional but appreciated) + +## Security Best Practices for Users + +To keep your OpenList instance secure: + +- Always update to the latest release. +- Use a strong, unique admin password and change it after first login. +- Enable HTTPS (TLS) for your deployment — do **not** expose OpenList over plain HTTP on the public internet. +- Limit exposed ports using a reverse proxy (e.g., Nginx, Caddy). +- Set up access controls and avoid enabling guest access unless necessary. +- Regularly review mounted storage permissions and revoke unused API tokens. +- When using Docker, avoid running the container as root if possible. + +## Acknowledgments + +We sincerely thank all security researchers and community members who responsibly disclose vulnerabilities and help make OpenList safer for everyone. + +--- + +# 安全政策 + +## 支持的版本 + +我们仅对最新稳定版本提供安全补丁。强烈建议始终保持 OpenList 为最新版本。 + +| 版本 | 是否支持 | +| ------------------ | ------------------ | +| 最新稳定版（v4.x） | :white_check_mark: | +| 旧版本 | :x: | + +## 报告漏洞 + +**请勿通过公开的 GitHub Issues 报告安全漏洞。** + +如果您在 OpenList 中发现安全漏洞，请通过以下渠道之一负责任地进行报告： + +- **GitHub 私密安全公告**（推荐）：[点击提交](https://github.com/OpenListTeam/OpenList/security/advisories/new) +- **Telegram**：通过 [@OpenListTeam](https://t.me/OpenListTeam) 私信联系维护者 + +报告时，请尽量提供以下信息： + +- 漏洞描述及其潜在影响 +- 受影响的版本 +- 复现问题的详细步骤 +- 概念验证代码或截图（如有） +- 建议的缓解措施或修复方案（可选，但非常欢迎） + +## 用户安全最佳实践 + +为保障您的 OpenList 实例安全： + +- 始终更新至最新版本。 +- 使用强且唯一的管理员密码，并在首次登录后立即修改。 +- 为您的部署启用 HTTPS（TLS）—— **请勿**在公网上以明文 HTTP 方式暴露 OpenList。 +- 使用反向代理（如 Nginx、Caddy）限制对外暴露的端口。 +- 配置访问控制，非必要情况下不要开启访客访问。 +- 定期检查已挂载存储的权限，并撤销未使用的 API 令牌。 +- 使用 Docker 部署时，尽可能避免以 root 用户运行容器。 + +## 致谢 + +我们衷心感谢所有负责任地披露漏洞、帮助 OpenList 变得更加安全的安全研究人员和社区成员。 From 82ae2d5890ccaf281a103398bf4fd3fab9ba9cf4 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 19 Feb 2026 17:15:40 +0800 Subject: [PATCH 16/86] chore(handles/auth): improve error response (#2148) * chore(handles/auth): improve error response Signed-off-by: MadDogOwner * Apply suggestion from @xrgzs Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner --- internal/model/user.go | 10 +++++++++- server/handles/auth.go | 19 ++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/model/user.go b/internal/model/user.go index 640e3b2e3..3bad4ebb9 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -20,7 +20,15 @@ const ( ADMIN ) -const StaticHashSalt = "https://github.com/alist-org/alist" +const ( + StaticHashSalt = "https://github.com/alist-org/alist" + + InvalidUsernameOrPassword = "Invalid username or password" + Invalid2FACode = "Invalid 2FA code" + TooManyAttempts = "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later." + GuestCannotUpdateProfile = "Guest user can not update profile" + GuestCannotGenerate2FA = "Guest user can not generate 2FA code" +) var LoginCache = cache.NewMemCache[int]() diff --git a/server/handles/auth.go b/server/handles/auth.go index 35776ba6a..780069091 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -45,27 +45,28 @@ func loginHash(c *gin.Context, req *LoginReq) { ip := c.ClientIP() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { - common.ErrorStrResp(c, "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.", 429) + common.ErrorStrResp(c, model.TooManyAttempts, 429) model.LoginCache.Expire(ip, model.DefaultLockDuration) return } // check username user, err := op.GetUserByName(req.Username) if err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // check 2FA if user.OtpSecret != "" { if !totp.Validate(req.OtpCode, user.OtpSecret) { - common.ErrorStrResp(c, "Invalid 2FA code", 402) + // 402 - need opt + common.ErrorStrResp(c, model.Invalid2FACode, 402) model.LoginCache.Set(ip, count+1) return } @@ -73,7 +74,7 @@ func loginHash(c *gin.Context, req *LoginReq) { // generate token token, err := common.GenerateToken(user) if err != nil { - common.ErrorResp(c, err, 400, true) + common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, gin.H{"token": token}) @@ -107,7 +108,7 @@ func UpdateCurrent(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not update profile", 403) + common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } user.Username = req.Username @@ -125,7 +126,7 @@ func UpdateCurrent(c *gin.Context) { func Generate2FA(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } key, err := totp.Generate(totp.GenerateOpts{ @@ -164,11 +165,11 @@ func Verify2FA(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } if !totp.Validate(req.Code, req.Secret) { - common.ErrorStrResp(c, "Invalid 2FA code", 400) + common.ErrorStrResp(c, model.Invalid2FACode, 400) return } user.OtpSecret = req.Secret From 795a18b56533dcfa0dfd3cdf0cc67acabf0b7589 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sat, 21 Feb 2026 15:21:05 +0800 Subject: [PATCH 17/86] feat(drivers/thunder*): implement GetDetails (#2113) Signed-off-by: MadDogOwner --- drivers/thunder/driver.go | 26 ++++++++++++++++++++++++++ drivers/thunder/types.go | 18 ++++++++++++++++++ drivers/thunder_browser/driver.go | 26 ++++++++++++++++++++++++++ drivers/thunder_browser/types.go | 7 +++++++ drivers/thunderx/driver.go | 26 ++++++++++++++++++++++++++ drivers/thunderx/types.go | 7 +++++++ 6 files changed, 110 insertions(+) diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index cb352afed..492b9814f 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -433,6 +433,32 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi return nil } +func (xc *XunLeiCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string diff --git a/drivers/thunder/types.go b/drivers/thunder/types.go index 7b3ad5692..fcfa1fb1c 100644 --- a/drivers/thunder/types.go +++ b/drivers/thunder/types.go @@ -347,3 +347,21 @@ type ReviewData struct { Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } + +type AboutResponse struct { + // Kind string `json:"kind"` + Quota struct { + // Kind string `json:"kind"` + Limit string `json:"limit"` + Usage string `json:"usage"` + // UsageInTrash string `json:"usage_in_trash"` + // PlayTimesLimit string `json:"play_times_limit"` + // PlayTimesUsage string `json:"play_times_usage"` + // IsUnlimited bool `json:"is_unlimited"` + // UpgradeType string `json:"upgrade_type"` + } `json:"quota"` + // ExpiresAt string `json:"expires_at"` + // Quotas struct { + // } `json:"quotas"` + // IsSearchFlushed bool `json:"is_search_flushed"` +} diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index 8e0c6e1bc..e60522e52 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -543,6 +543,32 @@ func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream return nil } +func (xc *XunLeiBrowserCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, dir model.Obj, path string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string diff --git a/drivers/thunder_browser/types.go b/drivers/thunder_browser/types.go index 6b2a41023..a5fda2a5d 100644 --- a/drivers/thunder_browser/types.go +++ b/drivers/thunder_browser/types.go @@ -376,3 +376,10 @@ type ReviewData struct { Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + } `json:"quota"` +} diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index 86ff22bdc..acbb82513 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -423,6 +423,32 @@ func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.F return nil } +func (xc *XunLeiXCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiXCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string diff --git a/drivers/thunderx/types.go b/drivers/thunderx/types.go index e5fbaa241..728fdea31 100644 --- a/drivers/thunderx/types.go +++ b/drivers/thunderx/types.go @@ -303,3 +303,10 @@ type Media struct { IsVisible bool `json:"is_visible"` Category string `json:"category"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + } `json:"quota"` +} From b5626b275bdca8d97c8462f6a7e423746781ec85 Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Wed, 25 Feb 2026 16:32:09 +0800 Subject: [PATCH 18/86] chore(ci)!: update issue CI configuration [skip ci] (#2166) --- .github/ISSUE_TEMPLATE/00-bug_report_zh.yml | 5 +- .github/ISSUE_TEMPLATE/01-bug_report_en.yml | 5 +- .../ISSUE_TEMPLATE/02-feature_request_zh.yml | 4 +- .../ISSUE_TEMPLATE/03-feature_request_en.yml | 6 +- .github/workflows/issue_pr_comment.yml | 56 ++++++++++++++++--- 5 files changed, 58 insertions(+), 18 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index 7dadfc3f4..c6aa3fc62 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -13,7 +13,7 @@ body: attributes: label: 请确认以下事项 description: | - 您必须阅读并检查以下内容，否则您的问题一定会被直接关闭。 + 您必须阅读、检查、确认、同意以下内容，否则您的问题一定会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | @@ -35,8 +35,7 @@ body: - label: | 我已确认这个问题在最新版本中没有被修复。 - label: | - 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue - + 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue 。 - type: input id: version attributes: diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index 5d5d91d00..d99968d90 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -13,7 +13,7 @@ body: attributes: label: Please confirm the following description: | - You must read and check all the following, otherwise your issue will definitely be closed directly. + You must read, check, confirm, and agree to all the following, otherwise your issue will definitely be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | @@ -35,8 +35,7 @@ body: - label: | I confirm this issue is not fixed in the latest version. - label: | - I have not read these checkboxes and therefore I just ticked them all, Please close this issue - + I have not read these checkboxes and therefore I just ticked them all, Please close this issue. - type: input id: version attributes: diff --git a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml index 821b2c446..8339d947f 100644 --- a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml +++ b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml @@ -7,7 +7,7 @@ body: attributes: label: 请确认以下事项 description: | - 您必须勾选以下内容，否则您的问题可能会被直接关闭。 + 您必须阅读、检查、确认、同意以下内容，否则您的问题可能会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | @@ -28,6 +28,8 @@ body: 我已确认此功能尚未被实现。 - label: | 我已确认此功能是合理的，且有普遍需求，并非我个人需要。 + - label: | + 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue 。 - type: textarea id: feature-description attributes: diff --git a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml index 85b024883..41c9990cf 100644 --- a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml +++ b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml @@ -1,13 +1,13 @@ name: "Feature Request" description: Feature Request / Enhancement -title: "[Feature] Please change the title to your feature name" +title: "[Feature] Please modify the title to your feature name" labels: [enhancement] body: - type: checkboxes attributes: label: Please confirm the following description: | - You must check all the following, otherwise your request may be closed directly. + You must read, check, confirm, and agree to all the following, otherwise your request may be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | @@ -28,6 +28,8 @@ body: I confirm this feature has not been implemented yet. - label: | I confirm this feature is reasonable and has general demand, not just my personal need. + - label: | + I have not read these checkboxes and therefore I just ticked them all, Please close this issue. - type: textarea id: feature-description attributes: diff --git a/.github/workflows/issue_pr_comment.yml b/.github/workflows/issue_pr_comment.yml index f15acd424..1b51e23b9 100644 --- a/.github/workflows/issue_pr_comment.yml +++ b/.github/workflows/issue_pr_comment.yml @@ -19,11 +19,49 @@ jobs: uses: actions/github-script@v7 with: script: | + let comment = ""; + const issueTitle = context.payload.issue.title || ""; + const titleNotEdited = /(请修改标题|Please modify the title)/i.test(issueTitle); + if (titleNotEdited) { + comment = "⚠️ 请修改标题以更好地描述您的问题或需求，并删除示例提示。当前 Issue 将被自动关闭。如需继续提交，请创建新的 Issue。\n"; + comment += "⚠️ Please modify the title to better describe your issue or request, and remove the example prompt. This issue will be automatically closed. If you wish to proceed, please create a new issue.\n"; + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); + await github.rest.issues.update({ + ...context.repo, + issue_number: context.issue.number, + state: 'closed', + state_reason: 'not_planned', + labels: ['invalid'] + }); + return; + } const issueBody = context.payload.issue.body || ""; - const unchecked = /- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); - let comment = "感谢您联系OpenList。我们会尽快回复您。\n"; - comment += "Thanks for contacting OpenList. We will reply to you as soon as possible.\n\n"; - if (unchecked) { + const confirmHasRead = /- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); + const confirmNotRead = /- \[[xX]\] (?:我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); + if (confirmNotRead) { + comment = "⚠️ 你的 Issue 不符合提交规则。请先阅读相关规范后再重新提交。当前 Issue 将被自动关闭。如需继续提交，请确认已了解规则后重新打开或创建新的 Issue。\n"; + comment += "⚠️ Your issue does not comply with the submission rules. Please read the guidelines before submitting again. This issue will be automatically closed. If you wish to proceed, please confirm that you have reviewed the rules before reopening or creating a new issue.\n"; + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); + await github.rest.issues.update({ + ...context.repo, + issue_number: context.issue.number, + state: 'closed', + state_reason: 'not_planned', + labels: ['invalid'] + }); + return; + } + if (confirmHasRead) { + comment = "感谢您联系OpenList。我们会尽快回复您。\n"; + comment += "Thanks for contacting OpenList. We will reply to you as soon as possible.\n\n"; comment += "由于您提出的 Issue 中包含部分未确认的项目，为了更好地管理项目，在人工审核后可能会直接关闭此问题。\n"; comment += "如果您能确认并补充相关未确认项目的信息，欢迎随时重新提交。我们会及时关注并处理。感谢您的理解与支持！\n"; comment += "Since your issue contains some unchecked tasks, it may be closed after manual review.\n"; @@ -31,12 +69,12 @@ jobs: comment += "We will pay attention and handle it in a timely manner.\n\n"; comment += "感谢您的理解与支持！\n"; comment += "Thank you for your understanding and support!\n"; + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); } - await github.rest.issues.createComment({ - ...context.repo, - issue_number: context.issue.number, - body: comment - }); pr-title-check: runs-on: ubuntu-latest From 5eaef96078280c3814942e7de76dfe66ca1abe3d Mon Sep 17 00:00:00 2001 From: Roy <358963981@qq.com> Date: Mon, 9 Mar 2026 21:25:30 +0800 Subject: [PATCH 19/86] fix(azure): remove properties and fix prefix (#2209) Remove properties from azure blob response fix azure blob prefix filter: prefix should be empty if it is "/" --- drivers/azure_blob/driver.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/azure_blob/driver.go b/drivers/azure_blob/driver.go index ddfe3ff68..1c0bb9a9c 100644 --- a/drivers/azure_blob/driver.go +++ b/drivers/azure_blob/driver.go @@ -85,6 +85,9 @@ func (d *AzureBlob) Drop(ctx context.Context) error { // List retrieves blobs and directories under the specified path. func (d *AzureBlob) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { prefix := ensureTrailingSlash(dir.GetPath()) + if prefix == "/" { + prefix = "" + } pager := d.containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{ Prefix: &prefix, @@ -100,10 +103,11 @@ func (d *AzureBlob) List(ctx context.Context, dir model.Obj, args model.ListArgs // Process directories for _, blobPrefix := range page.Segment.BlobPrefixes { objs = append(objs, &model.Object{ - Name: path.Base(strings.TrimSuffix(*blobPrefix.Name, "/")), - Path: *blobPrefix.Name, - Modified: *blobPrefix.Properties.LastModified, - Ctime: *blobPrefix.Properties.CreationTime, + Name: path.Base(strings.TrimSuffix(*blobPrefix.Name, "/")), + Path: *blobPrefix.Name, + // Azure does not support properties now. + //Modified: *blobPrefix.Properties.LastModified, + //Ctime: *blobPrefix.Properties.CreationTime, IsFolder: true, }) } From f3428e65bc126ed2c917289c4d9eb02f20cf58f8 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 16 Mar 2026 22:15:00 +0800 Subject: [PATCH 20/86] fix(net): honor proxy settings when uploading to 115/115 Open/PikPak OSS (#2222) * Initial plan * fix: honor HTTPS proxy for OSS uploads Co-authored-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * Honor HTTPS proxy settings for 115/115 Open/PikPak OSS uploads Co-authored-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> * revert * chore --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: jyxjjj <16695261+jyxjjj@users.noreply.github.com> Co-authored-by: jyxjjj <773933146@qq.com> --- drivers/115/util.go | 5 ++-- drivers/115_open/upload.go | 5 ++-- drivers/pikpak/util.go | 5 ++-- internal/net/oss.go | 9 ++++++ internal/net/oss_test.go | 54 ++++++++++++++++++++++++++++++++++++ internal/net/request_test.go | 2 +- 6 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 internal/net/oss.go create mode 100644 internal/net/oss_test.go diff --git a/drivers/115/util.go b/drivers/115/util.go index 7ae375b75..57c987349 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -19,6 +19,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + netutil "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" @@ -222,7 +223,7 @@ func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSPar if err != nil { return nil, err } - ossClient, err := oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret) + ossClient, err := netutil.NewOSSClient(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret) if err != nil { return nil, err } @@ -283,7 +284,7 @@ func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.Upload return nil, err } - if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil { + if ossClient, err = netutil.NewOSSClient(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil { return nil, err } diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index 3575678c2..d02640e2c 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -9,6 +9,7 @@ import ( sdk "github.com/OpenListTeam/115-sdk-go" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + netutil "github.com/OpenListTeam/OpenList/v4/internal/net" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -36,7 +37,7 @@ func calPartSize(fileSize int64) int64 { } func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { - ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) + ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) if err != nil { return err } @@ -70,7 +71,7 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp // } func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { - ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) + ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) if err != nil { return err } diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 9b7207fa2..1d091217a 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -19,6 +19,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + netutil "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -418,7 +419,7 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err } func (d *PikPak) UploadByOSS(ctx context.Context, params *S3Params, s model.FileStreamer, up driver.UpdateProgress) error { - ossClient, err := oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret) + ossClient, err := netutil.NewOSSClient(params.Endpoint, params.AccessKeyID, params.AccessKeySecret) if err != nil { return err } @@ -451,7 +452,7 @@ func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSi bucket *oss.Bucket ) - if ossClient, err = oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret); err != nil { + if ossClient, err = netutil.NewOSSClient(params.Endpoint, params.AccessKeyID, params.AccessKeySecret); err != nil { return err } diff --git a/internal/net/oss.go b/internal/net/oss.go new file mode 100644 index 000000000..a897161f1 --- /dev/null +++ b/internal/net/oss.go @@ -0,0 +1,9 @@ +package net + +import "github.com/aliyun/aliyun-oss-go-sdk/oss" + +func NewOSSClient(endpoint, accessKeyID, accessKeySecret string, options ...oss.ClientOption) (*oss.Client, error) { + clientOptions := []oss.ClientOption{oss.HTTPClient(NewHttpClient())} + clientOptions = append(clientOptions, options...) + return oss.New(endpoint, accessKeyID, accessKeySecret, clientOptions...) +} diff --git a/internal/net/oss_test.go b/internal/net/oss_test.go new file mode 100644 index 000000000..9001cd39d --- /dev/null +++ b/internal/net/oss_test.go @@ -0,0 +1,54 @@ +package net + +import ( + "net/http" + "net/url" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) { + oldConf := conf.Conf + conf.Conf = conf.DefaultConfig("data") + defer func() { + conf.Conf = oldConf + }() + + t.Setenv("HTTP_PROXY", "") + t.Setenv("http_proxy", "") + t.Setenv("HTTPS_PROXY", "http://127.0.0.1:7890") + t.Setenv("https_proxy", "") + t.Setenv("NO_PROXY", "") + t.Setenv("no_proxy", "") + + client, err := NewOSSClient("https://oss-cn-hangzhou.aliyuncs.com", "test-access-key", "test-access-secret") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if client.HTTPClient == nil { + t.Fatal("expected OSS client to use a custom HTTP client") + } + + transport, ok := client.HTTPClient.Transport.(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport, got %T", client.HTTPClient.Transport) + } + + if transport.Proxy == nil { + t.Fatal("expected proxy function to be configured") + } + + req := &http.Request{URL: &url.URL{Scheme: "https", Host: "oss-cn-hangzhou.aliyuncs.com"}} + proxyURL, err := transport.Proxy(req) + if err != nil { + t.Fatalf("expected no proxy lookup error, got %v", err) + } + if proxyURL == nil { + t.Fatal("expected HTTPS proxy to be used") + } + if got, want := proxyURL.String(), "http://127.0.0.1:7890"; got != want { + t.Fatalf("expected proxy %q, got %q", want, got) + } +} diff --git a/internal/net/request_test.go b/internal/net/request_test.go index 00ba8a134..da16a3165 100644 --- a/internal/net/request_test.go +++ b/internal/net/request_test.go @@ -153,7 +153,7 @@ func (c *downloadCaptureClient) HttpRequest(ctx context.Context, params *HttpReq c.GetObjectInvocations++ - if ¶ms.Range != nil { + if params.Range.Length != 0 { c.RetrievedRanges = append(c.RetrievedRanges, fmt.Sprintf("%d-%d", params.Range.Start, params.Range.Length)) } From 9a2ba1dabe3a9006ef6260d4168f0c5fb0ed1364 Mon Sep 17 00:00:00 2001 From: Jealous Date: Mon, 16 Mar 2026 07:22:55 -0700 Subject: [PATCH 21/86] fix(server): add missing return after error responses (#2150) In BeginAuthnRegistration (webauthn.go), missing return statements after error responses caused the function to continue executing with a nil authnInstance, potentially leading to a nil pointer panic. In OIDCLoginCallback and SSOLoginCallback (ssologin.go), missing return statements after GenerateToken/autoRegister errors caused the handler to send a second response, resulting in a superfluous response write. In SetThunderBrowser (offline_download.go), the default case of the storage type switch sent an error response but did not return, causing SaveSettingItems and tool initialization to continue executing even when driver type validation failed. --- server/handles/offline_download.go | 1 + server/handles/ssologin.go | 3 +++ server/handles/webauthn.go | 3 +++ 3 files changed, 7 insertions(+) diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 153b27293..b726d7152 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -448,6 +448,7 @@ func SetThunderBrowser(c *gin.Context) { case *thunder_browser.ThunderBrowser, *thunder_browser.ThunderBrowserExpert: default: common.ErrorStrResp(c, "unsupported storage driver for offline download, only ThunderBrowser is supported", 400) + return } } items := []model.SettingItem{ diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index a36e79d38..4baabf6c1 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -256,11 +256,13 @@ func OIDCLoginCallback(c *gin.Context) { user, err = autoRegister(userID, userID, err) if err != nil { common.ErrorResp(c, err, 400) + return } } token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) + return } if useCompatibility { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) @@ -427,6 +429,7 @@ func SSOLoginCallback(c *gin.Context) { token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) + return } if usecompatibility { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index c7ad4edfe..b2a0fbfb4 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -130,17 +130,20 @@ func BeginAuthnRegistration(c *gin.Context) { authnInstance, err := authn.NewAuthnInstance(c) if err != nil { common.ErrorResp(c, err, 400) + return } options, sessionData, err := authnInstance.BeginRegistration(user) if err != nil { common.ErrorResp(c, err, 400) + return } val, err := json.Marshal(sessionData) if err != nil { common.ErrorResp(c, err, 400) + return } common.SuccessResp(c, gin.H{ From e41b683efbd12634cb5bf030b8604ff26178fa7f Mon Sep 17 00:00:00 2001 From: Elegant1E <104549918+Elegant1E@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:19:45 +0800 Subject: [PATCH 22/86] feat(drivers): add doubao_new driver (#2114) --- drivers/all.go | 1 + drivers/doubao_new/auth.go | 597 +++++++++++++++++++++++++++++ drivers/doubao_new/driver.go | 529 ++++++++++++++++++++++++++ drivers/doubao_new/meta.go | 39 ++ drivers/doubao_new/types.go | 192 ++++++++++ drivers/doubao_new/upload.go | 283 ++++++++++++++ drivers/doubao_new/util.go | 713 +++++++++++++++++++++++++++++++++++ 7 files changed, 2354 insertions(+) create mode 100644 drivers/doubao_new/auth.go create mode 100644 drivers/doubao_new/driver.go create mode 100644 drivers/doubao_new/meta.go create mode 100644 drivers/doubao_new/types.go create mode 100644 drivers/doubao_new/upload.go create mode 100644 drivers/doubao_new/util.go diff --git a/drivers/all.go b/drivers/all.go index 7e1c24bba..fb68d0395 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -29,6 +29,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/crypt" _ "github.com/OpenListTeam/OpenList/v4/drivers/degoo" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao" + _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_new" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/dropbox" _ "github.com/OpenListTeam/OpenList/v4/drivers/febbox" diff --git a/drivers/doubao_new/auth.go b/drivers/doubao_new/auth.go new file mode 100644 index 000000000..537b6a6ea --- /dev/null +++ b/drivers/doubao_new/auth.go @@ -0,0 +1,597 @@ +package doubao_new + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "math/big" + "net/url" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/pkg/cookie" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" + "golang.org/x/crypto/pbkdf2" +) + +const ( + defaultAuthRefreshAheadSeconds = int64(120) + defaultDpopRefreshAheadSeconds = int64(5) +) + +type Clock interface { + Now() (int64, error) +} + +type SystemClock struct{} + +func (SystemClock) Now() (int64, error) { return time.Now().Unix(), nil } + +type DPoPTokenInput struct { + KeyPair *ecdsa.PrivateKey + ExpiresIn int64 // 默认 15 + + JTI string + HTM string + HTU string + IAT int64 + Nonce string + Clock Clock +} + +type DPoPTokenOutput struct { + DPoPToken string `json:"dpopToken"` + ExpiredTime int64 `json:"expiredTime"` + ExpiresIn int64 `json:"expiresIn"` +} + +type JWTPayload struct { + Exp int64 `json:"exp,omitempty"` + Iat int64 `json:"iat,omitempty"` + Nbf int64 `json:"nbf,omitempty"` + Jti string `json:"jti,omitempty"` + Htm string `json:"htm,omitempty"` + Htu string `json:"htu,omitempty"` + Nonce string `json:"nonce,omitempty"` + Sub string `json:"sub,omitempty"` +} + +type jwkECPrivateKey struct { + Kty string `json:"kty"` + Crv string `json:"crv"` + X string `json:"x"` + Y string `json:"y"` + D string `json:"d"` +} + +type dpopKeyPairEnvelope struct { + PrivateKey *jwkECPrivateKey `json:"privateKey"` + KeyPair *jwkECPrivateKey `json:"keyPair"` + JWK *jwkECPrivateKey `json:"jwk"` +} + +type encryptedDpopKeyPair struct { + Data string `json:"data"` + Ciphertext string `json:"ciphertext"` + Encrypted string `json:"encrypted"` + Secret string `json:"secret"` + Password string `json:"password"` + Passphrase string `json:"passphrase"` +} + +func GenerateDPoPToken(in DPoPTokenInput) (*DPoPTokenOutput, error) { + if in.KeyPair == nil { + return nil, errors.New("keyPair required") + } + if in.KeyPair.Curve != elliptic.P256() { + return nil, errors.New("ES256 requires P-256 key") + } + if in.Clock == nil { + in.Clock = SystemClock{} + } + if in.ExpiresIn <= 0 { + in.ExpiresIn = 15 + } + + now, err := in.Clock.Now() + if err != nil { + return nil, err + } + + payload := map[string]any{ + "jti": pickStr(in.JTI, uuid.NewString()), + "htm": pickStr(in.HTM, ""), + "htu": pickStr(in.HTU, ""), + "iat": pickI64(in.IAT, now), + "nonce": pickStr(in.Nonce, uuid.NewString()), + } + if in.ExpiresIn > 0 { + payload["exp"] = payload["iat"].(int64) + in.ExpiresIn + } + + pub := in.KeyPair.PublicKey + header := map[string]any{ + "typ": "dpop+jwt", + "alg": "ES256", + "jwk": map[string]string{ + "kty": "EC", + "crv": "P-256", + "x": b64url(pad32(pub.X.Bytes())), + "y": b64url(pad32(pub.Y.Bytes())), + }, + } + + hb, err := json.Marshal(header) + if err != nil { + return nil, err + } + pb, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + hEnc := b64url(hb) + pEnc := b64url(pb) + signingInput := hEnc + "." + pEnc + + sum := sha256.Sum256([]byte(signingInput)) + r, s, err := ecdsa.Sign(rand.Reader, in.KeyPair, sum[:]) + if err != nil { + return nil, err + } + + sig := append(pad32(r.Bytes()), pad32(s.Bytes())...) + token := signingInput + "." + b64url(sig) + + iat := payload["iat"].(int64) + return &DPoPTokenOutput{ + DPoPToken: token, + ExpiredTime: iat + in.ExpiresIn, + ExpiresIn: in.ExpiresIn, + }, nil +} + +func GenerateDPoPKeyPair() (*ecdsa.PrivateKey, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + return validateP256Key(key) +} + +func ParseJWTPayload(token string, out any) error { + token = strings.TrimSpace(trimTokenScheme(token)) + parts := strings.Split(token, ".") + if len(parts) < 2 { + return fmt.Errorf("invalid JWT format") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return fmt.Errorf("failed to decode JWT payload: %w", err) + } + if err := json.Unmarshal(payload, out); err != nil { + return fmt.Errorf("failed to parse JWT payload: %w", err) + } + return nil +} + +func parseECPrivateKeyJWK(raw string) (*ecdsa.PrivateKey, error) { + var jwk jwkECPrivateKey + if err := json.Unmarshal([]byte(raw), &jwk); err != nil { + return nil, err + } + if jwk.D == "" || jwk.X == "" || jwk.Y == "" { + var env dpopKeyPairEnvelope + if err := json.Unmarshal([]byte(raw), &env); err != nil { + return nil, err + } + switch { + case env.PrivateKey != nil: + jwk = *env.PrivateKey + case env.KeyPair != nil: + jwk = *env.KeyPair + case env.JWK != nil: + jwk = *env.JWK + default: + return nil, errors.New("missing private key JWK") + } + } + + if jwk.Kty != "" && jwk.Kty != "EC" { + return nil, errors.New("unsupported JWK kty") + } + if jwk.Crv != "" && jwk.Crv != "P-256" { + return nil, errors.New("unsupported JWK curve") + } + if jwk.D == "" || jwk.X == "" || jwk.Y == "" { + return nil, errors.New("incomplete JWK") + } + + xBytes, err := base64.RawURLEncoding.DecodeString(jwk.X) + if err != nil { + return nil, fmt.Errorf("invalid jwk x: %w", err) + } + yBytes, err := base64.RawURLEncoding.DecodeString(jwk.Y) + if err != nil { + return nil, fmt.Errorf("invalid jwk y: %w", err) + } + dBytes, err := base64.RawURLEncoding.DecodeString(jwk.D) + if err != nil { + return nil, fmt.Errorf("invalid jwk d: %w", err) + } + + key := &ecdsa.PrivateKey{ + PublicKey: ecdsa.PublicKey{ + Curve: elliptic.P256(), + X: new(big.Int).SetBytes(xBytes), + Y: new(big.Int).SetBytes(yBytes), + }, + D: new(big.Int).SetBytes(dBytes), + } + return validateP256Key(key) +} + +func parseEncryptedDPoPKeyPair(raw, secret string) (*ecdsa.PrivateKey, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, errors.New("empty encrypted key pair") + } + + var payload encryptedDpopKeyPair + ciphertext := raw + if strings.HasPrefix(raw, "{") { + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + return nil, err + } + switch { + case strings.TrimSpace(payload.Data) != "": + ciphertext = strings.TrimSpace(payload.Data) + case strings.TrimSpace(payload.Ciphertext) != "": + ciphertext = strings.TrimSpace(payload.Ciphertext) + case strings.TrimSpace(payload.Encrypted) != "": + ciphertext = strings.TrimSpace(payload.Encrypted) + default: + return nil, errors.New("missing encrypted dpop payload") + } + } + + decoded, err := decodeBase64Loose(ciphertext) + if err != nil { + return nil, err + } + if len(decoded) <= 12 { + return nil, errors.New("encrypted dpop payload too short") + } + + plain, err := decryptDoubaoKeyPair(decoded, secret) + if err != nil { + return nil, fmt.Errorf("failed to decrypt with secret: %w", err) + } + return parseECPrivateKeyJWK(string(plain)) +} + +func decryptDoubaoKeyPair(ciphertext []byte, secret string) ([]byte, error) { + key := pbkdf2.Key([]byte(secret), []byte("fixed-salt"), 100000, 32, sha256.New) + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + aead, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonceSize := aead.NonceSize() + if len(ciphertext) <= nonceSize { + return nil, errors.New("ciphertext too short") + } + nonce := ciphertext[:nonceSize] + enc := ciphertext[nonceSize:] + return aead.Open(nil, nonce, enc, nil) +} + +func decodeBase64Loose(raw string) ([]byte, error) { + raw = strings.TrimSpace(raw) + raw = strings.ReplaceAll(raw, "\n", "") + raw = strings.ReplaceAll(raw, "\r", "") + raw = strings.ReplaceAll(raw, "\t", "") + raw = strings.ReplaceAll(raw, " ", "") + + encodings := []*base64.Encoding{ + base64.StdEncoding, + base64.RawStdEncoding, + base64.URLEncoding, + base64.RawURLEncoding, + } + var lastErr error + for _, enc := range encodings { + decoded, err := enc.DecodeString(raw) + if err == nil { + return decoded, nil + } + lastErr = err + } + if lastErr == nil { + lastErr = errors.New("invalid base64") + } + return nil, lastErr +} + +func validateP256Key(key *ecdsa.PrivateKey) (*ecdsa.PrivateKey, error) { + if key == nil { + return nil, errors.New("nil private key") + } + if key.Curve != elliptic.P256() { + return nil, errors.New("ES256 requires P-256 key") + } + if key.PublicKey.X == nil || key.PublicKey.Y == nil || key.D == nil { + return nil, errors.New("invalid private key") + } + if !key.Curve.IsOnCurve(key.PublicKey.X, key.PublicKey.Y) { + return nil, errors.New("public key is not on P-256 curve") + } + return key, nil +} + +func trimTokenScheme(token string) string { + token = strings.TrimSpace(token) + if i := strings.IndexByte(token, ' '); i > 0 { + scheme := strings.ToLower(strings.TrimSpace(token[:i])) + if scheme == "bearer" || scheme == "dpop" { + return strings.TrimSpace(token[i+1:]) + } + } + return token +} + +func b64url(b []byte) string { + return base64.RawURLEncoding.EncodeToString(b) +} + +func pad32(b []byte) []byte { + if len(b) >= 32 { + return b[len(b)-32:] + } + out := make([]byte, 32) + copy(out[32-len(b):], b) + return out +} + +func pickStr(v, def string) string { + if v != "" { + return v + } + return def +} + +func pickI64(v, def int64) int64 { + if v != 0 { + return v + } + return def +} + +func (d *DoubaoNew) resolveAuthorization() string { + auth := trimTokenScheme(d.Authorization) + if auth == "" { + return "" + } + return "DPoP " + auth +} + +func shouldRefreshJWT(token string) bool { + if token == "" { + return true + } + var payload JWTPayload + if err := ParseJWTPayload(token, &payload); err != nil { + return true + } + if payload.Exp <= 0 { + return false + } + return payload.Exp <= time.Now().Unix()+defaultAuthRefreshAheadSeconds +} + +func (d *DoubaoNew) fetchBizAuth(dpop string, public bool) (string, error) { + var reqUrl string + client := base.RestyClient.Clone() + req := client.R() + req.SetHeader("accept", "application/json, text/javascript") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + req.SetHeader("content-type", "application/x-www-form-urlencoded") + if public { + reqUrl = DoubaoURL + "/passport/anonymity_user/biz_auth/" + } else { + reqUrl = DoubaoURL + "/passport/user/biz_auth/" + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + if csrf := strings.TrimSpace(cookie.GetStr(d.Cookie, "passport_csrf_token")); csrf != "" { + req.SetHeader("x-tt-passport-csrf-token", csrf) + } + } + if oldAuth := d.resolveAuthorization(); oldAuth != "" { + req.SetHeader("authorization", oldAuth) + } + } + if dpop != "" { + req.SetHeader("dpop", dpop) + } + values := url.Values{} + values.Set("client_id", d.AuthClientID) + values.Set("client_type", d.AuthClientType) + values.Set("scope", d.AuthScope) + values.Set("d_pop", dpop) + req.SetBody(values.Encode()) + req.SetQueryParam("aid", d.AppID) + req.SetQueryParam("account_sdk_source", d.AuthSDKSource) + req.SetQueryParam("sdk_version", d.AuthSDKVersion) + + res, err := req.Post(reqUrl) + if err != nil { + return "", err + } + var resp bizAuthResp + if err = json.Unmarshal(res.Body(), &resp); err != nil { + return "", err + } + if resp.Message != "success" || resp.Data.AccessToken == "" { + return "", fmt.Errorf("[doubao_new] %s: %s", resp.Message, resp.Data.Description) + } + return resp.Data.AccessToken, nil +} + +func (d *DoubaoNew) refreshAuthorizationWithDPoP(dpop string) (string, error) { + token, err := d.fetchBizAuth(dpop, false) + if err == nil && token != "" { + return token, nil + } + if err == nil { + err = errors.New("biz auth refresh failed") + } + return "", err +} + +func (d *DoubaoNew) resolveDpopForRequest(method, rawURL string) (string, error) { + if d.DPoPKeyPair != nil { + proof, err := GenerateDPoPToken(DPoPTokenInput{ + KeyPair: d.DPoPKeyPair, + HTM: strings.ToUpper(strings.TrimSpace(method)), + HTU: normalizeDPoPURL(rawURL), + }) + if err != nil { + return "", err + } + return proof.DPoPToken, nil + } + + static := d.DPoP + if static == "" { + return "", nil + } + if !d.IgnoreJWTCheck { + if payload, err := parseDPoPPayload(static); err == nil && payload.Exp > 0 { + now := time.Now().Unix() + if payload.Exp <= now+defaultDpopRefreshAheadSeconds { + return "", errors.New("static dpop token expired or near expiry; configure dpop_key_pair for automatic refresh") + } + } + } + return static, nil +} + +func (d *DoubaoNew) ensureAuthAdditons() bool { + return d.DPoPKeySecret != "" && d.AuthClientID != "" && d.AuthClientType != "" && + d.AuthScope != "" && d.AuthSDKSource != "" && d.AuthSDKVersion != "" +} + +func (d *DoubaoNew) resolveAuthorizationForRequest(method, rawURL string) (string, error) { + if !shouldRefreshJWT(d.Authorization) { + return d.resolveAuthorization(), nil + } + + if d.DPoPKeyPair == nil || strings.TrimSpace(d.Cookie) == "" || !d.ensureAuthAdditons() { + return d.resolveAuthorization(), nil + } + + d.authRefreshMu.Lock() + defer d.authRefreshMu.Unlock() + + if !shouldRefreshJWT(d.Authorization) { + return d.resolveAuthorization(), nil + } + + refreshDpop, err := d.resolveDpopForRequest(method, rawURL) + if err != nil || refreshDpop == "" { + return "", err + } + + newToken, err := d.refreshAuthorizationWithDPoP(refreshDpop) + if err != nil { + if auth := d.resolveAuthorization(); auth != "" { + return auth, nil + } + return "", err + } + d.Authorization = trimTokenScheme(newToken) + return d.resolveAuthorization(), nil +} + +func (d *DoubaoNew) resolveAuthorizationForPublic() (dpop string, auth string, err error) { + if d.DPoPPublic != "" && !shouldRefreshJWT(d.AuthorizationPublic) { + return d.DPoPPublic, "DPoP " + d.AuthorizationPublic, nil + } + + if !d.ensureAuthAdditons() { + return "", "", fmt.Errorf("[doubao_new] missing auth additions, please fill them all") + } + + d.authRefreshPublicMu.Lock() + defer d.authRefreshPublicMu.Unlock() + + if d.DPoPPublic != "" && !shouldRefreshJWT(d.AuthorizationPublic) { + return d.DPoPPublic, "DPoP " + d.AuthorizationPublic, nil + } + + // generate new public dpop + keypair, err := GenerateDPoPKeyPair() + if err != nil { + return "", "", err + } + proof, err := GenerateDPoPToken(DPoPTokenInput{ + KeyPair: keypair, + }) + d.DPoPPublic = proof.DPoPToken + + // get authorization token + d.AuthorizationPublic, err = d.fetchBizAuth(proof.DPoPToken, true) + if err != nil { + return "", "", err + } + + return d.DPoPPublic, "DPoP " + d.AuthorizationPublic, nil +} + +func (d *DoubaoNew) applyAuthHeaders(req *resty.Request, method, rawURL string) error { + auth, err := d.resolveAuthorizationForRequest(method, rawURL) + if err != nil { + return err + } + if auth != "" { + req.SetHeader("authorization", auth) + } + dpop, err := d.resolveDpopForRequest(method, rawURL) + if err != nil { + return err + } + if dpop != "" { + req.SetHeader("dpop", dpop) + } + return nil +} + +func normalizeDPoPURL(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + u.Fragment = "" + return u.String() +} + +func parseDPoPPayload(token string) (*JWTPayload, error) { + var payload JWTPayload + if err := ParseJWTPayload(token, &payload); err != nil { + return nil, err + } + return &payload, nil +} diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go new file mode 100644 index 000000000..2a90cf16c --- /dev/null +++ b/drivers/doubao_new/driver.go @@ -0,0 +1,529 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/cookie" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type DoubaoNew struct { + model.Storage + Addition + TtLogid string + + // DPoP access token (Authorization header value, without DPoP prefix) + Authorization string + AuthorizationPublic string + // DPoP header value + DPoP string + DPoPPublic string + // DPoP key pair for generating DPoP + DPoPKeyPairStr string + DPoPKeyPair *ecdsa.PrivateKey + + authRefreshMu sync.Mutex + authRefreshPublicMu sync.Mutex +} + +func (d *DoubaoNew) Config() driver.Config { + return config +} + +func (d *DoubaoNew) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *DoubaoNew) Init(ctx context.Context) error { + if cookieStr := strings.TrimSpace(d.Cookie); cookieStr != "" { + d.Cookie = cookieStr + auth := trimTokenScheme(cookie.GetStr(d.Cookie, "LARK_SUITE_ACCESS_TOKEN")) + if auth != "" { + d.Authorization = auth + } + dpop := strings.TrimSpace(cookie.GetStr(d.Cookie, "LARK_SUITE_DPOP")) + if dpop != "" { + d.DPoP = dpop + } + keypair := strings.TrimSpace(cookie.GetStr(d.Cookie, "feishu_dpop_keypair")) + if keypair != "" && d.DPoPKeySecret != "" { + d.DPoPKeyPairStr = keypair + d.DPoPKeyPair, _ = parseEncryptedDPoPKeyPair(keypair, d.DPoPKeySecret) + } + } + return nil +} + +func (d *DoubaoNew) Drop(ctx context.Context) error { + if d.Authorization != "" { + d.Cookie = cookie.SetStr(d.Cookie, "LARK_SUITE_ACCESS_TOKEN", d.Authorization) + } + if d.DPoP != "" { + d.Cookie = cookie.SetStr(d.Cookie, "LARK_SUITE_DPOP", d.DPoP) + } + if d.DPoPKeyPairStr != "" { + d.Cookie = cookie.SetStr(d.Cookie, "feishu_dpop_keypair", d.DPoPKeyPairStr) + } + op.MustSaveDriverStorage(d) + return nil +} + +func (d *DoubaoNew) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + nodes, err := d.listAllChildren(ctx, dir.GetID()) + if err != nil { + return nil, err + } + + objs := make([]model.Obj, 0, len(nodes)) + for _, node := range nodes { + if node.NodeToken == "" || node.ObjToken == "" { + continue + } + + size := parseSize(node.Extra.Size) + isFolder := node.Type == 0 + if isFolder && node.NodeToken == dir.GetID() { + continue + } + + obj := &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: dir.GetID(), + Name: node.Name, + Size: size, + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: isFolder, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + } + objs = append(objs, obj) + } + + return objs, nil +} + +func (d *DoubaoNew) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj, ok := file.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + if obj.IsFolder { + return nil, fmt.Errorf("link is directory") + } + var ( + err error + auth, dpop string + ) + if d.ShareLink { + err := d.createShare(ctx, obj) + if err != nil { + return nil, err + } + dpop, auth, err = d.resolveAuthorizationForPublic() + } else { + // TODO: append previewLink() with auth args to support ShareLink + if args.Type == "preview" || args.Type == "thumb" { + if link, err := d.previewLink(ctx, obj, args); err == nil { + return link, nil + } + } + auth = d.resolveAuthorization() + dpop, err = d.resolveDpopForRequest(http.MethodGet, DownloadBaseURL+"/space/api/box/stream/download/all/"+obj.ObjToken+"/") + } + if err != nil { + return nil, err + } + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + + query := url.Values{} + query.Set("authorization", auth) + query.Set("dpop", dpop) + + downloadURL := DownloadBaseURL + "/space/api/box/stream/download/all/" + obj.ObjToken + "/?" + query.Encode() + + headers := http.Header{ + "Referer": []string{DoubaoURL + "/"}, + "User-Agent": []string{base.UserAgent}, + } + + return &model.Link{ + URL: downloadURL, + Header: headers, + }, nil +} + +func (d *DoubaoNew) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + node, err := d.createFolder(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + return &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: parentDir.GetID(), + Name: node.Name, + Size: parseSize(node.Extra.Size), + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: true, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + }, nil +} + +func (d *DoubaoNew) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if dstDir == nil { + return nil, errors.New("nil destination dir") + } + srcToken := srcObj.GetID() + if srcToken == "" { + if obj, ok := srcObj.(*Object); ok { + srcToken = obj.ObjToken + } + } + if srcToken == "" { + return nil, errors.New("missing source token") + } + if err := d.moveObj(ctx, srcToken, dstDir.GetID()); err != nil { + return nil, err + } + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Path = dstDir.GetID() + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if srcObj.IsDir() { + if err := d.renameFolder(ctx, srcObj.GetID(), newName); err != nil { + return nil, err + } + } else { + fileToken := "" + if obj, ok := srcObj.(*Object); ok { + fileToken = obj.ObjToken + } + if fileToken == "" { + fileToken = srcObj.GetID() + } + if err := d.renameFile(ctx, fileToken, newName); err != nil { + return nil, err + } + } + + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Name = newName + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Remove(ctx context.Context, obj model.Obj) error { + if obj == nil { + return errors.New("nil object") + } + token := obj.GetID() + if token == "" { + if o, ok := obj.(*Object); ok { + token = o.ObjToken + } + } + if token == "" { + return errors.New("missing object token") + } + return d.removeObj(ctx, []string{token}) +} + +func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if file == nil { + return nil, errors.New("nil file") + } + if file.GetSize() <= 0 { + return nil, errors.New("invalid file size") + } + + uploadPrep, err := d.prepareUpload(ctx, file.GetName(), file.GetSize(), dstDir.GetID()) + if err != nil { + return nil, err + } + if uploadPrep.BlockSize <= 0 { + return nil, errors.New("invalid block size from prepare") + } + + tmpFile, err := utils.CreateTempFile(file, file.GetSize()) + if err != nil { + return nil, err + } + defer tmpFile.Close() + + blockSize := uploadPrep.BlockSize + totalSize := file.GetSize() + numBlocks := int((totalSize + blockSize - 1) / blockSize) + blocks := make([]UploadBlock, 0, numBlocks) + blockMeta := make(map[int]UploadBlock, numBlocks) + + for seq := 0; seq < numBlocks; seq++ { + offset := int64(seq) * blockSize + length := blockSize + if remain := totalSize - offset; remain < length { + length = remain + } + buf := make([]byte, int(length)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + buf = buf[:n] + sum := sha256.Sum256(buf) + hash := base64.StdEncoding.EncodeToString(sum[:]) + checksum := adler32String(buf) + + block := UploadBlock{ + Hash: hash, + Seq: seq, + Size: int64(n), + Checksum: checksum, + IsUploaded: true, + } + blocks = append(blocks, block) + blockMeta[seq] = block + } + + needed, err := d.uploadBlocks(ctx, uploadPrep.UploadID, blocks, "explorer") + if err != nil { + return nil, err + } + + if len(needed.NeededUploadBlocks) > 0 { + sort.Slice(needed.NeededUploadBlocks, func(i, j int) bool { + return needed.NeededUploadBlocks[i].Seq < needed.NeededUploadBlocks[j].Seq + }) + const maxMergeBlockCount = 20 + var ( + groupSeqs []int + groupChecksums []string + groupSizes []int64 + groupRealSize int64 + groupExpectSum int64 + groupBuf bytes.Buffer + uploadedBytes int64 + ) + + flushGroup := func() error { + if len(groupSeqs) == 0 { + return nil + } + data := groupBuf.Bytes() + expectLen := groupExpectSum + if int64(len(data)) != expectLen { + return fmt.Errorf("[doubao_new] merge blocks invalid body len: got=%d expect=%d seqs=%v", len(data), expectLen, groupSeqs) + } + mergeResp, err := d.mergeUploadBlocks(ctx, uploadPrep.UploadID, groupSeqs, groupChecksums, groupSizes, blockSize, data) + if err != nil { + return err + } + if len(mergeResp.SuccessSeqList) != len(groupSeqs) { + return fmt.Errorf("[doubao_new] merge blocks incomplete: %v", mergeResp.SuccessSeqList) + } + success := make(map[int]bool, len(mergeResp.SuccessSeqList)) + for _, seq := range mergeResp.SuccessSeqList { + success[seq] = true + } + for _, seq := range groupSeqs { + if !success[seq] { + return fmt.Errorf("[doubao_new] merge blocks missing seq %d", seq) + } + } + + uploadedBytes += groupRealSize + groupSeqs = groupSeqs[:0] + groupChecksums = groupChecksums[:0] + groupSizes = groupSizes[:0] + groupRealSize = 0 + groupExpectSum = 0 + groupBuf.Reset() + if up != nil { + percent := float64(uploadedBytes) / float64(totalSize) * 100 + up(percent) + } + return nil + } + + for _, item := range needed.NeededUploadBlocks { + if _, ok := blockMeta[item.Seq]; !ok { + return nil, fmt.Errorf("[doubao_new] missing block meta for seq %d", item.Seq) + } + if item.Size <= 0 { + return nil, fmt.Errorf("[doubao_new] invalid block size from needed list: seq=%d size=%d", item.Seq, item.Size) + } + offset := int64(item.Seq) * blockSize + buf := make([]byte, int(item.Size)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + if n != len(buf) { + return nil, fmt.Errorf("[doubao_new] short read: seq=%d want=%d got=%d", item.Seq, len(buf), n) + } + buf = buf[:n] + realAdler := adler32String(buf) + if realAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] block checksum mismatch: seq=%d offset=%d adler32=%s step2=%s", item.Seq, offset, realAdler, item.Checksum) + } + payloadStart := groupBuf.Len() + groupBuf.Write(buf) + payloadEnd := groupBuf.Len() + payloadAdler := adler32String(groupBuf.Bytes()[payloadStart:payloadEnd]) + if payloadAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] payload checksum mismatch: seq=%d start=%d end=%d adler32=%s step2=%s", item.Seq, payloadStart, payloadEnd, payloadAdler, item.Checksum) + } + groupSeqs = append(groupSeqs, item.Seq) + groupChecksums = append(groupChecksums, item.Checksum) + groupSizes = append(groupSizes, item.Size) + groupRealSize += int64(n) + groupExpectSum += item.Size + if len(groupSeqs) >= maxMergeBlockCount { + if err := flushGroup(); err != nil { + return nil, err + } + } + } + + if err := flushGroup(); err != nil { + return nil, err + } + if up != nil { + up(100) + } + } else if up != nil { + up(100) + } + + numBlocksFinish := uploadPrep.NumBlocks + if numBlocksFinish <= 0 { + numBlocksFinish = numBlocks + } + finish, err := d.finishUpload(ctx, uploadPrep.UploadID, numBlocksFinish, "explorer") + if err != nil { + return nil, err + } + + nodeToken := finish.Extra.NodeToken + if nodeToken == "" { + nodeToken = finish.FileToken + } + now := time.Now() + return &Object{ + Object: model.Object{ + ID: nodeToken, + Path: dstDir.GetID(), + Name: file.GetName(), + Size: file.GetSize(), + Modified: now, + Ctime: now, + IsFolder: false, + }, + ObjToken: finish.FileToken, + }, nil +} + +func (d *DoubaoNew) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + data, err := d.getUserStorage(ctx) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: data.TotalSizeLimitBytes, + UsedSpace: data.UsedSizeBytes, + }, + }, nil +} + +func (d *DoubaoNew) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch args.Method { + case "doubao_preview", "preview": + obj, ok := args.Obj.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errs.NotSupport + } + + imgExt := ".webp" + pageNums := 1 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + if extra.PageNums > 0 { + pageNums = extra.PageNums + } + } + } + + return base.Json{ + "version": info.Version, + "img_ext": imgExt, + "page_nums": pageNums, + }, nil + default: + return nil, errs.NotSupport + } +} + +var _ driver.Driver = (*DoubaoNew)(nil) diff --git a/drivers/doubao_new/meta.go b/drivers/doubao_new/meta.go new file mode 100644 index 000000000..2345357bf --- /dev/null +++ b/drivers/doubao_new/meta.go @@ -0,0 +1,39 @@ +package doubao_new + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + Cookie string `json:"cookie" required:"true" help:"Web Cookie"` + AppID string `json:"app_id" required:"true" default:"497858" help:"Doubao App ID"` + DPoPKeySecret string `json:"dpop_key_secret" help:"DPoP Key Secret for generating DPoP token"` + AuthClientID string `json:"auth_client_id" help:"Doubao Biz Auth Client ID"` + AuthClientType string `json:"auth_client_type" help:"Doubao Biz Auth Client Type"` + AuthScope string `json:"auth_scope" help:"Doubao Biz Auth Scope"` + AuthSDKSource string `json:"auth_sdk_source" help:"Doubao Biz Auth SDK Source"` + AuthSDKVersion string `json:"auth_sdk_version" help:"Doubao Biz Auth SDK Version"` + ShareLink bool `json:"share_link" help:"Whether to use share link for download"` + IgnoreJWTCheck bool `json:"ignore_jwt_check" help:"Whether to ignore JWT check to prevent time issue"` +} + +var config = driver.Config{ + Name: "DoubaoNew", + LocalSort: true, + DefaultRoot: "", + Alert: `danger|Do not use 302 if the storage is public accessible. +Otherwise, the download link may leak sensitive information such as access token or signature. +Others may use the leaked link to access all your files.`, + NoOverwriteUpload: false, + PreferProxy: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &DoubaoNew{} + }) +} diff --git a/drivers/doubao_new/types.go b/drivers/doubao_new/types.go new file mode 100644 index 000000000..3b64a2a20 --- /dev/null +++ b/drivers/doubao_new/types.go @@ -0,0 +1,192 @@ +package doubao_new + +import "github.com/OpenListTeam/OpenList/v4/internal/model" + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` +} + +type ListResp struct { + BaseResp + Data ListData `json:"data"` +} + +type ListData struct { + HasMore bool `json:"has_more"` + LastLabel string `json:"last_label"` + NodeList []string `json:"node_list"` + Entities struct { + Nodes map[string]Node `json:"nodes"` + Users map[string]User `json:"users"` + } `json:"entities"` +} + +type Node struct { + Token string `json:"token"` + NodeToken string `json:"node_token"` + ObjToken string `json:"obj_token"` + Name string `json:"name"` + Type int `json:"type"` + NodeType int `json:"node_type"` + OwnerID string `json:"owner_id"` + EditUID string `json:"edit_uid"` + CreateTime int64 `json:"create_time"` + EditTime int64 `json:"edit_time"` + URL string `json:"url"` + Extra struct { + Size string `json:"size"` + } `json:"extra"` +} + +type User struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type Object struct { + model.Object + ObjToken string + NodeType int + ObjType int + URL string +} + +type CreateFolderResp struct { + BaseResp + Data struct { + Entities struct { + Nodes map[string]Node `json:"nodes"` + } `json:"entities"` + NodeList []string `json:"node_list"` + } `json:"data"` +} + +type FileInfoResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileInfo `json:"data"` +} + +type FileInfo struct { + Name string `json:"name"` + NumBlocks int `json:"num_blocks"` + Version string `json:"version"` + MimeType string `json:"mime_type"` + MountPoint string `json:"mount_point"` + PreviewMeta PreviewMeta `json:"preview_meta"` +} + +type PreviewMeta struct { + Data map[string]PreviewMetaEntry `json:"data"` +} + +type PreviewMetaEntry struct { + Status int `json:"status"` + Extra string `json:"extra"` + PreviewFileSize int64 `json:"preview_file_size"` +} + +type PreviewImageExtra struct { + ImgExt string `json:"img_ext"` + PageNums int `json:"page_nums"` +} + +type UserStorageResp struct { + BaseResp + Data UserStorageData `json:"data"` +} + +type UserStorageData struct { + ShowSizeLimit bool `json:"show_size_limit"` + TotalSizeLimitBytes int64 `json:"total_size_limit_bytes"` + UsedSizeBytes int64 `json:"used_size_bytes"` +} + +type UploadPrepareResp struct { + BaseResp + Data UploadPrepareData `json:"data"` +} + +type UploadPrepareData struct { + BlockSize int64 `json:"block_size"` + NumBlocks int `json:"num_blocks"` + OptionBlockSize int64 `json:"option_block_size"` + DedupeSupport bool `json:"dedupe_support"` + UploadID string `json:"upload_id"` +} + +type UploadBlock struct { + Hash string `json:"hash"` + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + IsUploaded bool `json:"isUploaded"` +} + +type UploadBlocksResp struct { + BaseResp + Data UploadBlocksData `json:"data"` +} + +type UploadBlocksData struct { + NeededUploadBlocks []UploadBlockNeed `json:"needed_upload_blocks"` +} + +type UploadBlockNeed struct { + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + Hash string `json:"hash"` +} + +type UploadMergeResp struct { + BaseResp + Data UploadMergeData `json:"data"` +} + +type UploadMergeData struct { + SuccessSeqList []int `json:"success_seq_list"` +} + +type UploadFinishResp struct { + BaseResp + Data UploadFinishData `json:"data"` +} + +type UploadFinishData struct { + Version string `json:"version"` + DataVersion string `json:"data_version"` + Extra struct { + NodeToken string `json:"node_token"` + } `json:"extra"` + FileToken string `json:"file_token"` +} + +type RemoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + } `json:"data"` +} + +type TaskStatusResp struct { + BaseResp + Data TaskStatusData `json:"data"` +} + +type TaskStatusData struct { + IsFinish bool `json:"is_finish"` + IsFail bool `json:"is_fail"` +} + +type bizAuthResp struct { + Data struct { + AccessToken string `json:"access_token"` + AuthScheme string `json:"auth_scheme"` + ExpiresIn int64 `json:"expires_in"` + Description string `json:"description,omitempty"` + } `json:"data"` + Message string `json:"message"` +} diff --git a/drivers/doubao_new/upload.go b/drivers/doubao_new/upload.go new file mode 100644 index 000000000..6f3a8b068 --- /dev/null +++ b/drivers/doubao_new/upload.go @@ -0,0 +1,283 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/go-resty/resty/v2" +) + +func (d *DoubaoNew) prepareUpload(ctx context.Context, name string, size int64, mountNodeToken string) (UploadPrepareData, error) { + var resp UploadPrepareResp + _, err := d.request(ctx, "/space/api/box/upload/prepare/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.prepare") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + body := base.Json{ + "mount_point": "explorer", + "mount_node_token": "", + "name": name, + "size": size, + "size_checker": true, + } + if mountNodeToken != "" { + body["mount_node_token"] = mountNodeToken + } + req.SetBody(body) + }, &resp) + if err != nil { + return UploadPrepareData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlocks(ctx context.Context, uploadID string, blocks []UploadBlock, mountPoint string) (UploadBlocksData, error) { + if uploadID == "" { + return UploadBlocksData{}, fmt.Errorf("[doubao_new] upload blocks missing upload_id") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadBlocksResp + _, err := d.request(ctx, "/space/api/box/upload/blocks/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.blocks") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetBody(base.Json{ + "blocks": blocks, + "upload_id": uploadID, + "mount_point": mountPoint, + }) + }, &resp) + if err != nil { + return UploadBlocksData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) mergeUploadBlocks(ctx context.Context, uploadID string, seqList []int, checksumList []string, sizeList []int64, blockOriginSize int64, data []byte) (UploadMergeData, error) { + if uploadID == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing upload_id") + } + if len(seqList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty seq list") + } + if len(checksumList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty checksum list") + } + if len(sizeList) != len(seqList) { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks size list mismatch") + } + if blockOriginSize <= 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks invalid block origin size") + } + if len(data) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty data") + } + + seqHeader := joinIntComma(seqList) + checksumHeader := buildCommaHeader(checksumList) + + client := base.NewRestyClient() + client.SetCookieJar(nil) + req := client.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("content-type", "application/octet-stream") + req.SetHeader("x-block-list-checksum", checksumHeader) + req.SetHeader("x-seq-list", seqHeader) + req.SetHeader("x-block-origin-size", strconv.FormatInt(blockOriginSize, 10)) + req.SetHeader("x-command", "space.api.box.stream.upload.merge_block") + req.SetHeader("x-csrftoken", "") + reqID := "" + if buf := make([]byte, 16); true { + if _, err := rand.Read(buf); err == nil { + reqID = hex.EncodeToString(buf) + } + } + if reqID != "" { + req.SetHeader("x-request-id", reqID) + } + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + urlStr := DownloadBaseURL + "/space/api/box/stream/upload/merge_block/?" + values.Encode() + if err := d.applyAuthHeaders(req, http.MethodPost, urlStr); err != nil { + return UploadMergeData{}, err + } + req.Header.Del("cookie") + if req.Header.Get("x-command") == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing x-command header") + } + req.SetBody(data) + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return UploadMergeData{}, err + } + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + body := res.Body() + var resp UploadMergeResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UploadMergeData{}, fmt.Errorf("%s", msg) + } + if resp.Code != 0 { + if res != nil && res.StatusCode() == http.StatusBadRequest && resp.Code == 2 { + success := make([]int, 0, len(seqList)) + offset := 0 + for i, seq := range seqList { + size := sizeList[i] + if size <= 0 { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback invalid size: seq=%d size=%d", seq, size) + } + if offset+int(size) > len(data) { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback payload out of range: seq=%d offset=%d size=%d total=%d", seq, offset, size, len(data)) + } + payload := data[offset : offset+int(size)] + block := UploadBlockNeed{ + Seq: seq, + Size: size, + Checksum: checksumList[i], + } + if err := d.uploadBlockV3(ctx, uploadID, block, payload); err != nil { + return UploadMergeData{SuccessSeqList: success}, err + } + success = append(success, seq) + offset += int(size) + } + return UploadMergeData{SuccessSeqList: success}, nil + } + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UploadMergeData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlockV3(ctx context.Context, uploadID string, block UploadBlockNeed, data []byte) error { + if uploadID == "" { + return fmt.Errorf("[doubao_new] upload v3 block missing upload_id") + } + if block.Seq < 0 { + return fmt.Errorf("[doubao_new] upload v3 block invalid seq") + } + if len(data) == 0 { + return fmt.Errorf("[doubao_new] upload v3 block empty data") + } + + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("x-block-seq", strconv.Itoa(block.Seq)) + req.SetHeader("x-block-checksum", block.Checksum) + req.SetMultipartFormData(map[string]string{ + "upload_id": uploadID, + "size": strconv.FormatInt(int64(len(data)), 10), + }) + req.SetMultipartField("file", "blob", "application/octet-stream", bytes.NewReader(data)) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("seq", strconv.Itoa(block.Seq)) + values.Set("size", strconv.FormatInt(int64(len(data)), 10)) + values.Set("checksum", block.Checksum) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + urlStr := DownloadBaseURL + "/space/api/box/stream/upload/v3/block/?" + values.Encode() + if err := d.applyAuthHeaders(req, http.MethodPost, urlStr); err != nil { + return err + } + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return err + } + body := res.Body() + if err := decodeBaseResp(body, res); err != nil { + return err + } + return nil +} + +func (d *DoubaoNew) finishUpload(ctx context.Context, uploadID string, numBlocks int, mountPoint string) (UploadFinishData, error) { + if uploadID == "" { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload missing upload_id") + } + if numBlocks <= 0 { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload invalid num_blocks") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadFinishResp + _, err := d.request(ctx, "/space/api/box/upload/finish/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.finish") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetHeader("biz-scene", "file_upload") + req.SetHeader("biz-ua-type", "Web") + req.SetBody(base.Json{ + "upload_id": uploadID, + "num_blocks": numBlocks, + "mount_point": mountPoint, + "push_open_history_record": 1, + }) + }, &resp) + if err != nil { + return UploadFinishData{}, err + } + return resp.Data, nil +} diff --git a/drivers/doubao_new/util.go b/drivers/doubao_new/util.go new file mode 100644 index 000000000..8ee38625d --- /dev/null +++ b/drivers/doubao_new/util.go @@ -0,0 +1,713 @@ +package doubao_new + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "hash/adler32" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/cookie" + "github.com/go-resty/resty/v2" +) + +const ( + BaseURL = "https://my.feishu.cn" + DownloadBaseURL = "https://internal-api-drive-stream.feishu.cn" + DoubaoURL = "https://www.doubao.com" +) + +var defaultObjTypes = []string{"124", "0", "12", "30", "123", "22"} + +func (d *DoubaoNew) request(ctx context.Context, path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, method, BaseURL+path); err != nil { + return nil, err + } + + if callback != nil { + callback(req) + } + + res, err := req.Execute(method, BaseURL+path) + if err != nil { + return nil, err + } + if res != nil { + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + } + + body := res.Body() + var common BaseResp + if err = json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return body, fmt.Errorf("%s", msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return body, fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + if resp != nil { + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func adler32String(data []byte) string { + sum := adler32.Checksum(data) + return strconv.FormatUint(uint64(sum), 10) +} + +func buildCommaHeader(items []string) string { + return strings.Join(items, ",") +} + +func joinIntComma(items []int) string { + if len(items) == 0 { + return "" + } + var sb strings.Builder + for i, v := range items { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(strconv.Itoa(v)) + } + return sb.String() +} + +func previewList(items []string, n int) string { + if n <= 0 || len(items) == 0 { + return "" + } + if len(items) < n { + n = len(items) + } + return strings.Join(items[:n], ",") +} + +func parseSize(size string) int64 { + if size == "" { + return 0 + } + val, err := strconv.ParseInt(size, 10, 64) + if err != nil { + return 0 + } + return val +} + +func (d *DoubaoNew) listChildren(ctx context.Context, parentToken string, lastLabel string, length int) (ListData, error) { + var resp ListResp + _, err := d.request(ctx, "/space/api/explorer/doubao/children/list/", http.MethodGet, func(req *resty.Request) { + values := url.Values{} + for _, t := range defaultObjTypes { + values.Add("obj_type", t) + } + values.Set("length", strconv.Itoa(length)) + values.Set("rank", "0") + values.Set("asc", "0") + values.Set("min_length", "40") + values.Set("thumbnail_width", "1028") + values.Set("thumbnail_height", "1028") + values.Set("thumbnail_policy", "4") + if parentToken != "" { + values.Set("token", parentToken) + } + if lastLabel != "" { + values.Set("last_label", lastLabel) + } + req.SetQueryParamsFromValues(values) + }, &resp) + if err != nil { + return ListData{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) listAllChildren(ctx context.Context, parentToken string) ([]Node, error) { + length := 50 + nodes := make([]Node, 0, length) + lastLabel := "" + for range 100 { + data, err := d.listChildren(ctx, parentToken, lastLabel, length) + if err != nil { + return nil, err + } + + if len(data.NodeList) > 0 { + for _, token := range data.NodeList { + node, ok := data.Entities.Nodes[token] + if !ok { + continue + } + nodes = append(nodes, node) + } + } else { + for _, node := range data.Entities.Nodes { + nodes = append(nodes, node) + } + } + + if !data.HasMore || data.LastLabel == "" || data.LastLabel == lastLabel { + break + } + lastLabel = data.LastLabel + } + + if len(nodes) == 0 { + return nil, nil + } + return nodes, nil +} + +func (d *DoubaoNew) getFileInfo(ctx context.Context, fileToken string) (FileInfo, error) { + var resp FileInfoResp + _, err := d.request(ctx, "/space/api/box/file/info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "caller": "explorer", + "file_token": fileToken, + "mount_point": "explorer", + "option_params": []string{"preview_meta", "check_cipher"}, + }) + }, &resp) + if err != nil { + return FileInfo{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) previewLink(ctx context.Context, obj *Object, args model.LinkArgs) (*model.Link, error) { + auth := d.resolveAuthorization() + dpop, err := d.resolveDpopForRequest(http.MethodGet, fmt.Sprintf("%s/space/api/box/stream/download/preview_sub/%s", BaseURL, obj.ObjToken)) + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errors.New("preview not available") + } + + subID := "" + pageIndex := 0 + + if subID == "" { + imgExt := ".webp" + pageNums := 0 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + pageNums = extra.PageNums + } + } + if pageNums > 0 && pageIndex >= pageNums { + pageIndex = pageNums - 1 + } + subID = fmt.Sprintf("img_%d%s", pageIndex, imgExt) + } + + query := url.Values{} + query.Set("preview_type", "22") + query.Set("sub_id", subID) + if info.Version != "" { + query.Set("version", info.Version) + } + previewURL := fmt.Sprintf("%s/space/api/box/stream/download/preview_sub/%s?%s", BaseURL, obj.ObjToken, query.Encode()) + + headers := http.Header{ + "Referer": []string{DoubaoURL + "/"}, + "User-Agent": []string{base.UserAgent}, + "Authorization": []string{auth}, + "Dpop": []string{dpop}, + } + + return &model.Link{ + URL: previewURL, + Header: headers, + }, nil +} + +func (d *DoubaoNew) createShare(ctx context.Context, obj *Object) error { + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/suite/permission/public/update.v5/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "external_access_entity": 1, + "link_share_entity": 4, + "token": obj.ObjToken, + "type": obj.ObjType, + }) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/suite/permission/public/update.v5/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + if err := decodeBaseResp(body, res); err != nil { + return err + } + return nil +} + +func (d *DoubaoNew) createFolder(ctx context.Context, parentToken, name string) (Node, error) { + data := url.Values{} + data.Set("name", name) + data.Set("source", "0") + if parentToken != "" { + data.Set("parent_token", parentToken) + } + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/explorer/v2/create/folder/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/create/folder/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return Node{}, err + } + if err := decodeBaseResp(body, res); err != nil { + return Node{}, err + } + + var resp CreateFolderResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return Node{}, fmt.Errorf("%s", msg) + } + + var node Node + if len(resp.Data.NodeList) > 0 { + if n, ok := resp.Data.Entities.Nodes[resp.Data.NodeList[0]]; ok { + node = n + } + } + if node.Token == "" { + for _, n := range resp.Data.Entities.Nodes { + node = n + break + } + } + if node.Token == "" && node.ObjToken == "" && node.NodeToken == "" { + return Node{}, fmt.Errorf("[doubao_new] create folder failed: empty response") + } + if node.NodeToken == "" { + if node.Token != "" { + node.NodeToken = node.Token + } else if node.ObjToken != "" { + node.NodeToken = node.ObjToken + } + } + if node.ObjToken == "" && node.Token != "" { + node.ObjToken = node.Token + } + return node, nil +} + +func (d *DoubaoNew) renameFolder(ctx context.Context, token, name string) error { + if token == "" { + return fmt.Errorf("[doubao_new] rename folder missing token") + } + data := url.Values{} + data.Set("token", token) + data.Set("name", name) + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/explorer/v2/rename/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/rename/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func isCsrfTokenError(body []byte, res *resty.Response) bool { + if len(body) == 0 { + return false + } + if strings.Contains(strings.ToLower(string(body)), "csrf token error") { + return true + } + if res != nil && res.StatusCode() == http.StatusForbidden { + return true + } + return false +} + +func doRequestWithCsrf(doRequest func(csrfToken string) (*resty.Response, []byte, error)) (*resty.Response, []byte, error) { + res, body, err := doRequest("") + if err != nil { + return res, body, err + } + if isCsrfTokenError(body, res) { + csrfToken := extractCsrfTokenFromResponse(res) + if csrfToken != "" { + return doRequest(csrfToken) + } + } + return res, body, err +} + +func extractCsrfTokenFromResponse(res *resty.Response) string { + if res == nil || res.Request == nil { + return "" + } + if res.Request.RawRequest != nil { + if csrf := cookie.GetStr(res.Request.RawRequest.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + } + if csrf := cookie.GetStr(res.Request.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + for _, c := range res.Cookies() { + if c.Name == "_csrf_token" { + return c.Value + } + } + return "" +} + +func decodeBaseResp(body []byte, res *resty.Response) error { + var common BaseResp + if err := json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf("%s", msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + return nil +} + +func (d *DoubaoNew) renameFile(ctx context.Context, fileToken, name string) error { + if fileToken == "" { + return fmt.Errorf("[doubao_new] rename file missing file token") + } + _, err := d.request(ctx, "/space/api/box/file/update_info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "file_token": fileToken, + "name": name, + }) + }, nil) + return err +} + +func (d *DoubaoNew) moveObj(ctx context.Context, srcToken, destToken string) error { + if srcToken == "" { + return fmt.Errorf("[doubao_new] move missing src token") + } + data := url.Values{} + data.Set("src_token", srcToken) + if destToken != "" { + data.Set("dest_token", destToken) + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/explorer/v2/move/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/move/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func (d *DoubaoNew) removeObj(ctx context.Context, tokens []string) error { + if len(tokens) == 0 { + return fmt.Errorf("[doubao_new] remove missing tokens") + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/explorer/v3/remove/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "tokens": tokens, + "apply": 1, + }) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v3/remove/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + var resp RemoveResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf("%s", msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + if resp.Data.TaskID == "" { + return nil + } + return d.waitTask(ctx, resp.Data.TaskID) +} + +func (d *DoubaoNew) getUserStorage(ctx context.Context) (UserStorageData, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + req.SetHeader("agw-js-conv", "str") + req.SetHeader("content-type", "application/json") + if err := d.applyAuthHeaders(req, http.MethodPost, DoubaoURL+"/alice/aispace/facade/get_user_storage"); err != nil { + return UserStorageData{}, err + } + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + } + req.SetBody(base.Json{}) + + res, err := req.Execute(http.MethodPost, DoubaoURL+"/alice/aispace/facade/get_user_storage") + if err != nil { + return UserStorageData{}, err + } + + body := res.Body() + var resp UserStorageResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UserStorageData{}, fmt.Errorf("%s", msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UserStorageData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) waitTask(ctx context.Context, taskID string) error { + const ( + taskPollInterval = time.Second + taskPollMaxAttempts = 120 + ) + var lastErr error + for attempt := 0; attempt < taskPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, taskPollInterval); err != nil { + return err + } + } + status, err := d.getTaskStatus(ctx, taskID) + if err != nil { + lastErr = err + continue + } + if status.IsFail { + return fmt.Errorf("[doubao_new] remove task failed: %s", taskID) + } + if status.IsFinish { + return nil + } + } + if lastErr != nil { + return lastErr + } + return fmt.Errorf("[doubao_new] remove task timed out: %s", taskID) +} + +func (d *DoubaoNew) getTaskStatus(ctx context.Context, taskID string) (TaskStatusData, error) { + if taskID == "" { + return TaskStatusData{}, fmt.Errorf("[doubao_new] task status missing task_id") + } + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodGet, BaseURL+"/space/api/explorer/v2/task/"); err != nil { + return TaskStatusData{}, err + } + req.SetQueryParam("task_id", taskID) + res, err := req.Execute(http.MethodGet, BaseURL+"/space/api/explorer/v2/task/") + if err != nil { + return TaskStatusData{}, err + } + body := res.Body() + var resp TaskStatusResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return TaskStatusData{}, fmt.Errorf("%s", msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return TaskStatusData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + return resp.Data, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} From d598ef756928872257f3f108c999fb3982a0a63e Mon Sep 17 00:00:00 2001 From: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> Date: Wed, 25 Mar 2026 17:45:37 +0800 Subject: [PATCH 23/86] refactor(db)!: replace SQLite Driver with glebarez/sqlite to avoid CGO (#2211) * mod(db): driver/sqlite ->glebarez/sqlite * mod(db): driver/sqlite ->glebarez/sqlite * [WIP] Refactor SQLite Driver with glebarez/sqlite to avoid CGO (#2213) * Initial plan * fix: address review comments - fix import order, update test, remove CGO sqlite deps Co-authored-by: PIKACHUIM <40362270+PIKACHUIM@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: PIKACHUIM <40362270+PIKACHUIM@users.noreply.github.com> --------- Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> --- go.mod | 15 +- go.sum | 318 ++++++++---------------------------- internal/bootstrap/db.go | 2 +- internal/op/storage_test.go | 2 +- 4 files changed, 77 insertions(+), 260 deletions(-) diff --git a/go.mod b/go.mod index c36ac1ca0..6f2248015 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.4 github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 + github.com/glebarez/sqlite v1.11.0 github.com/go-resty/resty/v2 v2.16.5 github.com/go-webauthn/webauthn v0.13.4 github.com/golang-jwt/jwt/v4 v4.5.2 @@ -84,13 +85,12 @@ require ( gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.9 - gorm.io/driver/sqlite v1.5.6 - gorm.io/gorm v1.25.11 + gorm.io/gorm v1.30.0 ) require ( - cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/BurntSushi/toml v1.6.0 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect @@ -104,6 +104,7 @@ require ( github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cronokirby/saferith v0.33.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.8.4 // indirect github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect @@ -121,6 +122,7 @@ require ( github.com/minio/minlz v1.0.0 // indirect github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -203,6 +205,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect + github.com/glebarez/go-sqlite v1.22.0 // indirect github.com/go-chi/chi/v5 v5.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -243,7 +246,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -271,6 +273,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rfjakob/eme v1.1.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect @@ -299,6 +302,10 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect + modernc.org/libc v1.55.3 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.8.0 // indirect + modernc.org/sqlite v1.33.1 // indirect ) replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0 diff --git a/go.sum b/go.sum index b9a4570bd..bb5c2e47b 100644 --- a/go.sum +++ b/go.sum @@ -1,28 +1,10 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go/auth v0.16.2 h1:QvBAGFPLrDeoiNjyfVunhQ10HKNYuOwZ5noee0M5df4= -cloud.google.com/go/auth v0.16.2/go.mod h1:sRBas2Y1fB1vZTdurouM0AzuYQBMZinrUYL8EufhtEA= +cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= +cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0= +cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= -cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= @@ -35,9 +17,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZY github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= @@ -205,7 +186,6 @@ github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCN github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= @@ -226,12 +206,8 @@ github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQ github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= @@ -270,6 +246,8 @@ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cn github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= @@ -278,8 +256,6 @@ github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7 github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 h1:q1b+gv6AG2TDPN+f0QAkbRrAvJ3ZosnwRLTKNxSXlaA= @@ -305,12 +281,14 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= +github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= +github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= +github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= @@ -348,32 +326,14 @@ github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXe github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -381,22 +341,17 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= +github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= -github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= -github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= +github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= @@ -419,8 +374,6 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= @@ -432,7 +385,6 @@ github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0 github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= @@ -478,8 +430,6 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg= @@ -498,11 +448,8 @@ github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -535,8 +482,6 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.32.0 h1:cWcycpONSH3VLTZ5npUl1O5aXPkNM0vUx6bywnYqGbE= github.com/meilisearch/meilisearch-go v0.32.0/go.mod h1:aNtyuwurDg/ggxQIcKqWH6G9g2ptc8GyY7PLY4zMn/g= github.com/mholt/archives v0.1.3 h1:aEAaOtNra78G+TvV5ohmXrJOAzf++dIlYeDW3N9q458= @@ -592,6 +537,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncw/swift/v2 v2.0.4 h1:hHWVFxn5/YaTWAASmn4qyq2p6OyP/Hm3vMLzkjEqR7w= github.com/ncw/swift/v2 v2.0.4/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.1.1 h1:OJaYalXdliBUXPmC8CZGQ7oZDxzX1/5mQmgn0/GASew= @@ -621,7 +568,6 @@ github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= @@ -636,16 +582,16 @@ github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE= github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo= github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= @@ -729,30 +675,24 @@ github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 h1:PSRw github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3/go.mod h1:CKriYB8bkNgSbYUQF1khSpejKb5IsV6cR7MdaAR7Fc0= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= -go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= -go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -760,9 +700,6 @@ golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -771,63 +708,23 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -839,48 +736,22 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -900,8 +771,6 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= @@ -914,13 +783,9 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -932,111 +797,42 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.236.0 h1:CAiEiDVtO4D/Qja2IA9VzlFrgPnK3XVMmRoJZlSWbc0= -google.golang.org/api v0.236.0/go.mod h1:X1WF9CU2oTc+Jml1tiIxGmWFK/UZezdqEu09gcxZAj4= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ= +google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= @@ -1056,22 +852,36 @@ gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= -gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= -gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= -gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= +gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= +modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= +modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= +modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= +modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= +modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= +modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= +modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= +modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= +modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= +modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= +modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= +modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index d97cb6796..e4b81bf40 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -9,10 +9,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/glebarez/sqlite" log "github.com/sirupsen/logrus" "gorm.io/driver/mysql" "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" "gorm.io/gorm/schema" diff --git a/internal/op/storage_test.go b/internal/op/storage_test.go index 2b191bd56..d7db25040 100644 --- a/internal/op/storage_test.go +++ b/internal/op/storage_test.go @@ -10,7 +10,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" mapset "github.com/deckarep/golang-set/v2" - "gorm.io/driver/sqlite" + "github.com/glebarez/sqlite" "gorm.io/gorm" ) From d85f084acb69b23221dd0ad948bb4354f103f00f Mon Sep 17 00:00:00 2001 From: Jealous Date: Thu, 26 Mar 2026 14:42:35 +0800 Subject: [PATCH 24/86] feat(permissions): implement fine-grained permission control (#2145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(permission): rename permission check functions for clarity - User.CanWrite() → User.CanCreateFilesOrFolders() - common.CanWrite() → common.CanWriteContentBypassUserPerms() - common.IsApply() → common.MetaCoversPath() Improves code readability by making function names more descriptive. The new MetaCoversPath name clearly indicates it checks if a meta rule covers a specific path. It better conveys that it's a query function rather than an action, and the applyToSubFolder parameter is more explicit than applySub. Also adds comprehensive test coverage: - 10 tests for MetaCoversPath core logic - 6 tests for CanWriteContent UserPerms - 7 tests for getReadme - 5 tests for getHeader - 6 tests for isEncrypt - 9 tests for whetherHide Total: 43 test scenarios covering all path matching and permission inheritance logic. Tests verify both normal behavior and bug fixes for Readme/Header information leakage and write permission bypass. Co-Authored-By: Claude Sonnet 4.5 * feat(permission): implement fine-grained user permissions for read/write operations Add per-user read and write permission controls at the meta level to enable more granular access control beyond the existing permission flags. Key changes: - Add ReadUsers/WriteUsers fields to Meta model with sub-directory inheritance flags - Implement CanRead and CanWrite permission check functions in server/common - Filter file list results based on user read permissions - Add permission checks across all file operations (FTP, HTTP handlers, WebDAV) - Simplify error handling pattern for MetaNotFound errors throughout codebase This allows administrators to restrict specific users from accessing or modifying certain paths, providing finer control over file system permissions. Note: Batch and recursive operations (FsMove, FsCopy, FsRemove, FsRecursiveMove, FsBatchRename, FsRegexRename) currently check parent directory permissions only. Individual item permission checks are not performed for performance reasons. Co-Authored-By: Claude Sonnet 4.5 * test(permission): add comprehensive tests for CanRead, CanWrite, and combined permission checks Add TestCanRead, TestCanWrite, TestCanAccessWithReadPermissions, and TestWritePermissionCombinations to validate the three-layer permission system including nil user/meta, sub-path inheritance, user whitelists, and root-level restrictions. Co-Authored-By: Claude Sonnet 4.6 * fix(webdav): use safe type assertion for MetaPassKey to prevent panic Bearer-token and OPTIONS auth paths do not set MetaPassKey in context, causing a panic when handlers perform a forced type assertion on nil. Co-Authored-By: Claude Sonnet 4.6 * fix(permission): treat nil user as system context in CanRead/CanWrite Previously, CanRead/CanWrite returned false for nil user, causing filterReadableObjs to return an empty list when fs.List is called from internal contexts without a user (e.g. context.Background()). A nil user represents an internal/system call and should bypass per-user restrictions, consistent with how whetherHide already handles nil user. Co-Authored-By: Claude Sonnet 4.6 * fix(fsmanage): prevent path traversal in FsRemove The previous check only skipped names that resolved to "/", but did not prevent traversal to sibling directories (e.g. "../secret"), which could bypass the CanWrite permission check that is only applied to req.Dir. Replace with a post-join prefix check to ensure each resolved path stays within reqPath. Co-Authored-By: Claude Sonnet 4.6 * fix(webdav): align MetaPassKey behavior with FTP auth logic For guest users, the WebDAV password input serves as the meta folder password (consistent with FTP anonymous/guest handling). For authenticated users, MetaPassKey is set to empty string since their login password is not the meta folder password. Co-Authored-By: Claude Sonnet 4.6 * fix(permission): require write auth for fs list refresh * refactor(permission): use MetaCoversPath in CanRead/CanWrite for consistency Replace inline `(Sub || meta.Path == path)` logic with MetaCoversPath, consistent with CanWriteContentBypassUserPerms. Also fix a copy-paste error in the CanWrite comment (read → write). Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.5 Co-authored-by: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> --- internal/fs/list.go | 31 +- internal/fs/list_test.go | 151 +++++ internal/model/meta.go | 28 +- internal/model/user.go | 6 +- server/common/check.go | 42 +- server/common/check_test.go | 982 ++++++++++++++++++++++++++++- server/ftp/fsmanage.go | 42 +- server/ftp/fsread.go | 18 +- server/ftp/fsup.go | 18 +- server/handles/archive.go | 25 +- server/handles/fsbatch.go | 54 +- server/handles/fsmanage.go | 124 +++- server/handles/fsread.go | 73 +-- server/handles/fsread_test.go | 255 ++++++++ server/handles/offline_download.go | 11 + server/middlewares/down.go | 8 +- server/middlewares/fsup.go | 21 +- server/webdav.go | 13 +- server/webdav/file.go | 35 + server/webdav/webdav.go | 103 ++- 20 files changed, 1850 insertions(+), 190 deletions(-) create mode 100644 internal/fs/list_test.go create mode 100644 server/handles/fsread_test.go diff --git a/internal/fs/list.go b/internal/fs/list.go index 1f92c7d46..113ba8231 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -2,13 +2,14 @@ package fs import ( "context" - "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "path" ) // List files @@ -43,7 +44,29 @@ func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) om.InitHideReg(meta.Hide) } objs := om.Merge(_objs, virtualFiles...) - return objs, nil + objs, err = filterReadableObjs(objs, user, path, meta) + return objs, err +} + +func filterReadableObjs(objs []model.Obj, user *model.User, reqPath string, parentMeta *model.Meta) ([]model.Obj, error) { + var result []model.Obj + for _, obj := range objs { + var meta *model.Meta + objPath := path.Join(reqPath, obj.GetName()) + if obj.IsDir() { + var err error + meta, err = op.GetNearestMeta(objPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return result, err + } + } else { + meta = parentMeta + } + if common.CanRead(user, meta, objPath) { + result = append(result, obj) + } + } + return result, nil } func whetherHide(user *model.User, meta *model.Meta, path string) bool { @@ -60,7 +83,7 @@ func whetherHide(user *model.User, meta *model.Meta, path string) bool { return false } // if meta doesn't apply to sub_folder, don't hide - if !utils.PathEqual(meta.Path, path) && !meta.HSub { + if !common.MetaCoversPath(meta.Path, path, meta.HSub) { return false } // if is guest, hide diff --git a/internal/fs/list_test.go b/internal/fs/list_test.go new file mode 100644 index 000000000..ebaf4371e --- /dev/null +++ b/internal/fs/list_test.go @@ -0,0 +1,151 @@ +package fs + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestWhetherHide(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil user", + user: nil, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/folder", + want: false, + reason: "nil user (treated as admin) should not hide", + }, + { + name: "user with can_see_hides permission", + user: &model.User{ + Role: model.GENERAL, + Permission: 1, // bit 0 set = can see hides + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/folder", + want: false, + reason: "user with can_see_hides permission should not hide", + }, + { + name: "nil meta", + user: &model.User{ + Role: model.GUEST, + }, + meta: nil, + path: "/folder", + want: false, + reason: "nil meta should not hide", + }, + { + name: "empty hide string", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "", + HSub: true, + }, + path: "/folder", + want: false, + reason: "empty hide string should not hide", + }, + { + name: "exact path match with HSub=false", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: false, + }, + path: "/folder", + want: true, + reason: "exact path match should hide for guest", + }, + { + name: "sub path with HSub=true", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/folder/subfolder", + want: true, + reason: "sub path with HSub=true should hide for guest", + }, + { + name: "sub path with HSub=false", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: false, + }, + path: "/folder/subfolder", + want: false, + reason: "sub path with HSub=false should not hide", + }, + { + name: "non-sub path with HSub=true", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/other", + want: false, + reason: "non-sub path should not hide even with HSub=true", + }, + { + name: "user without can_see_hides permission", + user: &model.User{ + Role: model.GENERAL, + Permission: 0, // bit 0 not set = cannot see hides + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/folder", + want: true, + reason: "user without can_see_hides permission should hide", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := whetherHide(tt.user, tt.meta, tt.path) + if got != tt.want { + t.Errorf("whetherHide() = %v, want %v\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} diff --git a/internal/model/meta.go b/internal/model/meta.go index 0446137a2..a105f38c6 100644 --- a/internal/model/meta.go +++ b/internal/model/meta.go @@ -1,16 +1,20 @@ package model type Meta struct { - ID uint `json:"id" gorm:"primaryKey"` - Path string `json:"path" gorm:"unique" binding:"required"` - Password string `json:"password"` - PSub bool `json:"p_sub"` - Write bool `json:"write"` - WSub bool `json:"w_sub"` - Hide string `json:"hide"` - HSub bool `json:"h_sub"` - Readme string `json:"readme"` - RSub bool `json:"r_sub"` - Header string `json:"header"` - HeaderSub bool `json:"header_sub"` + ID uint `json:"id" gorm:"primaryKey"` + Path string `json:"path" gorm:"unique" binding:"required"` + ReadUsers []uint `json:"read_users" gorm:"serializer:json"` + ReadUsersSub bool `json:"read_users_sub"` + WriteUsers []uint `json:"write_users" gorm:"serializer:json"` + WriteUsersSub bool `json:"write_users_sub"` + Password string `json:"password"` + PSub bool `json:"p_sub"` + Write bool `json:"write"` + WSub bool `json:"w_sub"` + Hide string `json:"hide"` + HSub bool `json:"h_sub"` + Readme string `json:"readme"` + RSub bool `json:"r_sub"` + Header string `json:"header"` + HeaderSub bool `json:"header_sub"` } diff --git a/internal/model/user.go b/internal/model/user.go index 3bad4ebb9..61252ce95 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -123,12 +123,12 @@ func (u *User) CanAddOfflineDownloadTasks() bool { return CanAddOfflineDownloadTasks(u.Permission) } -func CanWrite(permission int32) bool { +func CanWriteContent(permission int32) bool { return (permission>>3)&1 == 1 } -func (u *User) CanWrite() bool { - return CanWrite(u.Permission) +func (u *User) CanWriteContent() bool { + return CanWriteContent(u.Permission) } func CanRename(permission int32) bool { diff --git a/server/common/check.go b/server/common/check.go index 90074aeeb..27be3103c 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -2,6 +2,7 @@ package common import ( "path" + "slices" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -17,24 +18,39 @@ func IsStorageSignEnabled(rawPath string) bool { return storage != nil && storage.GetStorage().EnableSign } -func CanWrite(meta *model.Meta, path string) bool { - if meta == nil || !meta.Write { +func CanRead(user *model.User, meta *model.Meta, path string) bool { + // nil user is treated as internal/system context and bypasses per-user read restrictions + if user == nil { + return true + } + if meta != nil && len(meta.ReadUsers) > 0 && !slices.Contains(meta.ReadUsers, user.ID) && MetaCoversPath(meta.Path, path, meta.ReadUsersSub) { return false } - return meta.WSub || meta.Path == path + return true } -func IsApply(metaPath, reqPath string, applySub bool) bool { - if utils.PathEqual(metaPath, reqPath) { +func CanWrite(user *model.User, meta *model.Meta, path string) bool { + // nil user is treated as internal/system context and bypasses per-user write restrictions + if user == nil { return true } - return utils.IsSubPath(metaPath, reqPath) && applySub + if meta != nil && len(meta.WriteUsers) > 0 && !slices.Contains(meta.WriteUsers, user.ID) && MetaCoversPath(meta.Path, path, meta.WriteUsersSub) { + return false + } + return true +} + +func CanWriteContentBypassUserPerms(meta *model.Meta, path string) bool { + if meta == nil || !meta.Write { + return false + } + return MetaCoversPath(meta.Path, path, meta.WSub) } func CanAccess(user *model.User, meta *model.Meta, reqPath string, password string) bool { // if the reqPath is in hide (only can check the nearest meta) and user can't see hides, can't access if meta != nil && !user.CanSeeHides() && meta.Hide != "" && - IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path + MetaCoversPath(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path for _, hide := range strings.Split(meta.Hide, "\n") { re := regexp2.MustCompile(hide, regexp2.None) if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { @@ -42,6 +58,9 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri } } } + if !CanRead(user, meta, reqPath) { + return false + } // if is not guest and can access without password if user.CanAccessWithoutPassword() { return true @@ -51,13 +70,20 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri return true } // if meta doesn't apply to sub_folder, can access - if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { + if !MetaCoversPath(meta.Path, reqPath, meta.PSub) { return true } // validate password return meta.Password == password } +func MetaCoversPath(metaPath, reqPath string, applyToSubFolder bool) bool { + if utils.PathEqual(metaPath, reqPath) { + return true + } + return utils.IsSubPath(metaPath, reqPath) && applyToSubFolder +} + // ShouldProxy TODO need optimize // when should be proxy? // 1. config.MustProxy() diff --git a/server/common/check_test.go b/server/common/check_test.go index 33114603b..18abca8e9 100644 --- a/server/common/check_test.go +++ b/server/common/check_test.go @@ -1,24 +1,986 @@ package common -import "testing" +import ( + "testing" -func TestIsApply(t *testing.T) { - datas := []struct { + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestCoversPath(t *testing.T) { + tests := []struct { + name string metaPath string reqPath string applySub bool - result bool + want bool }{ { + name: "exact path match with applySub=false", + metaPath: "/folder", + reqPath: "/folder", + applySub: false, + want: true, + }, + { + name: "exact path match with applySub=true", + metaPath: "/folder", + reqPath: "/folder", + applySub: true, + want: true, + }, + { + name: "sub path with applySub=true", + metaPath: "/folder", + reqPath: "/folder/subfolder", + applySub: true, + want: true, + }, + { + name: "sub path with applySub=false", + metaPath: "/folder", + reqPath: "/folder/subfolder", + applySub: false, + want: false, + }, + { + name: "non-sub path with applySub=true", + metaPath: "/folder", + reqPath: "/other", + applySub: true, + want: false, + }, + { + name: "non-sub path with applySub=false", + metaPath: "/folder", + reqPath: "/other", + applySub: false, + want: false, + }, + { + name: "root path covers all with applySub=true", metaPath: "/", - reqPath: "/test", + reqPath: "/any/deep/path", + applySub: true, + want: true, + }, + { + name: "root path exact match", + metaPath: "/", + reqPath: "/", + applySub: false, + want: true, + }, + { + name: "deep sub path with applySub=true", + metaPath: "/folder", + reqPath: "/folder/sub1/sub2/file.txt", applySub: true, - result: true, + want: true, + }, + { + name: "sibling paths with applySub=true", + metaPath: "/folder1", + reqPath: "/folder2", + applySub: true, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := MetaCoversPath(tt.metaPath, tt.reqPath, tt.applySub) + if got != tt.want { + t.Errorf("MetaCoversPath(%q, %q, %v) = %v, want %v", + tt.metaPath, tt.reqPath, tt.applySub, got, tt.want) + } + }) + } +} + +func TestCanWriteContentIgnoringUserPerms(t *testing.T) { + tests := []struct { + name string + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil meta", + meta: nil, + path: "/any", + want: false, + reason: "nil meta should deny write", + }, + { + name: "meta.Write=false", + meta: &model.Meta{ + Path: "/folder", + Write: false, + }, + path: "/folder", + want: false, + reason: "Write=false should deny write", + }, + { + name: "exact path match with WSub=false", + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: false, + }, + path: "/folder", + want: true, + reason: "exact path match should allow write", + }, + { + name: "sub path with WSub=true", + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: true, + }, + path: "/folder/subfolder", + want: true, + reason: "sub path with WSub=true should allow write", + }, + { + name: "sub path with WSub=false (BEHAVIOR CHANGE)", + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: false, + }, + path: "/folder/subfolder", + want: false, + reason: "sub path with WSub=false should deny write (fixed bug)", + }, + { + name: "non-sub path with WSub=true", + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: true, + }, + path: "/other", + want: false, + reason: "non-sub path should deny write even with WSub=true", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CanWriteContentBypassUserPerms(tt.meta, tt.path) + if got != tt.want { + t.Errorf("CanWriteContentBypassUserPerms() = %v, want %v\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} + +func TestCanRead(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil user should allow access", + user: nil, + meta: nil, + path: "/any", + want: true, + reason: "nil user represents internal/system context and bypasses per-user read restrictions", + }, + { + name: "nil meta should allow access", + user: &model.User{ + ID: 1, + }, + meta: nil, + path: "/any", + want: true, + reason: "nil meta means no restrictions", + }, + { + name: "empty ReadUsers list should allow access", + user: &model.User{ + ID: 1, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{}, + }, + path: "/folder", + want: true, + reason: "empty ReadUsers means no user-level restrictions", + }, + { + name: "user in ReadUsers list with exact path match", + user: &model.User{ + ID: 1, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: false, + }, + path: "/folder", + want: true, + reason: "user ID 1 is in ReadUsers list", + }, + { + name: "user not in ReadUsers list with exact path match", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: false, + }, + path: "/folder", + want: false, + reason: "user ID 5 is not in ReadUsers list and path matches", + }, + { + name: "user not in ReadUsers list with ReadUsersSub=true for sub path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: true, + }, + path: "/folder/subfolder", + want: false, + reason: "user ID 5 is not in ReadUsers list and ReadUsersSub applies to sub paths", + }, + { + name: "user not in ReadUsers list with ReadUsersSub=false for sub path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: false, + }, + path: "/folder/subfolder", + want: true, + reason: "ReadUsersSub=false means restriction doesn't apply to sub paths", + }, + { + name: "user in ReadUsers list with ReadUsersSub=true for sub path", + user: &model.User{ + ID: 2, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: true, + }, + path: "/folder/subfolder/deep", + want: true, + reason: "user ID 2 is in ReadUsers list so can access sub paths", + }, + { + name: "user not in ReadUsers list for different path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: false, + }, + path: "/other", + want: true, + reason: "meta path doesn't match request path, so restriction doesn't apply", + }, + { + name: "root level restriction with ReadUsersSub=true", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: true, + }, + path: "/any/deep/path", + want: false, + reason: "root level restriction with ReadUsersSub affects all paths", }, } - for i, data := range datas { - if IsApply(data.metaPath, data.reqPath, data.applySub) != data.result { - t.Errorf("TestIsApply %d failed", i) - } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CanRead(tt.user, tt.meta, tt.path) + if got != tt.want { + t.Errorf("CanRead() = %v, want %v\nReason: %s\nUser ID: %v, Meta: %+v, Path: %s", + got, tt.want, tt.reason, getUserID(tt.user), tt.meta, tt.path) + } + }) + } +} + +func TestCanWrite(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil user should allow access", + user: nil, + meta: nil, + path: "/any", + want: true, + reason: "nil user represents internal/system context and bypasses per-user write restrictions", + }, + { + name: "nil meta should allow access", + user: &model.User{ + ID: 1, + }, + meta: nil, + path: "/any", + want: true, + reason: "nil meta means no restrictions", + }, + { + name: "empty WriteUsers list should allow access", + user: &model.User{ + ID: 1, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{}, + }, + path: "/folder", + want: true, + reason: "empty WriteUsers means no user-level restrictions", + }, + { + name: "user in WriteUsers list with exact path match", + user: &model.User{ + ID: 1, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "user ID 1 is in WriteUsers list", + }, + { + name: "user not in WriteUsers list with exact path match", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: false, + }, + path: "/folder", + want: false, + reason: "user ID 5 is not in WriteUsers list and path matches", + }, + { + name: "user not in WriteUsers list with WriteUsersSub=true for sub path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: true, + }, + path: "/folder/subfolder", + want: false, + reason: "user ID 5 is not in WriteUsers list and WriteUsersSub applies to sub paths", + }, + { + name: "user not in WriteUsers list with WriteUsersSub=false for sub path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: false, + }, + path: "/folder/subfolder", + want: true, + reason: "WriteUsersSub=false means restriction doesn't apply to sub paths", + }, + { + name: "user in WriteUsers list with WriteUsersSub=true for sub path", + user: &model.User{ + ID: 2, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: true, + }, + path: "/folder/subfolder/deep", + want: true, + reason: "user ID 2 is in WriteUsers list so can write to sub paths", + }, + { + name: "user not in WriteUsers list for different path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: false, + }, + path: "/other", + want: true, + reason: "meta path doesn't match request path, so restriction doesn't apply", + }, + { + name: "multiple users with mixed permissions", + user: &model.User{ + ID: 10, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 5, 10, 15}, + WriteUsersSub: true, + }, + path: "/folder/file.txt", + want: true, + reason: "user ID 10 is in WriteUsers list", + }, + { + name: "write restriction at root level", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/", + WriteUsers: []uint{1}, + WriteUsersSub: true, + }, + path: "/any/path", + want: false, + reason: "only user ID 1 can write when root has WriteUsers restriction", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CanWrite(tt.user, tt.meta, tt.path) + if got != tt.want { + t.Errorf("CanWrite() = %v, want %v\nReason: %s\nUser ID: %v, Meta: %+v, Path: %s", + got, tt.want, tt.reason, getUserID(tt.user), tt.meta, tt.path) + } + }) + } +} + +func TestCanAccessWithReadPermissions(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + reqPath string + password string + want bool + reason string + }{ + { + name: "user with read permission and correct password", + user: &model.User{ + ID: 1, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2}, + ReadUsersSub: true, + Password: "secret", + PSub: true, + }, + reqPath: "/folder/file.txt", + password: "secret", + want: true, + reason: "user in ReadUsers list with correct password", + }, + { + name: "user without read permission even with correct password", + user: &model.User{ + ID: 5, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2}, + ReadUsersSub: true, + Password: "secret", + PSub: true, + }, + reqPath: "/folder/file.txt", + password: "secret", + want: false, + reason: "user not in ReadUsers list, should be denied before password check", + }, + { + name: "user with read permission but wrong password", + user: &model.User{ + ID: 1, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2}, + ReadUsersSub: true, + Password: "secret", + PSub: true, + }, + reqPath: "/folder/file.txt", + password: "wrong", + want: false, + reason: "user in ReadUsers list but wrong password", + }, + { + name: "user without read permission and no password", + user: &model.User{ + ID: 5, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2}, + ReadUsersSub: true, + }, + reqPath: "/folder/file.txt", + password: "", + want: false, + reason: "user not in ReadUsers list should be denied", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CanAccess(tt.user, tt.meta, tt.reqPath, tt.password) + if got != tt.want { + t.Errorf("CanAccess() = %v, want %v\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} + +// Helper function to safely get user ID +func getUserID(user *model.User) uint { + if user == nil { + return 0 + } + return user.ID +} + +// TestWritePermissionCombinations tests the combined permission check logic +// that is actually used in the codebase: +// +// if !user.CanWriteContent() && !CanWriteContentBypassUserPerms(meta, path) { +// deny +// } +// if !CanWrite(user, meta, path) { +// deny +// } +// +// This ensures the three-layer permission system works correctly: +// 1. User-level global write permission (CanWriteContent) +// 2. Meta-level global write permission (CanWriteContentBypassUserPerms) +// 3. Meta-level user whitelist (CanWrite) +func TestWritePermissionCombinations(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + path string + want bool + reason string + checkFirstLayer bool // whether first layer should pass + checkSecondLayer bool // whether second layer should pass + expectedDenyReason string + }{ + // === Scenario 1: User has global write permission === + { + name: "user has CanWriteContent + in WriteUsers whitelist", + user: &model.User{ + ID: 1, + Permission: 1 << 3, // CanWriteContent = true + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, + WriteUsers: []uint{1}, + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "user has global write permission AND is in whitelist", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "user has CanWriteContent but NOT in WriteUsers whitelist", + user: &model.User{ + ID: 1, + Permission: 1 << 3, // CanWriteContent = true + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, + WriteUsers: []uint{2, 3}, // user 1 not in list + WriteUsersSub: false, + }, + path: "/folder", + want: false, + reason: "even with global write permission, must pass whitelist check", + checkFirstLayer: true, + checkSecondLayer: false, + expectedDenyReason: "whitelist check failed", + }, + + // === Scenario 2: User lacks global permission but meta.Write=true === + { + name: "no CanWriteContent + meta.Write=true + in WriteUsers", + user: &model.User{ + ID: 1, + Permission: 0, // CanWriteContent = false + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, // bypass enabled + WSub: false, + WriteUsers: []uint{1}, + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "meta.Write bypasses user permission check, and user is in whitelist", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "no CanWriteContent + meta.Write=true + NOT in WriteUsers (KEY TEST)", + user: &model.User{ + ID: 5, + Permission: 0, // CanWriteContent = false + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, // bypass enabled + WSub: false, + WriteUsers: []uint{1, 2, 3}, // user 5 not in list + WriteUsersSub: false, + }, + path: "/folder", + want: false, + reason: "CRITICAL: meta.Write cannot bypass whitelist check (new behavior)", + checkFirstLayer: true, + checkSecondLayer: false, + expectedDenyReason: "whitelist check failed even with meta.Write=true", + }, + + // === Scenario 3: Both checks fail === + { + name: "no CanWriteContent + meta.Write=false", + user: &model.User{ + ID: 1, + Permission: 0, // CanWriteContent = false + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, // no bypass + WriteUsers: []uint{1}, + WriteUsersSub: false, + }, + path: "/folder", + want: false, + reason: "denied at first layer: no global permission and no bypass", + checkFirstLayer: false, + checkSecondLayer: false, + expectedDenyReason: "first layer check failed", + }, + + // === Scenario 4: Empty WriteUsers (no whitelist restriction) === + { + name: "user has CanWriteContent + empty WriteUsers", + user: &model.User{ + ID: 1, + Permission: 1 << 3, // CanWriteContent = true + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, + WriteUsers: []uint{}, // empty = no restriction + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "empty WriteUsers means no whitelist restriction", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "no CanWriteContent + meta.Write=true + empty WriteUsers", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: false, + WriteUsers: []uint{}, // empty = no restriction + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "meta.Write bypasses first check, empty whitelist passes second", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + + // === Scenario 5: Nil meta (no restrictions) === + { + name: "user has CanWriteContent + nil meta", + user: &model.User{ + ID: 1, + Permission: 1 << 3, + }, + meta: nil, + path: "/folder", + want: true, + reason: "nil meta means no restrictions", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "no CanWriteContent + nil meta", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: nil, + path: "/folder", + want: false, + reason: "nil meta cannot bypass lack of user permission", + checkFirstLayer: false, + checkSecondLayer: true, // would pass if first layer passed + expectedDenyReason: "first layer check failed", + }, + + // === Scenario 6: Sub-directory inheritance === + { + name: "meta.Write with WSub=true for subdirectory", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: true, // applies to subdirectories + WriteUsers: []uint{1}, + WriteUsersSub: true, + }, + path: "/folder/subfolder", + want: true, + reason: "WSub=true applies meta.Write to subdirectories", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "meta.Write with WSub=false for subdirectory", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: false, // does NOT apply to subdirectories + WriteUsers: []uint{1}, + WriteUsersSub: false, + }, + path: "/folder/subfolder", + want: false, + reason: "WSub=false means meta.Write doesn't apply to subdirectories", + checkFirstLayer: false, + checkSecondLayer: true, + expectedDenyReason: "first layer check failed (WSub=false)", + }, + { + name: "WriteUsersSub=false for subdirectory bypasses whitelist", + user: &model.User{ + ID: 5, // not in WriteUsers + Permission: 1 << 3, + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, + WriteUsers: []uint{1, 2}, + WriteUsersSub: false, // whitelist does NOT apply to subdirectories + }, + path: "/folder/subfolder", + want: true, + reason: "WriteUsersSub=false means whitelist doesn't apply to subdirectories", + checkFirstLayer: true, + checkSecondLayer: true, // passes because restriction doesn't apply + expectedDenyReason: "", + }, + + // === Scenario 7: Root level restriction === + { + name: "root level meta.Write with user in whitelist", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/", + Write: true, + WSub: true, + WriteUsers: []uint{1}, + WriteUsersSub: true, + }, + path: "/any/deep/path", + want: true, + reason: "root level permissions apply to all paths", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "root level restriction denies non-whitelisted user", + user: &model.User{ + ID: 5, + Permission: 1 << 3, // has global permission + }, + meta: &model.Meta{ + Path: "/", + Write: false, + WriteUsers: []uint{1, 2}, + WriteUsersSub: true, + }, + path: "/any/path", + want: false, + reason: "root level whitelist restricts all paths", + checkFirstLayer: true, + checkSecondLayer: false, + expectedDenyReason: "not in root level whitelist", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Simulate the actual permission check logic + firstLayerPass := tt.user.CanWriteContent() || CanWriteContentBypassUserPerms(tt.meta, tt.path) + secondLayerPass := CanWrite(tt.user, tt.meta, tt.path) + + // Verify our understanding of each layer + if firstLayerPass != tt.checkFirstLayer { + t.Errorf("First layer check mismatch: got %v, expected %v\n"+ + "CanWriteContent()=%v, CanWriteContentBypassUserPerms()=%v", + firstLayerPass, tt.checkFirstLayer, + tt.user.CanWriteContent(), CanWriteContentBypassUserPerms(tt.meta, tt.path)) + } + + if firstLayerPass && secondLayerPass != tt.checkSecondLayer { + t.Errorf("Second layer check mismatch: got %v, expected %v\n"+ + "CanWrite()=%v", + secondLayerPass, tt.checkSecondLayer, + CanWrite(tt.user, tt.meta, tt.path)) + } + + // Final result + got := firstLayerPass && secondLayerPass + + if got != tt.want { + t.Errorf("Permission check failed:\n"+ + " Result: %v, want %v\n"+ + " Reason: %s\n"+ + " First layer (CanWriteContent || CanWriteContentBypassUserPerms): %v\n"+ + " Second layer (CanWrite): %v\n"+ + " User: ID=%d, Permission=%d, CanWriteContent=%v\n"+ + " Meta: Path=%s, Write=%v, WSub=%v, WriteUsers=%v, WriteUsersSub=%v\n"+ + " Check Path: %s", + got, tt.want, + tt.reason, + firstLayerPass, + secondLayerPass, + tt.user.ID, tt.user.Permission, tt.user.CanWriteContent(), + getMetaPath(tt.meta), getMetaWrite(tt.meta), getMetaWSub(tt.meta), + getMetaWriteUsers(tt.meta), getMetaWriteUsersSub(tt.meta), + tt.path) + } + }) + } +} + +// Helper functions to safely extract meta fields +func getMetaPath(meta *model.Meta) string { + if meta == nil { + return "nil" + } + return meta.Path +} + +func getMetaWrite(meta *model.Meta) bool { + if meta == nil { + return false + } + return meta.Write +} + +func getMetaWSub(meta *model.Meta) bool { + if meta == nil { + return false + } + return meta.WSub +} + +func getMetaWriteUsers(meta *model.Meta) []uint { + if meta == nil { + return nil + } + return meta.WriteUsers +} + +func getMetaWriteUsersSub(meta *model.Meta) bool { + if meta == nil { + return false } + return meta.WriteUsersSub } diff --git a/server/ftp/fsmanage.go b/server/ftp/fsmanage.go index 48f72794e..3e98d6d14 100644 --- a/server/ftp/fsmanage.go +++ b/server/ftp/fsmanage.go @@ -15,20 +15,23 @@ import ( func Mkdir(ctx context.Context, path string) error { user := ctx.Value(conf.UserKey).(*model.User) + if !user.CanFTPManage() { + return errs.PermissionDenied + } reqPath, err := user.JoinPath(path) if err != nil { return err } - if !user.CanWrite() || !user.CanFTPManage() { - meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return err - } - } - if !common.CanWrite(meta, reqPath) { - return errs.PermissionDenied - } + parentPath := stdpath.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + return errs.PermissionDenied + } + if !common.CanWrite(user, parentMeta, parentPath) { + return errs.PermissionDenied } return fs.MakeDir(ctx, reqPath) } @@ -42,6 +45,13 @@ func Remove(ctx context.Context, path string) error { if err != nil { return err } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + if !common.CanWrite(user, meta, reqPath) { + return errs.PermissionDenied + } if err = RemoveStage(reqPath); !errors.Is(err, errs.ObjectNotFound) { return err } @@ -60,8 +70,12 @@ func Rename(ctx context.Context, oldPath, newPath string) error { } srcDir, srcBase := stdpath.Split(srcPath) dstDir, dstBase := stdpath.Split(dstPath) + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } if srcDir == dstDir { - if !user.CanRename() || !user.CanFTPManage() { + if !user.CanRename() || !user.CanFTPManage() || !common.CanWrite(user, dstMeta, dstDir) { return errs.PermissionDenied } if err = MoveStage(srcPath, dstPath); !errors.Is(err, errs.ObjectNotFound) { @@ -69,7 +83,11 @@ func Rename(ctx context.Context, oldPath, newPath string) error { } return fs.Rename(ctx, srcPath, dstBase) } else { - if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) { + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + if !user.CanMove() || !user.CanFTPManage() || (srcBase != dstBase && !user.CanRename()) || !common.CanWrite(user, srcMeta, srcDir) || !common.CanWrite(user, dstMeta, dstDir) { return errs.PermissionDenied } if err = MoveStage(srcPath, dstPath); !errors.Is(err, errs.ObjectNotFound) { diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index 9080bae17..54a3de8f2 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -27,10 +27,8 @@ type FileDownloadProxy struct { func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownloadProxy, error) { user := ctx.Value(conf.UserKey).(*model.User) meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return nil, err - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err } ctx = context.WithValue(ctx, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, ctx.Value(conf.MetaPassKey).(string)) { @@ -121,10 +119,8 @@ func Stat(ctx context.Context, path string) (os.FileInfo, error) { return nil, err } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return nil, err - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err } ctx = context.WithValue(ctx, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, ctx.Value(conf.MetaPassKey).(string)) { @@ -147,10 +143,8 @@ func List(ctx context.Context, path string) ([]os.FileInfo, error) { return nil, err } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return nil, err - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err } ctx = context.WithValue(ctx, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, ctx.Value(conf.MetaPassKey).(string)) { diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index c549a1943..7a96a4f65 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -33,14 +33,18 @@ type FileUploadProxy struct { func uploadAuth(ctx context.Context, path string) error { user := ctx.Value(conf.UserKey).(*model.User) - meta, err := op.GetNearestMeta(stdpath.Dir(path)) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return err - } + if !user.CanFTPManage() { + return errs.PermissionDenied + } + parentPath := stdpath.Dir(path) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + return errs.PermissionDenied } - if !(common.CanAccess(user, meta, path, ctx.Value(conf.MetaPassKey).(string)) && - ((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) { + if !common.CanWrite(user, parentMeta, parentPath) { return errs.PermissionDenied } return nil diff --git a/server/handles/archive.go b/server/handles/archive.go index 4fd405688..d46f83c86 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -101,11 +101,9 @@ func FsArchiveMeta(c *gin.Context, req *ArchiveMetaReq, user *model.User) { return } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { @@ -186,11 +184,9 @@ func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { return } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { @@ -264,6 +260,15 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, err, 403) return } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, dstMeta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } tasks := make([]task.TaskExtensionInfo, 0, len(srcPaths)) for _, srcPath := range srcPaths { t, e := fs.ArchiveDecompress(c.Request.Context(), srcPath, dstDir, model.ArchiveDecompressArgs{ diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 162419f7b..28588d668 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -22,6 +22,7 @@ type RecursiveMoveReq struct { ConflictPolicy string `json:"conflict_policy"` } +// FsRecursiveMove recursively moves files (individual item permission checks skipped for performance). func FsRecursiveMove(c *gin.Context) { var req RecursiveMoveReq if err := c.ShouldBind(&req); err != nil { @@ -39,20 +40,31 @@ func FsRecursiveMove(c *gin.Context) { common.ErrorResp(c, err, 403) return } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, srcMeta, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + common.GinWithValue(c, conf.MetaKey, srcMeta) + dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - - meta, err := op.GetNearestMeta(srcDir) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, dstMeta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } - common.GinWithValue(c, conf.MetaKey, meta) rootFiles, err := fs.List(c.Request.Context(), srcDir, &fs.ListArgs{}) if err != nil { @@ -143,6 +155,7 @@ type BatchRenameReq struct { } `json:"rename_objects"` } +// FsBatchRename performs batch rename (individual item permission checks skipped for performance). func FsBatchRename(c *gin.Context) { var req BatchRenameReq if err := c.ShouldBind(&req); err != nil { @@ -162,11 +175,13 @@ func FsBatchRename(c *gin.Context) { } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } common.GinWithValue(c, conf.MetaKey, meta) for _, renameObject := range req.RenameObjects { @@ -193,6 +208,7 @@ type RegexRenameReq struct { NewNameRegex string `json:"new_name_regex"` } +// FsRegexRename renames files by regex (individual item permission checks skipped for performance). func FsRegexRename(c *gin.Context) { var req RegexRenameReq if err := c.ShouldBind(&req); err != nil { @@ -212,11 +228,13 @@ func FsRegexRename(c *gin.Context) { } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } common.GinWithValue(c, conf.MetaKey, meta) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 62382a27c..9ead8d60a 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -36,18 +36,19 @@ func FsMkdir(c *gin.Context) { common.ErrorResp(c, err, 403) return } - if !user.CanWrite() { - meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } - } - if !common.CanWrite(meta, reqPath) { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } + parentPath := stdpath.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if !common.CanWrite(user, parentMeta, parentPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } if err := fs.MakeDir(c.Request.Context(), reqPath); err != nil { common.ErrorResp(c, err, 500) @@ -65,6 +66,7 @@ type MoveCopyReq struct { Merge bool `json:"merge"` } +// FsMove performs batch move (individual item permission checks skipped for performance). func FsMove(c *gin.Context) { var req MoveCopyReq if err := c.ShouldBind(&req); err != nil { @@ -80,11 +82,34 @@ func FsMove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } + srcDir, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, srcMeta, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, dstMeta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } validPaths := make([]string, 0, len(req.Names)) for _, name := range req.Names { @@ -140,6 +165,7 @@ func FsMove(c *gin.Context) { } } +// FsCopy performs batch copy (individual item permission checks skipped for performance). func FsCopy(c *gin.Context) { var req MoveCopyReq if err := c.ShouldBind(&req); err != nil { @@ -155,11 +181,34 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } + srcDir, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanRead(user, srcMeta, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, dstMeta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } validPaths := make([]string, 0, len(req.Names)) for _, name := range req.Names { @@ -245,6 +294,16 @@ func FsRename(c *gin.Context) { common.ErrorResp(c, err, 403) return } + parentPath := stdpath.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, parentMeta, parentPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) if dstPath != reqPath { @@ -273,6 +332,7 @@ type RemoveReq struct { Names []string `json:"names"` } +// FsRemove performs batch remove (individual item permission checks skipped for performance). func FsRemove(c *gin.Context) { var req RemoveReq if err := c.ShouldBind(&req); err != nil { @@ -288,19 +348,28 @@ func FsRemove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } + reqPath, err := user.JoinPath(req.Dir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } for i, name := range req.Names { - if strings.TrimSpace(utils.FixAndCleanPath(name)) == "/" { - log.Warnf("FsRemove: invalid item skipped: %s (parent directory: %s)\n", name, req.Dir) + fullPath := stdpath.Join(reqPath, name) + if !strings.HasPrefix(fullPath+"/", reqPath+"/") { + log.Warnf("FsRemove: path traversal attempt skipped: %s (dir: %s)\n", name, req.Dir) req.Names[i] = "" continue } - // ensure req.Names is not a relative path - var err error - req.Names[i], err = user.JoinPath(stdpath.Join(req.Dir, name)) - if err != nil { - common.ErrorResp(c, err, 403) - return - } + req.Names[i] = fullPath } for _, path := range req.Names { if path == "" { @@ -320,6 +389,7 @@ type RemoveEmptyDirectoryReq struct { SrcDir string `json:"src_dir"` } +// FsRemoveEmptyDirectory recursively removes empty directories (individual item permission checks skipped for performance). func FsRemoveEmptyDirectory(c *gin.Context) { var req RemoveEmptyDirectoryReq if err := c.ShouldBind(&req); err != nil { @@ -339,11 +409,13 @@ func FsRemoveEmptyDirectory(c *gin.Context) { } meta, err := op.GetNearestMeta(srcDir) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } common.GinWithValue(c, conf.MetaKey, meta) diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 886da9dc9..a90fc1082 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -47,13 +47,14 @@ type ObjResp struct { } type FsListResp struct { - Content []ObjResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` - DirectUploadTools []string `json:"direct_upload_tools,omitempty"` + Content []ObjResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + WriteContentBypass bool `json:"write_content_bypass"` + Provider string `json:"provider"` + DirectUploadTools []string `json:"direct_upload_tools,omitempty"` } func FsListSplit(c *gin.Context) { @@ -83,18 +84,17 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { return } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } - if !user.CanWrite() && !common.CanWrite(meta, reqPath) && req.Refresh { + canWriteContentAtPath := common.CanWrite(user, meta, reqPath) && (user.CanWriteContent() || common.CanWriteContentBypassUserPerms(meta, reqPath)) + if req.Refresh && !canWriteContentAtPath { common.ErrorStrResp(c, "Refresh without permission", 403) return } @@ -109,19 +109,20 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { total, objs := pagination(objs, &req.PageReq) provider := "unknown" var directUploadTools []string - if user.CanWrite() { + if canWriteContentAtPath { if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { directUploadTools = op.GetDirectUploadTools(storage) } } common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), - Total: int64(total), - Readme: getReadme(meta, reqPath), - Header: getHeader(meta, reqPath), - Write: user.CanWrite() || common.CanWrite(meta, reqPath), - Provider: provider, - DirectUploadTools: directUploadTools, + Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), + Total: int64(total), + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Write: common.CanWrite(user, meta, reqPath), + WriteContentBypass: common.CanWriteContentBypassUserPerms(meta, reqPath), + Provider: provider, + DirectUploadTools: directUploadTools, }) } @@ -147,11 +148,9 @@ func FsDirs(c *gin.Context) { reqPath = tmp } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { @@ -186,14 +185,14 @@ func filterDirs(objs []model.Obj) []DirResp { } func getReadme(meta *model.Meta, path string) string { - if meta != nil && (utils.PathEqual(meta.Path, path) || meta.RSub) { + if meta != nil && common.MetaCoversPath(meta.Path, path, meta.RSub) { return meta.Readme } return "" } func getHeader(meta *model.Meta, path string) string { - if meta != nil && (utils.PathEqual(meta.Path, path) || meta.HeaderSub) { + if meta != nil && common.MetaCoversPath(meta.Path, path, meta.HeaderSub) { return meta.Header } return "" @@ -206,7 +205,7 @@ func isEncrypt(meta *model.Meta, path string) bool { if meta == nil || meta.Password == "" { return false } - if !utils.PathEqual(meta.Path, path) && !meta.PSub { + if !common.MetaCoversPath(meta.Path, path, meta.PSub) { return false } return true @@ -288,11 +287,9 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { return } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { @@ -414,11 +411,9 @@ func FsOther(c *gin.Context) { return } meta, err := op.GetNearestMeta(req.Path) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, req.Path, req.Password) { diff --git a/server/handles/fsread_test.go b/server/handles/fsread_test.go new file mode 100644 index 000000000..3947ae27f --- /dev/null +++ b/server/handles/fsread_test.go @@ -0,0 +1,255 @@ +package handles + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestGetReadme(t *testing.T) { + tests := []struct { + name string + meta *model.Meta + path string + want string + reason string + }{ + { + name: "nil meta", + meta: nil, + path: "/any", + want: "", + reason: "nil meta should return empty", + }, + { + name: "exact path match with RSub=false", + meta: &model.Meta{ + Path: "/folder", + Readme: "Welcome", + RSub: false, + }, + path: "/folder", + want: "Welcome", + reason: "exact path should show readme", + }, + { + name: "sub path with RSub=true", + meta: &model.Meta{ + Path: "/folder", + Readme: "Welcome", + RSub: true, + }, + path: "/folder/subfolder", + want: "Welcome", + reason: "sub path with RSub=true should show readme", + }, + { + name: "sub path with RSub=false", + meta: &model.Meta{ + Path: "/folder", + Readme: "Welcome", + RSub: false, + }, + path: "/folder/subfolder", + want: "", + reason: "sub path with RSub=false should not show readme", + }, + { + name: "non-sub path with RSub=true (BEHAVIOR CHANGE - BUG FIX)", + meta: &model.Meta{ + Path: "/folder", + Readme: "Welcome", + RSub: true, + }, + path: "/other", + want: "", + reason: "non-sub path should not show readme even with RSub=true (fixed bug)", + }, + { + name: "root readme applies to all with RSub=true", + meta: &model.Meta{ + Path: "/", + Readme: "Global Info", + RSub: true, + }, + path: "/any/path", + want: "Global Info", + reason: "root readme with RSub=true should apply to all paths", + }, + { + name: "empty readme", + meta: &model.Meta{ + Path: "/folder", + Readme: "", + RSub: true, + }, + path: "/folder", + want: "", + reason: "empty readme should return empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getReadme(tt.meta, tt.path) + if got != tt.want { + t.Errorf("getReadme() = %q, want %q\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} + +func TestGetHeader(t *testing.T) { + tests := []struct { + name string + meta *model.Meta + path string + want string + reason string + }{ + { + name: "nil meta", + meta: nil, + path: "/any", + want: "", + reason: "nil meta should return empty", + }, + { + name: "exact path match with HeaderSub=false", + meta: &model.Meta{ + Path: "/folder", + Header: "Custom Header", + HeaderSub: false, + }, + path: "/folder", + want: "Custom Header", + reason: "exact path should show header", + }, + { + name: "sub path with HeaderSub=true", + meta: &model.Meta{ + Path: "/folder", + Header: "Custom Header", + HeaderSub: true, + }, + path: "/folder/subfolder", + want: "Custom Header", + reason: "sub path with HeaderSub=true should show header", + }, + { + name: "sub path with HeaderSub=false", + meta: &model.Meta{ + Path: "/folder", + Header: "Custom Header", + HeaderSub: false, + }, + path: "/folder/subfolder", + want: "", + reason: "sub path with HeaderSub=false should not show header", + }, + { + name: "non-sub path with HeaderSub=true (BEHAVIOR CHANGE - BUG FIX)", + meta: &model.Meta{ + Path: "/folder", + Header: "Custom Header", + HeaderSub: true, + }, + path: "/other", + want: "", + reason: "non-sub path should not show header even with HeaderSub=true (fixed bug)", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getHeader(tt.meta, tt.path) + if got != tt.want { + t.Errorf("getHeader() = %q, want %q\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} + +func TestIsEncrypt(t *testing.T) { + tests := []struct { + name string + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil meta", + meta: nil, + path: "/any", + want: false, + reason: "nil meta should not be encrypted", + }, + { + name: "empty password", + meta: &model.Meta{ + Path: "/folder", + Password: "", + }, + path: "/folder", + want: false, + reason: "empty password should not be encrypted", + }, + { + name: "exact path match with PSub=false", + meta: &model.Meta{ + Path: "/folder", + Password: "secret", + PSub: false, + }, + path: "/folder", + want: true, + reason: "exact path with password should be encrypted", + }, + { + name: "sub path with PSub=true", + meta: &model.Meta{ + Path: "/folder", + Password: "secret", + PSub: true, + }, + path: "/folder/subfolder", + want: true, + reason: "sub path with PSub=true should be encrypted", + }, + { + name: "sub path with PSub=false", + meta: &model.Meta{ + Path: "/folder", + Password: "secret", + PSub: false, + }, + path: "/folder/subfolder", + want: false, + reason: "sub path with PSub=false should not be encrypted", + }, + { + name: "non-sub path with PSub=true", + meta: &model.Meta{ + Path: "/folder", + Password: "secret", + PSub: true, + }, + path: "/other", + want: false, + reason: "non-sub path should not be encrypted even with PSub=true", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isEncrypt(tt.meta, tt.path) + if got != tt.want { + t.Errorf("isEncrypt() = %v, want %v\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index b726d7152..32fa64a42 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -12,12 +12,14 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" + "github.com/pkg/errors" ) type SetAria2Req struct { @@ -499,6 +501,15 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 403) return } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } var tasks []task.TaskExtensionInfo for _, url := range req.Urls { // Filter out empty lines and whitespace-only strings diff --git a/server/middlewares/down.go b/server/middlewares/down.go index cb87eb3c3..c1f81b54b 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -25,11 +25,9 @@ func Down(verifyFunc func(string, string) error) func(c *gin.Context) { return func(c *gin.Context) { rawPath := c.Request.Context().Value(conf.PathKey).(string) meta, err := op.GetNearestMeta(rawPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorPage(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorPage(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) // verify sign diff --git a/server/middlewares/fsup.go b/server/middlewares/fsup.go index 08b160ee5..d99e62aea 100644 --- a/server/middlewares/fsup.go +++ b/server/middlewares/fsup.go @@ -15,7 +15,6 @@ import ( func FsUp(c *gin.Context) { path := c.GetHeader("File-Path") - password := c.GetHeader("Password") path, err := url.PathUnescape(path) if err != nil { common.ErrorResp(c, err, 400) @@ -28,15 +27,19 @@ func FsUp(c *gin.Context) { common.ErrorResp(c, err, 403) return } - meta, err := op.GetNearestMeta(stdpath.Dir(path)) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - c.Abort() - return - } + parentPath := stdpath.Dir(path) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + c.Abort() + return + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + c.Abort() + return } - if !(common.CanAccess(user, meta, path, password) && (user.CanWrite() || common.CanWrite(meta, stdpath.Dir(path)))) { + if !common.CanWrite(user, parentMeta, parentPath) { common.ErrorResp(c, errs.PermissionDenied, 403) c.Abort() return diff --git a/server/webdav.go b/server/webdav.go index 789236b8b..a949068f0 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -117,22 +117,22 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!user.CanWebdavManage() || !user.CanWrite()) { + if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "MOVE" && (!user.CanWebdavManage() || (!user.CanMove() && !user.CanRename())) { + if c.Request.Method == "MOVE" && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "COPY" && (!user.CanWebdavManage() || !user.CanCopy()) { + if c.Request.Method == "COPY" && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "DELETE" && (!user.CanWebdavManage() || !user.CanRemove()) { + if c.Request.Method == "DELETE" && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return @@ -143,6 +143,11 @@ func WebDAVAuth(c *gin.Context) { return } common.GinWithValue(c, conf.UserKey, user) + if user.IsGuest() { + common.GinWithValue(c, conf.MetaPassKey, password) + } else { + common.GinWithValue(c, conf.MetaPassKey, "") + } c.Next() } diff --git a/server/webdav/file.go b/server/webdav/file.go index debfcfe9e..ea6099735 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -11,9 +11,12 @@ import ( "path/filepath" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/pkg/errors" ) // slashClean is equivalent to but slightly more efficient than @@ -26,6 +29,7 @@ func slashClean(name string) string { } // moveFiles moves files and/or directories from src to dst. +// Individual item permission checks are skipped for performance reasons. // // See section 9.9.4 for when various HTTP status codes apply. func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int, err error) { @@ -40,6 +44,17 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int if srcName != dstName && !user.CanRename() { return http.StatusForbidden, nil } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, srcMeta, srcDir) || !common.CanWrite(user, dstMeta, dstDir) { + return http.StatusForbidden, nil + } if srcDir == dstDir { err = fs.Rename(ctx, src, dstName) } else { @@ -59,10 +74,30 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int } // copyFiles copies files and/or directories from src to dst. +// Individual item permission checks are skipped for performance reasons. // // See section 9.8.5 for when various HTTP status codes apply. func copyFiles(ctx context.Context, src, dst string, overwrite bool) (status int, err error) { + srcDir := path.Dir(src) dstDir := path.Dir(dst) + user := ctx.Value(conf.UserKey).(*model.User) + if !user.CanCopy() { + return http.StatusForbidden, nil + } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanRead(user, srcMeta, srcDir) { + return http.StatusForbidden, nil + } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, dstMeta, dstDir) { + return http.StatusForbidden, nil + } _, err = fs.Copy(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir) if err != nil { return http.StatusInternalServerError, err diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 504c5fc1d..06d1431ac 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -7,7 +7,6 @@ package webdav // import "golang.org/x/net/webdav" import ( "context" - "errors" "fmt" "io" "net/http" @@ -20,8 +19,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/net" + "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/pkg/errors" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" @@ -200,7 +201,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err } allow := "OPTIONS, LOCK, PUT, MKCOL" if fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil { @@ -226,10 +227,18 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta // TODO: check locks for read-only access?? ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) + password, _ := ctx.Value(conf.MetaPassKey).(string) reqPath, err = user.JoinPath(reqPath) if err != nil { return http.StatusForbidden, err } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanAccess(user, meta, reqPath, password) { + return http.StatusForbidden, errs.PermissionDenied + } fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { return http.StatusNotFound, err @@ -294,9 +303,12 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) + if !user.CanRemove() { + return http.StatusForbidden, nil + } reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err } // TODO: return MultiStatus where appropriate. @@ -309,6 +321,14 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i } return http.StatusMethodNotAllowed, err } + parentPath := path.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } if err := fs.Remove(ctx, reqPath); err != nil { return http.StatusMethodNotAllowed, err } @@ -363,6 +383,17 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { return http.StatusForbidden, errs.IgnoredSystemFile } + parentPath := path.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } + if !common.CanWrite(user, parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } fsStream := &stream.FileStream{ Obj: &obj, Reader: r.Body, @@ -407,7 +438,7 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err } if r.ContentLength > 0 { @@ -421,13 +452,23 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in } // RFC 4918 9.3.1 // 409 (Conflict) The server MUST NOT create those intermediate collections automatically. - reqDir := path.Dir(reqPath) - if _, err := fs.Get(ctx, reqDir, &fs.GetArgs{}); err != nil { + parentPath := path.Dir(reqPath) + if _, err := fs.Get(ctx, parentPath, &fs.GetArgs{}); err != nil { if errs.IsObjectNotFound(err) { return http.StatusConflict, err } return http.StatusMethodNotAllowed, err } + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } + if !common.CanWrite(user, parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } if err := fs.MakeDir(ctx, reqPath); err != nil { if os.IsNotExist(err) { return http.StatusConflict, err @@ -471,11 +512,11 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status user := ctx.Value(conf.UserKey).(*model.User) src, err = user.JoinPath(src) if err != nil { - return 403, err + return http.StatusForbidden, err } dst, err = user.JoinPath(dst) if err != nil { - return 403, err + return http.StatusForbidden, err } if r.Method == "COPY" { @@ -572,7 +613,14 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus } reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, meta, reqPath) { + return http.StatusForbidden, errs.PermissionDenied } ld = LockDetails{ Root: reqPath, @@ -630,6 +678,24 @@ func (h *Handler) handleUnlock(w http.ResponseWriter, r *http.Request) (status i } t = t[1 : len(t)-1] + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + ctx := r.Context() + user := ctx.Value(conf.UserKey).(*model.User) + reqPath, err = user.JoinPath(reqPath) + if err != nil { + return http.StatusForbidden, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, meta, reqPath) { + return http.StatusForbidden, errs.PermissionDenied + } + switch err = h.LockSystem.Unlock(time.Now(), t); err { case nil: return http.StatusNoContent, err @@ -653,9 +719,17 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status userAgent := r.Header.Get("User-Agent") ctx = context.WithValue(ctx, conf.UserAgentKey, userAgent) user := ctx.Value(conf.UserKey).(*model.User) + password, _ := ctx.Value(conf.MetaPassKey).(string) reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanAccess(user, meta, reqPath, password) { + return http.StatusForbidden, errs.PermissionDenied } fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { @@ -734,7 +808,14 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, meta, reqPath) { + return http.StatusForbidden, errs.PermissionDenied } if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil { if errs.IsObjectNotFound(err) { From 29447a41a34d40058e2a68526890ee06f1a1da0a Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Thu, 26 Mar 2026 21:01:48 +0800 Subject: [PATCH 25/86] revert(db)!: replace SQLite Driver with glebarez/sqlite to avoid CGO (#2269) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert "refactor(db)!: replace SQLite Driver with glebarez/sqlite to avoid CG…" This reverts commit d598ef756928872257f3f108c999fb3982a0a63e. --- go.mod | 15 +- go.sum | 318 ++++++++++++++++++++++++++++-------- internal/bootstrap/db.go | 2 +- internal/op/storage_test.go | 2 +- 4 files changed, 260 insertions(+), 77 deletions(-) diff --git a/go.mod b/go.mod index 6f2248015..c36ac1ca0 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,6 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.4 github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 - github.com/glebarez/sqlite v1.11.0 github.com/go-resty/resty/v2 v2.16.5 github.com/go-webauthn/webauthn v0.13.4 github.com/golang-jwt/jwt/v4 v4.5.2 @@ -85,12 +84,13 @@ require ( gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.9 - gorm.io/gorm v1.30.0 + gorm.io/driver/sqlite v1.5.6 + gorm.io/gorm v1.25.11 ) require ( + cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect - github.com/BurntSushi/toml v1.6.0 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect @@ -104,7 +104,6 @@ require ( github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cronokirby/saferith v0.33.0 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.8.4 // indirect github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect @@ -122,7 +121,6 @@ require ( github.com/minio/minlz v1.0.0 // indirect github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -205,7 +203,6 @@ require ( github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect - github.com/glebarez/go-sqlite v1.22.0 // indirect github.com/go-chi/chi/v5 v5.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -246,6 +243,7 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -273,7 +271,6 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.64.0 // indirect github.com/prometheus/procfs v0.16.1 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rfjakob/eme v1.1.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect @@ -302,10 +299,6 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.33.1 // indirect ) replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0 diff --git a/go.sum b/go.sum index bb5c2e47b..b9a4570bd 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,28 @@ -cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= -cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0= -cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go/auth v0.16.2 h1:QvBAGFPLrDeoiNjyfVunhQ10HKNYuOwZ5noee0M5df4= +cloud.google.com/go/auth v0.16.2/go.mod h1:sRBas2Y1fB1vZTdurouM0AzuYQBMZinrUYL8EufhtEA= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= @@ -17,8 +35,9 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZY github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= -github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= -github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= @@ -186,6 +205,7 @@ github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCN github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= @@ -206,8 +226,12 @@ github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQ github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= @@ -246,8 +270,6 @@ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cn github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= @@ -256,6 +278,8 @@ github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7 github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 h1:q1b+gv6AG2TDPN+f0QAkbRrAvJ3ZosnwRLTKNxSXlaA= @@ -281,14 +305,12 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= -github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= -github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= -github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= @@ -326,14 +348,32 @@ github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXe github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -341,17 +381,22 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= -github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= -github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= -github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= +github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= @@ -374,6 +419,8 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= @@ -385,6 +432,7 @@ github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0 github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= @@ -430,6 +478,8 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg= @@ -448,8 +498,11 @@ github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -482,6 +535,8 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.32.0 h1:cWcycpONSH3VLTZ5npUl1O5aXPkNM0vUx6bywnYqGbE= github.com/meilisearch/meilisearch-go v0.32.0/go.mod h1:aNtyuwurDg/ggxQIcKqWH6G9g2ptc8GyY7PLY4zMn/g= github.com/mholt/archives v0.1.3 h1:aEAaOtNra78G+TvV5ohmXrJOAzf++dIlYeDW3N9q458= @@ -537,8 +592,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncw/swift/v2 v2.0.4 h1:hHWVFxn5/YaTWAASmn4qyq2p6OyP/Hm3vMLzkjEqR7w= github.com/ncw/swift/v2 v2.0.4/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.1.1 h1:OJaYalXdliBUXPmC8CZGQ7oZDxzX1/5mQmgn0/GASew= @@ -568,6 +621,7 @@ github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= @@ -582,16 +636,16 @@ github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE= github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo= github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= @@ -675,24 +729,30 @@ github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 h1:PSRw github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3/go.mod h1:CKriYB8bkNgSbYUQF1khSpejKb5IsV6cR7MdaAR7Fc0= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= +go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -700,6 +760,9 @@ golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -708,23 +771,63 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -736,22 +839,48 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -771,6 +900,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= @@ -783,9 +914,13 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -797,42 +932,111 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ= -google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.236.0 h1:CAiEiDVtO4D/Qja2IA9VzlFrgPnK3XVMmRoJZlSWbc0= +google.golang.org/api v0.236.0/go.mod h1:X1WF9CU2oTc+Jml1tiIxGmWFK/UZezdqEu09gcxZAj4= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= @@ -852,36 +1056,22 @@ gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= +gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= -gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= +gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= +gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.33.1 h1:trb6Z3YYoeM9eDL1O8do81kP+0ejv+YzgyFo+Gwy0nM= -modernc.org/sqlite v1.33.1/go.mod h1:pXV2xHxhzXZsgT/RtTFAPY6JJDEvOTcTdwADQCCWD4k= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index e4b81bf40..d97cb6796 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -9,10 +9,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" - "github.com/glebarez/sqlite" log "github.com/sirupsen/logrus" "gorm.io/driver/mysql" "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" "gorm.io/gorm/schema" diff --git a/internal/op/storage_test.go b/internal/op/storage_test.go index d7db25040..2b191bd56 100644 --- a/internal/op/storage_test.go +++ b/internal/op/storage_test.go @@ -10,7 +10,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" mapset "github.com/deckarep/golang-set/v2" - "github.com/glebarez/sqlite" + "gorm.io/driver/sqlite" "gorm.io/gorm" ) From 7bea29c18e4e7ba49a7909e505b5f8225bc7cfb8 Mon Sep 17 00:00:00 2001 From: Suyunjing Date: Thu, 2 Apr 2026 23:02:12 +0800 Subject: [PATCH 26/86] refactor(db): migrate sqlite to pure-go and add mips compatibility switch (#2296) - Switch default SQLite path to github.com/glebarez/sqlite to reduce CGO dependency pressure. - Introduce a unified openSQLite entry in bootstrap and split driver selection by build tags. - Add sqlite_cgo_compat fallback for linux mips, mips64, loong64 and mipsle to keep legacy target builds working. - Update build.sh musl build flow to apply compatibility tag for mips-family targets. - Update beta_release workflow to pass compatibility tag cleanly and avoid conflicting flag composition. --- .github/workflows/beta_release.yml | 25 +- build.sh | 33 ++- go.mod | 10 +- go.sum | 293 +++---------------- internal/bootstrap/db.go | 5 +- internal/bootstrap/sqlite_driver_glebarez.go | 12 + internal/bootstrap/sqlite_driver_gorm.go | 12 + internal/op/storage_test.go | 2 +- 8 files changed, 130 insertions(+), 262 deletions(-) create mode 100644 internal/bootstrap/sqlite_driver_glebarez.go create mode 100644 internal/bootstrap/sqlite_driver_gorm.go diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 268a5833f..97312f52e 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -61,20 +61,38 @@ jobs: strategy: matrix: include: - - target: "!(*musl*|*windows-arm64*|*windows7-*|*android*|*freebsd*)" # xgo and loongarch + - target: "!(*musl*|*windows-arm64*|*windows7-*|*android*|*freebsd*)" # xgo and loongarch (exclude mips64le) hash: "md5" - - target: "linux-!(arm*)-musl*" #musl-not-arm + flags: "" + goflags: "" + - target: "linux-(mips|mips64|mipsle|mips64le|loong64)-musl*" # musl-compat-family + hash: "md5-linux-musl-mips" + flags: "" + goflags: "" + - target: "linux-!(arm*|mips|mips64|mipsle|mips64le|loong64)-musl*" # musl-not-arm (exclude compat-family) hash: "md5-linux-musl" + flags: "" + goflags: "" - target: "linux-arm*-musl*" #musl-arm hash: "md5-linux-musl-arm" + flags: "" + goflags: "" - target: "windows-arm64" #win-arm64 hash: "md5-windows-arm64" + flags: "" + goflags: "" - target: "windows7-*" #win7 hash: "md5-windows7" + flags: "" + goflags: "-tags=sqlite_cgo_compat" - target: "android-*" #android hash: "md5-android" + flags: "" + goflags: "" - target: "freebsd-*" #freebsd hash: "md5-freebsd" + flags: "" + goflags: "" name: Beta Release runs-on: ubuntu-latest @@ -99,6 +117,7 @@ jobs: uses: OpenListTeam/cgo-actions@v1.2.2 with: targets: ${{ matrix.target }} + flags: ${{ matrix.flags || '-ldflags=' }} musl-target-format: $os-$musl-$arch github-token: ${{ secrets.GITHUB_TOKEN }} out-dir: build @@ -110,6 +129,8 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + env: + GOFLAGS: ${{ matrix.goflags }} - name: Compress run: | diff --git a/build.sh b/build.sh index 26e5a301b..3198d7ce3 100644 --- a/build.sh +++ b/build.sh @@ -48,6 +48,19 @@ ldflags="\ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=$webVersion' \ " +# Keep sqlite driver tag selection centralized to avoid target drift. +GetBuildTagsForTarget() { + local target="$1" + case "$target" in + linux-loong64|linux-mips|linux-mips64|linux-mips64le|linux-mipsle|linux-musl-loong64|linux-musl-mips|linux-musl-mips64|linux-musl-mips64le|linux-musl-mipsle|windows-386|windows7-386|windows7-amd64) + echo "jsoniter,sqlite_cgo_compat" + ;; + *) + echo "jsoniter" + ;; + esac +} + FetchWebRolling() { pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') @@ -110,6 +123,7 @@ BuildWin7() { # Build for both 386 and amd64 architectures for arch in "386" "amd64"; do echo "building for windows7-${arch}" + build_tags=$(GetBuildTagsForTarget "windows7-${arch}") export GOOS=windows export GOARCH=${arch} export CGO_ENABLED=1 @@ -124,7 +138,7 @@ BuildWin7() { fi # Use the patched Go compiler for Win7 compatibility - $(pwd)/go-win7/bin/go build -o "${1}-${arch}.exe" -ldflags="$ldflags" -tags=jsoniter . + $(pwd)/go-win7/bin/go build -o "${1}-${arch}.exe" -ldflags="$ldflags" -tags="$build_tags" . done } @@ -193,11 +207,12 @@ BuildDockerMultiplatform() { cgo_cc=${CGO_ARGS[$i]} os=${os_arch%%-*} arch=${os_arch##*-} + build_tags=$(GetBuildTagsForTarget "$os_arch") export GOOS=$os export GOARCH=$arch export CC=${cgo_cc} echo "building for $os_arch" - go build -o build/$os/$arch/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + go build -o build/$os/$arch/"$appName" -ldflags="$docker_lflags" -tags="$build_tags" . done DOCKER_ARM_ARCHES=(linux-arm/v6 linux-arm/v7) @@ -237,6 +252,8 @@ BuildLoongGLIBC() { local target_abi="$2" local output_file="$1" local oldWorldGoVersion="1.25.0" + local loong_tags + loong_tags=$(GetBuildTagsForTarget "linux-loong64") if [ "$target_abi" = "abi1.0" ]; then echo building for linux-loong64-abi1.0 @@ -311,7 +328,7 @@ BuildLoongGLIBC() { CXX="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++" \ CGO_ENABLED=1 \ GOCACHE="$abi1_cache_dir" \ - $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then + $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags="$loong_tags" .; then echo "Error: Build failed with patched Go compiler" echo "Attempting retry with cache cleanup..." env GOCACHE="$abi1_cache_dir" $(pwd)/go-loong64-abi1.0/bin/go clean -cache @@ -320,7 +337,7 @@ BuildLoongGLIBC() { CXX="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++" \ CGO_ENABLED=1 \ GOCACHE="$abi1_cache_dir" \ - $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then + $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags="$loong_tags" .; then echo "Error: Build failed again after cache cleanup" echo "Build environment details:" echo "GOOS=linux" @@ -366,11 +383,11 @@ BuildLoongGLIBC() { # Use standard Go compiler for new-world build echo "Building with standard Go compiler for new-world ABI2.0..." - if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then + if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags="$loong_tags" .; then echo "Error: Build failed with standard Go compiler" echo "Attempting retry with cache cleanup..." go clean -cache - if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then + if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags="$loong_tags" .; then echo "Error: Build failed again after cache cleanup" echo "Build environment details:" echo "GOOS=$GOOS" @@ -391,6 +408,7 @@ BuildReleaseLinuxMusl() { mkdir -p "build" muslflags="--extldflags '-static -fpic' $ldflags" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" + # Keep mips-family targets enabled; sqlite driver selection is handled by Go build tags. FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross mips-linux-musl-cross mips64-linux-musl-cross mips64el-linux-musl-cross mipsel-linux-musl-cross powerpc64le-linux-musl-cross s390x-linux-musl-cross loongarch64-linux-musl-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" @@ -403,12 +421,13 @@ BuildReleaseLinuxMusl() { for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} + build_tags=$(GetBuildTagsForTarget "$os_arch") echo building for ${os_arch} export GOOS=${os_arch%%-*} export GOARCH=${os_arch##*-} export CC=${cgo_cc} export CGO_ENABLED=1 - go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags="$build_tags" . done } diff --git a/go.mod b/go.mod index c36ac1ca0..2fc141f2e 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.4 github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 + github.com/glebarez/sqlite v1.11.0 github.com/go-resty/resty/v2 v2.16.5 github.com/go-webauthn/webauthn v0.13.4 github.com/golang-jwt/jwt/v4 v4.5.2 @@ -89,8 +90,8 @@ require ( ) require ( - cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/BurntSushi/toml v1.6.0 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect @@ -104,10 +105,12 @@ require ( github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cronokirby/saferith v0.33.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.8.4 // indirect github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect github.com/geoffgarside/ber v1.2.0 // indirect + github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -123,11 +126,16 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.uber.org/mock v0.5.0 // indirect golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect golang.org/x/mod v0.30.0 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect + modernc.org/libc v1.22.5 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/sqlite v1.23.1 // indirect ) require ( diff --git a/go.sum b/go.sum index b9a4570bd..0f69ce117 100644 --- a/go.sum +++ b/go.sum @@ -1,28 +1,10 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go/auth v0.16.2 h1:QvBAGFPLrDeoiNjyfVunhQ10HKNYuOwZ5noee0M5df4= -cloud.google.com/go/auth v0.16.2/go.mod h1:sRBas2Y1fB1vZTdurouM0AzuYQBMZinrUYL8EufhtEA= +cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= +cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0= +cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= -cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= @@ -35,9 +17,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZY github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= @@ -205,7 +186,6 @@ github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCN github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= @@ -226,12 +206,8 @@ github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQ github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= @@ -270,6 +246,8 @@ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cn github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= @@ -278,8 +256,6 @@ github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7 github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 h1:q1b+gv6AG2TDPN+f0QAkbRrAvJ3ZosnwRLTKNxSXlaA= @@ -305,12 +281,14 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= +github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= +github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= +github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= @@ -348,32 +326,14 @@ github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXe github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -381,22 +341,17 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b h1:Qcx5LM0fSiks9uCyFZwDBUasd3lxd1RM0GYpL+Li5o4= +github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= -github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= -github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= +github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= @@ -419,8 +374,6 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= @@ -432,7 +385,6 @@ github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0 github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= @@ -478,8 +430,6 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg= @@ -498,11 +448,8 @@ github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -621,7 +568,6 @@ github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= @@ -636,16 +582,17 @@ github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE= github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo= github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= @@ -729,30 +676,24 @@ github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 h1:PSRw github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3/go.mod h1:CKriYB8bkNgSbYUQF1khSpejKb5IsV6cR7MdaAR7Fc0= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= -go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= -go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -760,9 +701,6 @@ golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -771,63 +709,23 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -839,48 +737,22 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -900,8 +772,6 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= @@ -914,13 +784,9 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -932,111 +798,42 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.236.0 h1:CAiEiDVtO4D/Qja2IA9VzlFrgPnK3XVMmRoJZlSWbc0= -google.golang.org/api v0.236.0/go.mod h1:X1WF9CU2oTc+Jml1tiIxGmWFK/UZezdqEu09gcxZAj4= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ= +google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= @@ -1061,17 +858,17 @@ gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDa gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index d97cb6796..7b91769f9 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -12,7 +12,6 @@ import ( log "github.com/sirupsen/logrus" "gorm.io/driver/mysql" "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" "gorm.io/gorm/schema" @@ -41,7 +40,7 @@ func InitDB() { var dB *gorm.DB var err error if flags.Dev { - dB, err = gorm.Open(sqlite.Open("file::memory:?cache=shared"), gormConfig) + dB, err = gorm.Open(openSQLite("file::memory:?cache=shared"), gormConfig) conf.Conf.Database.Type = "sqlite3" } else { database := conf.Conf.Database @@ -51,7 +50,7 @@ func InitDB() { if !(strings.HasSuffix(database.DBFile, ".db") && len(database.DBFile) > 3) { log.Fatalf("db name error.") } - dB, err = gorm.Open(sqlite.Open(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental", + dB, err = gorm.Open(openSQLite(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental", database.DBFile)), gormConfig) } case "mysql": diff --git a/internal/bootstrap/sqlite_driver_glebarez.go b/internal/bootstrap/sqlite_driver_glebarez.go new file mode 100644 index 000000000..a45a8baeb --- /dev/null +++ b/internal/bootstrap/sqlite_driver_glebarez.go @@ -0,0 +1,12 @@ +//go:build !sqlite_cgo_compat && !(linux && (mips || mips64 || mips64le || mipsle || loong64)) && !(windows && 386) + +package bootstrap + +import ( + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func openSQLite(dsn string) gorm.Dialector { + return sqlite.Open(dsn) +} diff --git a/internal/bootstrap/sqlite_driver_gorm.go b/internal/bootstrap/sqlite_driver_gorm.go new file mode 100644 index 000000000..e69630eac --- /dev/null +++ b/internal/bootstrap/sqlite_driver_gorm.go @@ -0,0 +1,12 @@ +//go:build sqlite_cgo_compat || (linux && (mips || mips64 || mips64le || mipsle || loong64)) || (windows && 386) + +package bootstrap + +import ( + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func openSQLite(dsn string) gorm.Dialector { + return sqlite.Open(dsn) +} diff --git a/internal/op/storage_test.go b/internal/op/storage_test.go index 2b191bd56..d7db25040 100644 --- a/internal/op/storage_test.go +++ b/internal/op/storage_test.go @@ -10,7 +10,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" mapset "github.com/deckarep/golang-set/v2" - "gorm.io/driver/sqlite" + "github.com/glebarez/sqlite" "gorm.io/gorm" ) From 9fdba3a730932fff6b52054b4b83f25ac35ac1a0 Mon Sep 17 00:00:00 2001 From: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:23:26 +0800 Subject: [PATCH 27/86] feat(drivers/123_open): support 123 official app api (#2293) * feat(driver): support 123 official app api * fix(123_open): migrate api refresh to token.go Signed-off-by: MadDogOwner * fix(drivers/123_open): trigger proactive refresh with client credentials * fix(drivers/123_open): use client-credential token endpoint for local refresh Keep renewapi parsing for expires_in and map it to internal expiry time handling. * fix(drivers/123_open): limit proactive refresh to client credentials * fix(drivers/123_open): allow renewapi refresh token proactive init * fix(drivers/123_open): update API address to use renewapi endpoint * fix(drivers/123_open): simplify token refresh parsing * fix(drivers/123_open): unify token expiration to expiredAt --------- Signed-off-by: MadDogOwner Co-authored-by: MadDogOwner Co-authored-by: Suyunmeng Co-authored-by: Suyunjing --- drivers/123_open/driver.go | 4 +- drivers/123_open/meta.go | 10 ++- drivers/123_open/token.go | 140 ++++++++++++++++++++----------------- drivers/123_open/types.go | 13 ++-- 4 files changed, 94 insertions(+), 73 deletions(-) diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index e20140277..78ff272b9 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -34,8 +34,8 @@ func (d *Open123) Init(ctx context.Context) error { d.UploadThread = 3 } - if d.RefreshToken != "" { - // refresh token 直接主动刷新 + if (d.UseOnlineAPI && d.RefreshToken != "" && len(d.APIAddress) > 0) || (d.ClientID != "" && d.ClientSecret != "") { + // proactive refresh by renewapi or client credentials d.AccessToken = "" d.tm = &tokenManager{} } else { diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go index 5481ef356..d23f8eec6 100644 --- a/drivers/123_open/meta.go +++ b/drivers/123_open/meta.go @@ -6,9 +6,6 @@ import ( ) type Addition struct { - // refresh_token方式的AccessToken 【对个人开发者暂未开放】 - RefreshToken string `json:"RefreshToken" required:"false"` - // 通过 https://www.123pan.com/developer 申请 ClientID string `json:"ClientID" required:"false"` ClientSecret string `json:"ClientSecret" required:"false"` @@ -16,6 +13,13 @@ type Addition struct { // 直接写入AccessToken, AccessToken有过期时间，不建议直接填写 AccessToken string `json:"AccessToken" required:"false"` + // refresh_token方式的AccessToken 【对个人开发者暂未开放】 + RefreshToken string `json:"RefreshToken" required:"false"` + + // 使用在线API + UseOnlineAPI bool `json:"use_online_api" default:"true"` + APIAddress string `json:"api_url_address" default:"https://api.oplist.org/123cloud/renewapi"` + // 用户名+密码方式登录的AccessToken可以兼容 //Username string `json:"username" required:"false"` //Password string `json:"password" required:"false"` diff --git a/drivers/123_open/token.go b/drivers/123_open/token.go index 3c5c416c9..a628d22fe 100644 --- a/drivers/123_open/token.go +++ b/drivers/123_open/token.go @@ -1,7 +1,6 @@ package _123_open import ( - "encoding/json" "errors" "fmt" "net/http" @@ -13,10 +12,16 @@ import ( ) var ( - AccessToken = "https://open-api.123pan.com/api/v1/access_token" - RefreshToken = "https://open-api.123pan.com/api/v1/oauth2/access_token" + AccessToken = "https://open-api.123pan.com/api/v1/access_token" ) +func expiresInToExpiredAt(expiresIn int64) (time.Time, error) { + if expiresIn <= 0 { + return time.Time{}, errors.New("invalid expires_in from official API") + } + return time.Now().UTC().Add(time.Duration(expiresIn) * time.Second), nil +} + type tokenManager struct { // accessToken string expiredAt time.Time @@ -43,73 +48,82 @@ func (d *Open123) getAccessToken(forceRefresh bool) (string, error) { } func (d *Open123) flushAccessToken() error { - // directly send request to avoid deadlock - req := base.RestyClient.R() - req.SetHeaders(map[string]string{ - "authorization": "Bearer " + d.AccessToken, - "platform": "open_platform", - "Content-Type": "application/json", - }) + // Official app renewapi response contains access_token, refresh_token and expires_in. + if d.UseOnlineAPI && d.RefreshToken != "" && len(d.APIAddress) > 0 { + var resp RefreshTokenResp + _, err := base.RestyClient.R(). + SetResult(&resp). + SetQueryParams(map[string]string{ + "refresh_ui": d.RefreshToken, + "server_use": "true", + "driver_txt": "123cloud_oa", + }). + Get(d.APIAddress) + if err != nil { + return err + } - if d.ClientID != "" { - if d.RefreshToken != "" { - var resp RefreshTokenResp - req.SetQueryParam("client_id", d.ClientID) - if d.ClientSecret != "" { - req.SetQueryParam("client_secret", d.ClientSecret) + if resp.AccessToken == "" || resp.RefreshToken == "" { + errMessage := resp.ErrorDescription + if errMessage == "" { + errMessage = resp.Text } - req.SetQueryParam("grant_type", "refresh_token") - req.SetQueryParam("refresh_token", d.RefreshToken) - req.SetResult(&resp) - res, err := req.Execute(http.MethodPost, RefreshToken) - if err != nil { - return err + if errMessage == "" { + errMessage = resp.Message } - body := res.Body() - var baseResp BaseResp - if err = json.Unmarshal(body, &baseResp); err != nil { - return err + if errMessage == "" { + errMessage = resp.Error } - if baseResp.Code != 0 { - return fmt.Errorf("get access token failed: %s", baseResp.Message) + if errMessage != "" { + return fmt.Errorf("failed to refresh token: %s", errMessage) } + return fmt.Errorf("empty access_token or refresh_token returned from official API") + } + expiredAt, err := expiresInToExpiredAt(resp.ExpiresIn) + if err != nil { + return err + } - d.AccessToken = resp.AccessToken - // add token expire time - d.tm.expiredAt = time.Now().Add(time.Duration(resp.ExpiresIn) * time.Second) - d.RefreshToken = resp.RefreshToken - op.MustSaveDriverStorage(d) - d.tm.blockRefresh = false - return nil - } else if d.ClientSecret != "" { - var resp AccessTokenResp - req.SetBody(base.Json{ - "clientID": d.ClientID, - "clientSecret": d.ClientSecret, - }) - req.SetResult(&resp) - res, err := req.Execute(http.MethodPost, AccessToken) - if err != nil { - return err - } - body := res.Body() - var baseResp BaseResp - if err = json.Unmarshal(body, &baseResp); err != nil { - return err - } - if baseResp.Code != 0 { - return fmt.Errorf("get access token failed: %s", baseResp.Message) - } - d.AccessToken = resp.Data.AccessToken - // parse token expire time - d.tm.expiredAt, err = time.Parse(time.RFC3339, resp.Data.ExpiredAt) - if err != nil { - return fmt.Errorf("parse expire time failed: %w", err) - } - op.MustSaveDriverStorage(d) - d.tm.blockRefresh = false - return nil + d.AccessToken = resp.AccessToken + d.RefreshToken = resp.RefreshToken + d.tm.expiredAt = expiredAt + op.MustSaveDriverStorage(d) + d.tm.blockRefresh = false + return nil + } + + // Developer API response contains code/message/data(accessToken, expiredAt). + if d.ClientID != "" && d.ClientSecret != "" { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "platform": "open_platform", + "Content-Type": "application/json", + }) + var resp AccessTokenResp + req.SetBody(base.Json{ + "clientID": d.ClientID, + "clientSecret": d.ClientSecret, + }) + req.SetResult(&resp) + _, err := req.Execute(http.MethodPost, AccessToken) + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("get access token failed: %s", resp.Message) + } + if resp.Data.AccessToken == "" || resp.Data.ExpiredAt == "" { + return errors.New("invalid token payload from developer API") + } + expiredAt, err := time.Parse(time.RFC3339, resp.Data.ExpiredAt) + if err != nil { + return fmt.Errorf("parse expire time failed: %w", err) } + d.AccessToken = resp.Data.AccessToken + d.tm.expiredAt = expiredAt.UTC() + op.MustSaveDriverStorage(d) + d.tm.blockRefresh = false + return nil } return errors.New("no valid authentication method available") } diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go index 7d586c8b0..b6e507ac4 100644 --- a/drivers/123_open/types.go +++ b/drivers/123_open/types.go @@ -125,11 +125,14 @@ type AccessTokenResp struct { } type RefreshTokenResp struct { - AccessToken string `json:"access_token"` - ExpiresIn int `json:"expires_in"` - RefreshToken string `json:"refresh_token"` - Scope string `json:"scope"` - TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int64 `json:"expires_in"` + Code int `json:"code"` + Message string `json:"message"` + ErrorDescription string `json:"error_description"` + Error string `json:"error"` + Text string `json:"text"` } type UserInfoResp struct { From 9e49adc3536a52572c496e11c4f555007da6467d Mon Sep 17 00:00:00 2001 From: Seven <53081179+sevxn007@users.noreply.github.com> Date: Fri, 3 Apr 2026 15:41:25 +0800 Subject: [PATCH 28/86] fix(drivers/openlist): pass through frontend refresh flag (#2307) * fix(drivers/openlist): pass through frontend refresh flag * fix(drivers/openlist): gate refresh flag forwarding by config --- drivers/openlist/driver.go | 2 +- drivers/openlist/meta.go | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/openlist/driver.go b/drivers/openlist/driver.go index 2ca60ff61..79fc51185 100644 --- a/drivers/openlist/driver.go +++ b/drivers/openlist/driver.go @@ -84,7 +84,7 @@ func (d *OpenList) List(ctx context.Context, dir model.Obj, args model.ListArgs) }, Path: dir.GetPath(), Password: d.MetaPassword, - Refresh: false, + Refresh: d.PassRefreshFlagToUpsteam && args.Refresh, }) }) if err != nil { diff --git a/drivers/openlist/meta.go b/drivers/openlist/meta.go index 16c6a155c..3c4d0801a 100644 --- a/drivers/openlist/meta.go +++ b/drivers/openlist/meta.go @@ -7,14 +7,15 @@ import ( type Addition struct { driver.RootPath - Address string `json:"url" required:"true"` - MetaPassword string `json:"meta_password"` - Username string `json:"username"` - Password string `json:"password"` - Token string `json:"token"` - PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` - PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` - ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` + Address string `json:"url" required:"true"` + MetaPassword string `json:"meta_password"` + Username string `json:"username"` + Password string `json:"password"` + Token string `json:"token"` + PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` + PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` + ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` + PassRefreshFlagToUpsteam bool `json:"pass_refresh_flag_to_upsteam" default:"false"` } var config = driver.Config{ From 12c9bdbd568bca15b6963433050e8d3499b262be Mon Sep 17 00:00:00 2001 From: sdvcrx Date: Fri, 3 Apr 2026 15:47:40 +0800 Subject: [PATCH 29/86] fix(offline_download): prevent infinite retry on status update failure (#2294) --- internal/offline_download/tool/download.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 50a4f6343..5ee6ef4ff 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -147,11 +147,11 @@ func (t *DownloadTask) Update() (bool, error) { if err != nil { t.callStatusRetried++ log.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) + if t.callStatusRetried > 5 { + return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) + } return false, nil } - if t.callStatusRetried > 5 { - return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) - } t.callStatusRetried = 0 t.SetProgress(info.Progress) t.SetTotalBytes(info.TotalBytes) From e11b8a82e7dc500e7fb26fedbac68d557474b70e Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sat, 4 Apr 2026 18:43:08 +0800 Subject: [PATCH 30/86] fix(drivers/cloudreve_v4): remove token check for share (#2274) Fixed the issue of token verification for shared links. --- drivers/cloudreve_v4/driver.go | 3 +++ drivers/cloudreve_v4/util.go | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/cloudreve_v4/driver.go b/drivers/cloudreve_v4/driver.go index cd5cf1b3b..2963bf467 100644 --- a/drivers/cloudreve_v4/driver.go +++ b/drivers/cloudreve_v4/driver.go @@ -46,6 +46,9 @@ func (d *CloudreveV4) Init(ctx context.Context) error { if d.ref != nil { return nil } + if d.isShare() { + return nil + } if d.canLogin() { return d.login() } diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go index f8fe5f269..5d0157ff1 100644 --- a/drivers/cloudreve_v4/util.go +++ b/drivers/cloudreve_v4/util.go @@ -33,6 +33,7 @@ const ( CodeLoginRequired = http.StatusUnauthorized CodePathNotExist = 40016 // Path not exist CodeCredentialInvalid = 40020 // Failed to issue token + // IncorrectSharePassword = 40069 // Incorrect share password ) var ( @@ -277,9 +278,16 @@ func (d *CloudreveV4) parseJWT(token string, jwt any) error { return nil } +func (d *CloudreveV4) isShare() bool { + return strings.HasSuffix(d.GetRootPath(), "@share") +} + // check if token is expired // https://github.com/cloudreve/frontend/blob/ddfacc1c31c49be03beb71de4cc114c8811038d6/src/session/index.ts#L177-L200 func (d *CloudreveV4) isTokenExpired() bool { + if d.isShare() { + return false + } if d.RefreshToken == "" { // login again if username and password is set if d.canLogin() { From 5b688a3002917a44c9ecbc69eeed9f9238c24ee4 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 4 Jan 2026 12:37:51 +0800 Subject: [PATCH 31/86] feat(docs): add CLAUDE.md for project guidance and development instructions refactor(build): restrict builds to x64 architecture and simplify Docker workflow fix(workflow): update beta image tag to remove unnecessary suffix --- .github/workflows/test_docker.yml | 48 ++--- CLAUDE.md | 297 ++++++++++++++++++++++++++++++ build.sh | 20 +- internal/stream/util.go | 95 +++++++--- 4 files changed, 389 insertions(+), 71 deletions(-) create mode 100644 CLAUDE.md diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index aa6fe8966..a3ca52258 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -1,5 +1,4 @@ name: Beta Release (Docker) - on: workflow_dispatch: push: @@ -7,51 +6,51 @@ on: - main pull_request: branches: - - main + - fix # 👈 允许你的 fix 分支触发 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: - DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} - GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} - IMAGE_NAME: openlist-git - IMAGE_NAME_DOCKERHUB: openlist + GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 + IMAGE_NAME: openlist REGISTRY: ghcr.io ARTIFACT_NAME: 'binaries_docker_release' - RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. - IMAGE_PUSH: ${{ github.event_name == 'push' }} + # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 + RELEASE_PLATFORMS: 'linux/amd64' + # 👇 关键修改：强制允许推送，不用管是不是 push 事件 + IMAGE_PUSH: 'true' IMAGE_TAGS_BETA: | type=ref,event=pr - type=raw,value=beta,enable={{is_default_branch}} + type=raw,value=beta-retry jobs: build_binary: - name: Build Binaries for Docker Release (Beta) + name: Build Binaries (x64 Only) runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-go@v5 with: go-version: '1.25.0' + # 即使只构建 x64，我们也需要 musl 工具链（因为 BuildDockerMultiplatform 默认会检查它） - name: Cache Musl id: cache-musl uses: actions/cache@v4 with: path: build/musl-libs key: docker-musl-libs-v2 - - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build go binary (beta) + - name: Build go binary + # 这里还是跑 docker-multiplatform，虽然会多编译一些架构，但这是兼容 Dockerfile 路径最稳妥的方法 run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -69,12 +68,13 @@ jobs: release_docker: needs: build_binary - name: Release Docker image (Beta) + name: Release Docker (x64) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: + # 你可以选择只构建 latest，或者保留全部变体 image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" @@ -102,46 +102,32 @@ jobs: with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # 👇 只保留 GitHub 登录，删除了 DockerHub 登录 - name: Login to GitHub Container Registry - if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to DockerHub Container Registry - if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} - ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} tags: ${{ env.IMAGE_TAGS_BETA }} - flavor: | - ${{ matrix.tag_favor }} + flavor: ${{ matrix.tag_favor }} - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci - push: ${{ env.IMAGE_PUSH == 'true' }} + push: true build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..6a1e1461c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,297 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Build and Development Commands + +```bash +# Development +go run main.go # Run backend server (default port 5244) +air # Hot reload during development (uses .air.toml) +./build.sh dev # Build development version with frontend +./build.sh release # Build release version + +# Testing +go test ./... # Run all tests + +# Docker +docker-compose up # Run with docker-compose +docker build -f Dockerfile . # Build docker image +``` + +**Build Script Details** (`build.sh`): +- Fetches frontend from OpenListTeam/OpenList-Frontend releases +- Injects version info via ldflags: `-X "github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$(date +'%F %T %z')"` +- Supports `dev`, `beta`, and release builds +- Downloads prebuilt frontend distribution automatically + +**Go Version**: Requires Go 1.23.4+ + +## Architecture Overview + +### Driver System (Storage Abstraction) + +OpenList uses a **driver pattern** to support 70+ cloud storage providers. Each driver implements the core `Driver` interface. + +**Location**: `drivers/*/` + +**Core Interfaces** (`internal/driver/driver.go`): +- `Reader`: List directories, generate download links (REQUIRED) +- `Writer`: Upload, delete, move files (optional) +- `ArchiveDriver`: Extract archives (optional) +- `LinkCacheModeResolver`: Custom cache TTL strategies (optional) + +**Driver Registration Pattern**: +```go +// In drivers/your_driver/meta.go +var config = driver.Config{ + Name: "YourDriver", + LocalSort: false, + NoCache: false, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &YourDriver{} + }) +} +``` + +**Adding a New Driver**: +1. Copy `drivers/template/` to `drivers/your_driver/` +2. Implement `List()` and `Link()` methods (required) +3. Define `Addition` struct with configuration fields using struct tags: + - `json:"field_name"` - JSON field name + - `type:"select"` - Input type (select, string, text, bool, number) + - `required:"true"` - Required field + - `options:"a,b,c"` - Dropdown options + - `default:"value"` - Default value +4. Register driver in `init()` function + +**Example Driver Structure**: +```go +type YourDriver struct { + model.Storage + Addition + client *YourClient +} + +func (d *YourDriver) Init(ctx context.Context) error { + // Initialize client, login, etc. +} + +func (d *YourDriver) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + // Return list of files/folders +} + +func (d *YourDriver) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + // Return download URL or RangeReader +} +``` + +### Request Flow + +``` +HTTP Request (Gin Router) + ↓ +Middleware (Auth, CORS, Logging) + ↓ +Handler (server/handles/) + ↓ +fs.List/Get/Link (mount path → storage path conversion) + ↓ +op.List/Get/Link (caching, driver lookup) + ↓ +Driver.List/Link (storage-specific API calls) + ↓ +Response (JSON / Proxy / Redirect) +``` + +### Internal Package Structure + +| Package | Purpose | +|---------|---------| +| `bootstrap/` | Initialization sequence: config, DB, storages, servers | +| `conf/` | Configuration management | +| `db/` | Database models (SQLite/MySQL/Postgres) | +| `driver/` | Driver interface definitions | +| `fs/` | Mount path abstraction (converts `/mount/path` to storage + path) | +| `op/` | Core operations with caching and driver management | +| `stream/` | Streaming, range readers, link refresh, rate limiting | +| `model/` | Data models (Obj, Link, Storage, User) | +| `cache/` | Multi-level caching (directories, links, users, settings) | +| `net/` | HTTP utilities, proxy config, download manager | + +### Link Generation and Caching + +**Link Types**: +1. **Direct URL** (`link.URL`): Simple redirect to storage provider +2. **RangeReader** (`link.RangeReader`): Custom streaming implementation +3. **Refreshable Link** (`link.Refresher`): Auto-refresh on expiration + +**Cache System** (`internal/op/cache.go`): +- **Directory Cache**: Stores file listings with configurable TTL +- **Link Cache**: Stores download URLs (30min default) +- **User Cache**: Authentication data (1hr default) +- **Custom Policies**: Pattern-based TTL via `pattern:ttl` format + +**Cache Key Pattern**: `{storageMountPath}/{relativePath}` + +**Invalidation**: Recursive tree deletion for directory operations + +### Range Reader and Streaming + +**Location**: `internal/stream/` + +**Purpose**: Handle partial content requests (HTTP 206), multi-threaded downloads, and link refresh during streaming. + +**Key Components**: + +1. **RangeReaderIF**: Core interface for range-based reading + ```go + type RangeReaderIF interface { + RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) + } + ``` + +2. **RefreshableRangeReader**: Wraps RangeReader with automatic link refresh + - Detects expired links via error strings or HTTP status codes (401, 403, 410, 500) + - Calls `link.Refresher(ctx)` to get new link + - Resumes download from current byte position + - Max 3 refresh attempts to prevent infinite loops + +3. **Multi-threaded Downloader** (`internal/net/downloader.go`): + - Splits file into parts based on `Concurrency` and `PartSize` + - Downloads parts in parallel + - Assembles final stream + +**Link Refresh Pattern**: +```go +// In op.Link(), a refresher is automatically attached +link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + // Get fresh link from storage driver + file, err := GetUnwrap(refreshCtx, storage, path) + newLink, err := storage.Link(refreshCtx, file, args) + return newLink, file, nil +} + +// RefreshableRangeReader uses this during streaming +if IsLinkExpiredError(err) && r.link.Refresher != nil { + newLink, _, err := r.link.Refresher(ctx) + // Resume from current position +} +``` + +**Proxy Function** (`server/common/proxy.go`): + +Handles multiple scenarios: +1. Multi-threaded download (`link.Concurrency > 0`) +2. Direct RangeReader (`link.RangeReader != nil`) +3. Refreshable link (`link.Refresher != nil`) ← Wraps with RefreshableRangeReader +4. Transparent proxy (forwards to `link.URL`) + +### Startup Sequence + +**Location**: `internal/bootstrap/run.go` + +Order of initialization: +1. `InitConfig()` - Load config, environment variables +2. `Log()` - Initialize logging +3. `InitDB()` - Connect to database +4. `data.InitData()` - Initialize default data +5. `LoadStorages()` - Load and initialize all storage drivers +6. `InitTaskManager()` - Start background tasks +7. `Start()` - Start HTTP/HTTPS/WebDAV/FTP/SFTP servers + +## Common Patterns + +### Error Handling + +Use custom errors from `internal/errs/`: +- `errs.NotImplement` - Feature not implemented +- `errs.ObjectNotFound` - File/folder not found +- `errs.NotFolder` - Path is not a directory +- `errs.StorageNotInit` - Storage driver not initialized + +**Link Expiry Detection**: +```go +// Checks error string for keywords: "expired", "invalid signature", "token expired" +// Also checks HTTP status: 401, 403, 410, 500 +if stream.IsLinkExpiredError(err) { + // Refresh link +} +``` + +### Saving Driver State + +When updating tokens or credentials: +```go +d.AccessToken = newToken +op.MustSaveDriverStorage(d) // Persists to database +``` + +### Rate Limiting + +Use `rate.Limiter` for API rate limits: +```go +type YourDriver struct { + limiter *rate.Limiter +} + +func (d *YourDriver) Init(ctx context.Context) error { + d.limiter = rate.NewLimiter(rate.Every(time.Second), 1) // 1 req/sec +} + +func (d *YourDriver) List(...) { + d.limiter.Wait(ctx) + // Make API call +} +``` + +### Context Cancellation + +Always respect context cancellation in long operations: +```go +select { +case <-ctx.Done(): + return nil, ctx.Err() +default: + // Continue operation +} +``` + +## Important Conventions + +**Naming**: +- Drivers: lowercase with underscores (e.g., `baidu_netdisk`, `aliyundrive_open`) +- Packages: lowercase (e.g., `internal/op`) +- Interfaces: PascalCase with suffix (e.g., `Reader`, `Writer`) + +**Driver Configuration Fields**: +- Use `driver.RootPath` or `driver.RootID` for root folder +- Add `omitempty` to optional JSON fields +- Use descriptive help text in struct tags + +**Retries and Timeouts**: +- Use `github.com/avast/retry-go` for retry logic +- Set reasonable timeouts on HTTP clients (default 30s in `base.RestyClient`) +- For unstable APIs, implement exponential backoff + +**Logging**: +- Use `logrus` via `log` package +- Levels: `log.Debugf`, `log.Infof`, `log.Warnf`, `log.Errorf` +- Include driver name in logs: `log.Infof("[driver_name] message")` + +## Project Context + +OpenList is a community-driven fork of AList, focused on: +- Long-term governance and trust +- Support for 70+ cloud storage providers +- Web UI for file management +- Multi-protocol support (HTTP, WebDAV, FTP, SFTP, S3) +- Offline downloads (Aria2, Transmission) +- Full-text search +- Archive extraction + +**License**: AGPL-3.0 diff --git a/build.sh b/build.sh index 3198d7ce3..c26d7c557 100644 --- a/build.sh +++ b/build.sh @@ -200,8 +200,8 @@ BuildDockerMultiplatform() { docker_lflags="--extldflags '-static -fpic' $ldflags" export CGO_ENABLED=1 - OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds - CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds + OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds + CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} @@ -220,14 +220,14 @@ BuildDockerMultiplatform() { GO_ARM=(6 7) export GOOS=linux export GOARCH=arm - for i in "${!DOCKER_ARM_ARCHES[@]}"; do - docker_arch=${DOCKER_ARM_ARCHES[$i]} - cgo_cc=${CGO_ARGS[$i]} - export GOARM=${GO_ARM[$i]} - export CC=${cgo_cc} - echo "building for $docker_arch" - go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . - done + # for i in "${!DOCKER_ARM_ARCHES[@]}"; do + # docker_arch=${DOCKER_ARM_ARCHES[$i]} + # cgo_cc=${CGO_ARGS[$i]} + # export GOARM=${GO_ARM[$i]} + # export CC=${cgo_cc} + # echo "building for $docker_arch" + # go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + # done } BuildRelease() { diff --git a/internal/stream/util.go b/internal/stream/util.go index 6aa3dda5d..83b20da71 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -174,6 +174,69 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } +// StreamHashFile 流式计算文件哈希值，避免将整个文件加载到内存 +// file: 文件流 +// hashType: 哈希算法类型 +// progressWeight: 进度权重（0-100），用于计算整体进度 +// up: 进度回调函数 +func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressWeight float64, up *model.UpdateProgress) (string, error) { + // 如果已经有完整缓存文件，直接使用 + if cache := file.GetFile(); cache != nil { + hashFunc := hashType.NewFunc() + cache.Seek(0, io.SeekStart) + _, err := io.Copy(hashFunc, cache) + if err != nil { + return "", err + } + if up != nil && progressWeight > 0 { + (*up)(progressWeight) + } + return hex.EncodeToString(hashFunc.Sum(nil)), nil + } + + hashFunc := hashType.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk + var offset int64 = 0 + const maxRetries = 3 + for offset < size { + readSize := chunkSize + if size-offset < chunkSize { + readSize = size - offset + } + + var lastErr error + for retry := 0; retry < maxRetries; retry++ { + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize}) + if err != nil { + lastErr = fmt.Errorf("range read for hash calculation failed: %w", err) + continue + } + _, err = io.Copy(hashFunc, reader) + if closer, ok := reader.(io.Closer); ok { + closer.Close() + } + if err == nil { + lastErr = nil + break + } + lastErr = fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + if lastErr != nil { + return "", lastErr + } + + offset += readSize + + if up != nil && progressWeight > 0 { + progress := progressWeight * float64(offset) / float64(size) + (*up)(progress) + } + } + + return hex.EncodeToString(hashFunc.Sum(nil)), nil +} + type StreamSectionReaderIF interface { // 线程不安全 GetSectionReader(off, length int64) (io.ReadSeeker, error) @@ -188,37 +251,9 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode } maxBufferSize = min(maxBufferSize, int(file.GetSize())) - if maxBufferSize > conf.MaxBufferLimit { - f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - - if f.Truncate(file.GetSize()) != nil { - // fallback to full cache - _, _ = f.Close(), os.Remove(f.Name()) - cache, err := file.CacheFullAndWriter(up, nil) - if err != nil { - return nil, err - } - return &cachedSectionReader{cache}, nil - } - - ss := &fileSectionReader{file: file, temp: f} - ss.bufPool = &pool.Pool[*offsetWriterWithBase]{ - New: func() *offsetWriterWithBase { - base := ss.tempOffset - ss.tempOffset += int64(maxBufferSize) - return &offsetWriterWithBase{io.NewOffsetWriter(ss.temp, base), base} - }, - } - file.Add(utils.CloseFunc(func() error { - ss.bufPool.Reset() - return errors.Join(ss.temp.Close(), os.Remove(ss.temp.Name())) - })) - return ss, nil - } + // 始终使用 directSectionReader，只在内存中缓存当前分片 + // 避免创建临时文件导致中间文件增长到整个文件大小 ss := &directSectionReader{file: file} if conf.MmapThreshold > 0 && maxBufferSize >= conf.MmapThreshold { ss.bufPool = &pool.Pool[[]byte]{ From 3b2f9d550dbd483e0255ec8c43f6689896b0926e Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 28 Dec 2025 18:17:05 +0800 Subject: [PATCH 32/86] fix(driver): fix file copy failure to 123pan due to incorrect etag fix(driver): improve etag handling for file uploads fix(driver): optimize SHA1 calculation for file uploads using chunked reading --- drivers/115_open/driver.go | 3 ++- drivers/123_open/driver.go | 39 ++++++++++++++++++++++++++++++++------ internal/stream/stream.go | 32 ++++++++----------------------- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index ec76a6bc8..1b5b43334 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -228,7 +228,8 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } sha1 := file.GetHash().GetHash(utils.SHA1) if len(sha1) != utils.SHA1.Width { - _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + // 流式计算SHA1 + sha1, err = stream.StreamHashFile(file, utils.SHA1, 100, &up) if err != nil { return err } diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 78ff272b9..d04bac59b 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -175,7 +175,7 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { } func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // 1. 创建文件 + // 1. 准备参数 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { @@ -197,14 +197,38 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } + // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) - if len(etag) < utils.MD5.Width { - _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) + if len(etag) >= utils.MD5.Width { + // 有etag时，先尝试秒传 + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err } + // 是否秒传 + if createResp.Data.Reuse { + // 秒传成功才会返回正确的 FileID，否则为 0 + if createResp.Data.FileID != 0 { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: createResp.Data.FileID, + Type: 2, + Etag: etag, + }, nil + } + } + // 秒传失败，etag可能不可靠，继续流式计算真实MD5 + } + + // 流式MD5计算 + etag, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + if err != nil { + return nil, err } + + // 2. 创建上传任务 createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err @@ -223,13 +247,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } - // 2. 上传分片 - err = d.Upload(ctx, file, createResp, up) + // 3. 上传分片 + uploadProgress := func(p float64) { + up(40 + p*0.6) + } + err = d.Upload(ctx, file, createResp, uploadProgress) if err != nil { return nil, err } - // 3. 上传完毕 + // 4. 合并分片/完成上传 for range 60 { uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) // 返回错误代码未知，如：20103，文档也没有具体说 diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 4c8238100..c29dbbec3 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -211,7 +211,9 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { return io.NewSectionReader(f.GetFile(), httpRange.Start, httpRange.Length), nil } - cache, err := f.cache(httpRange.Start + httpRange.Length) + // 限制缓存大小，避免累积缓存整个文件 + maxCache := min(httpRange.Start+httpRange.Length, int64(conf.MaxBufferLimit)) + cache, err := f.cache(maxCache) if err != nil { return nil, err } @@ -224,31 +226,13 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { // 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大 // 确保指定大小的数据被缓存 +// 注意：此方法只缓存到 maxCacheSize，不会缓存整个文件 func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { + // 限制缓存大小，避免超大文件占用过多资源 + // 如果需要缓存整个文件，应该显式调用 CacheFullAndWriter if maxCacheSize > int64(conf.MaxBufferLimit) { - size := f.GetSize() - reader := f.Reader - if f.peekBuff != nil { - size -= f.peekBuff.Size() - reader = f.oriReader - } - tmpF, err := utils.CreateTempFile(reader, size) - if err != nil { - return nil, err - } - f.Add(utils.CloseFunc(func() error { - return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) - })) - if f.peekBuff != nil { - peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) - if err != nil { - return nil, err - } - f.Reader = peekF - return peekF, nil - } - f.Reader = tmpF - return tmpF, nil + // 不再创建整个文件的临时文件，只缓存到 MaxBufferLimit + maxCacheSize = int64(conf.MaxBufferLimit) } if f.peekBuff == nil { From 79946a787901ec9336a14b8767d0a118b22199bb Mon Sep 17 00:00:00 2001 From: cyk Date: Thu, 1 Jan 2026 19:03:17 +0800 Subject: [PATCH 33/86] feat: Implement streaming upload for Baidu Netdisk - Added `upload.go` to handle streaming uploads without temporary file caching. - Introduced `calculateHashesStream` for efficient MD5 hash calculation during upload. - Implemented `uploadChunksStream` for concurrent chunk uploads using `StreamSectionReader`. - Refactored `uploadSliceStream` to accept `io.ReadSeeker` for better flexibility. - Enhanced error handling for upload ID expiration with retry logic. - Updated documentation to reflect changes in upload process and architecture. fix(driver): optimize MD5 hash calculation and stream handling for uploads feat(upload): add error handling for upload URL refresh on network errors feat(link): add retry logic with timeout for HEAD requests in linkOfficial function --- drivers/baidu_netdisk/driver.go | 214 +++-------------------- drivers/baidu_netdisk/upload.go | 299 ++++++++++++++++++++++++++++++++ drivers/baidu_netdisk/util.go | 19 +- 3 files changed, 345 insertions(+), 187 deletions(-) create mode 100644 drivers/baidu_netdisk/upload.go diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index fe77aca38..474dd2b98 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -1,30 +1,18 @@ package baidu_netdisk import ( - "bytes" "context" - "crypto/md5" - "encoding/hex" "errors" - "io" - "mime/multipart" - "net/http" "net/url" - "os" stdpath "path" "strconv" - "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/internal/net" - "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/avast/retry-go" log "github.com/sirupsen/logrus" ) @@ -37,6 +25,7 @@ type BaiduNetdisk struct { } var ErrUploadIDExpired = errors.New("uploadid expired") +var ErrUploadURLExpired = errors.New("upload url expired or unavailable") func (d *BaiduNetdisk) Config() driver.Config { return config @@ -199,80 +188,26 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return newObj, nil } - var ( - cache = stream.GetFile() - tmpF *os.File - err error - ) - if cache == nil { - tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - defer func() { - _ = tmpF.Close() - _ = os.Remove(tmpF.Name()) - }() - cache = tmpF - } - streamSize := stream.GetSize() sliceSize := d.getSliceSize(streamSize) count := 1 if streamSize > sliceSize { count = int((streamSize + sliceSize - 1) / sliceSize) } - lastBlockSize := streamSize % sliceSize - if lastBlockSize == 0 { - lastBlockSize = sliceSize - } - - // cal md5 for first 256k data - const SliceSize int64 = 256 * utils.KB - blockList := make([]string, 0, count) - byteSize := sliceSize - fileMd5H := md5.New() - sliceMd5H := md5.New() - sliceMd5H2 := md5.New() - slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) - writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} - if tmpF != nil { - writers = append(writers, tmpF) - } - written := int64(0) - for i := 1; i <= count; i++ { - if utils.IsCanceled(ctx) { - return nil, ctx.Err() - } - if i == count { - byteSize = lastBlockSize - } - n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) - written += n - if err != nil && err != io.EOF { - return nil, err - } - blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) - sliceMd5H.Reset() - } - if tmpF != nil { - if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) - } - _, err = tmpF.Seek(0, io.SeekStart) - if err != nil { - return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") - } - } - contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) - sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) - blockListStr, _ := utils.Json.MarshalToString(blockList) path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 尝试读取已保存进度 + // step.1 流式计算MD5哈希值（使用 RangeRead，不会消耗流） + contentMd5, sliceMd5, blockList, err := d.calculateHashesStream(ctx, stream, sliceSize, &up) + if err != nil { + return nil, err + } + + blockListStr, _ := utils.Json.MarshalToString(blockList) + + // step.2 尝试读取已保存进度或执行预上传 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { // 没有进度，走预上传 @@ -288,6 +223,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return fileToObj(precreateResp.File), nil } } + ensureUploadURL := func() { if precreateResp.UploadURL != "" { return @@ -295,58 +231,20 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) } - // step.2 上传分片 + // step.3 流式上传分片 + // 创建 StreamSectionReader 用于上传 + ss, err := streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } + uploadLoop: for range 2 { // 获取上传域名 ensureUploadURL() - // 并发上传 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(UPLOAD_RETRY_COUNT), - retry.Delay(UPLOAD_RETRY_WAIT_TIME), - retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), - retry.DelayType(retry.BackOffDelay), - retry.RetryIf(func(err error) bool { - return !errors.Is(err, ErrUploadIDExpired) - }), - retry.LastErrorOnly(true)) - - totalParts := len(precreateResp.BlockList) - - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break - } - if partseq < 0 { - continue - } - i, partseq := i, partseq - offset, size := int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - size = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), - } - section := io.NewSectionReader(cache, offset, size) - err := d.uploadSlice(ctx, precreateResp.UploadURL, params, stream.GetName(), section) - if err != nil { - return err - } - precreateResp.BlockList[i] = -1 - progress := float64(threadG.Success()+1) * 100 / float64(totalParts+1) - up(progress) - return nil - }) - } - err = threadG.Wait() + // 流式并发上传 + err = d.uploadChunksStream(ctx, ss, stream, precreateResp, path, sliceSize, count, up) if err == nil { break uploadLoop } @@ -372,13 +270,19 @@ uploadLoop: precreateResp.UploadURL = "" // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + + // 尝试重新创建 StreamSectionReader（如果流支持重新读取） + ss, err = streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } continue uploadLoop } return nil, err } defer up(100) - // step.3 创建文件 + // step.4 创建文件 var newFile File _, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) if err != nil { @@ -427,68 +331,6 @@ func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize in return &precreateResp, nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file *io.SectionReader) error { - b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) - mw := multipart.NewWriter(b) - _, err := mw.CreateFormFile("file", fileName) - if err != nil { - return err - } - headSize := b.Len() - err = mw.Close() - if err != nil { - return err - } - head := bytes.NewReader(b.Bytes()[:headSize]) - tail := bytes.NewReader(b.Bytes()[headSize:]) - rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) - if err != nil { - return err - } - query := req.URL.Query() - for k, v := range params { - query.Set(k, v) - } - req.URL.RawQuery = query.Encode() - req.Header.Set("Content-Type", mw.FormDataContentType()) - req.ContentLength = int64(b.Len()) + file.Size() - - client := net.NewHttpClient() - if d.UploadSliceTimeout > 0 { - client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) - } else { - client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT - } - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - b.Reset() - _, err = b.ReadFrom(resp.Body) - if err != nil { - return err - } - body := b.Bytes() - respStr := string(body) - log.Debugln(respStr) - lower := strings.ToLower(respStr) - // 合并 uploadid 过期检测逻辑 - if strings.Contains(lower, "uploadid") && - (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { - return ErrUploadIDExpired - } - - errCode := utils.Json.Get(body, "error_code").ToInt() - errNo := utils.Json.Get(body, "errno").ToInt() - if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) - } - return nil -} - func (d *BaiduNetdisk) GetDetails(ctx context.Context) (*model.StorageDetails, error) { du, err := d.quota(ctx) if err != nil { diff --git a/drivers/baidu_netdisk/upload.go b/drivers/baidu_netdisk/upload.go new file mode 100644 index 000000000..d3edec528 --- /dev/null +++ b/drivers/baidu_netdisk/upload.go @@ -0,0 +1,299 @@ +package baidu_netdisk + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "errors" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" +) + +// calculateHashesStream 流式计算文件的MD5哈希值 +// 返回：文件MD5、前256KB的MD5、每个分片的MD5列表 +// 注意：此函数使用 RangeRead 读取数据，不会消耗流 +func (d *BaiduNetdisk) calculateHashesStream( + ctx context.Context, + stream model.FileStreamer, + sliceSize int64, + up *driver.UpdateProgress, +) (contentMd5 string, sliceMd5 string, blockList []string, err error) { + streamSize := stream.GetSize() + count := 1 + if streamSize > sliceSize { + count = int((streamSize + sliceSize - 1) / sliceSize) + } + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 前256KB的MD5 + const SliceSize int64 = 256 * utils.KB + blockList = make([]string, 0, count) + fileMd5H := md5.New() + sliceMd5H2 := md5.New() + sliceWritten := int64(0) + + for i := 0; i < count; i++ { + if utils.IsCanceled(ctx) { + return "", "", nil, ctx.Err() + } + + offset := int64(i) * sliceSize + length := sliceSize + if i == count-1 { + length = lastBlockSize + } + + // 使用 RangeRead 读取数据，不会消耗流 + reader, err := stream.RangeRead(http_range.Range{Start: offset, Length: length}) + if err != nil { + return "", "", nil, err + } + + // 计算分片MD5 + sliceMd5Calc := md5.New() + + // 同时写入多个哈希计算器 + writers := []io.Writer{fileMd5H, sliceMd5Calc} + if sliceWritten < SliceSize { + remaining := SliceSize - sliceWritten + writers = append(writers, utils.LimitWriter(sliceMd5H2, remaining)) + } + + n, err := io.Copy(io.MultiWriter(writers...), reader) + // 关闭 reader（如果是 ReadCloser） + if rc, ok := reader.(io.Closer); ok { + rc.Close() + } + if err != nil { + return "", "", nil, err + } + sliceWritten += n + + blockList = append(blockList, hex.EncodeToString(sliceMd5Calc.Sum(nil))) + + // 更新进度（哈希计算占总进度的一小部分） + if up != nil { + progress := float64(i+1) * 10 / float64(count) + (*up)(progress) + } + } + + return hex.EncodeToString(fileMd5H.Sum(nil)), + hex.EncodeToString(sliceMd5H2.Sum(nil)), + blockList, nil +} + +// uploadChunksStream 流式上传所有分片 +func (d *BaiduNetdisk) uploadChunksStream( + ctx context.Context, + ss streamPkg.StreamSectionReaderIF, + stream model.FileStreamer, + precreateResp *PrecreateResp, + path string, + sliceSize int64, + count int, + up driver.UpdateProgress, +) error { + streamSize := stream.GetSize() + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 使用 OrderedGroup 保证 Before 阶段有序 + thread := min(d.uploadThread, len(precreateResp.BlockList)) + threadG, upCtx := errgroup.NewOrderedGroupWithContext(ctx, thread, + retry.Attempts(UPLOAD_RETRY_COUNT), + retry.Delay(UPLOAD_RETRY_WAIT_TIME), + retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), + retry.DelayType(retry.BackOffDelay), + retry.RetryIf(func(err error) bool { + return !errors.Is(err, ErrUploadIDExpired) + }), + retry.OnRetry(func(n uint, err error) { + // 重试前检测是否需要刷新上传 URL + if errors.Is(err, ErrUploadURLExpired) { + log.Infof("[baidu_netdisk] refreshing upload URL due to error: %v", err) + precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) + } + }), + retry.LastErrorOnly(true)) + + totalParts := len(precreateResp.BlockList) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) { + break + } + if partseq < 0 { + continue + } + + i, partseq := i, partseq + offset := int64(partseq) * sliceSize + size := sliceSize + if partseq+1 == count { + size = lastBlockSize + } + + var reader io.ReadSeeker + + threadG.GoWithLifecycle(errgroup.Lifecycle{ + Before: func(ctx context.Context) error { + var err error + reader, err = ss.GetSectionReader(offset, size) + return err + }, + Do: func(ctx context.Context) error { + reader.Seek(0, io.SeekStart) + err := d.uploadSliceStream(ctx, precreateResp.UploadURL, path, + precreateResp.Uploadid, partseq, stream.GetName(), reader, size) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 进度从10%开始（前10%是哈希计算） + progress := 10 + float64(threadG.Success()+1)*90/float64(totalParts+1) + up(progress) + return nil + }, + After: func(err error) { + ss.FreeSectionReader(reader) + }, + }) + } + + return threadG.Wait() +} + +// uploadSliceStream 上传单个分片（接受io.ReadSeeker） +func (d *BaiduNetdisk) uploadSliceStream( + ctx context.Context, + uploadUrl string, + path string, + uploadid string, + partseq int, + fileName string, + reader io.ReadSeeker, + size int64, +) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": uploadid, + "partseq": strconv.Itoa(partseq), + } + + b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) + mw := multipart.NewWriter(b) + _, err := mw.CreateFormFile("file", fileName) + if err != nil { + return err + } + headSize := b.Len() + err = mw.Close() + if err != nil { + return err + } + head := bytes.NewReader(b.Bytes()[:headSize]) + tail := bytes.NewReader(b.Bytes()[headSize:]) + rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) + if err != nil { + return err + } + query := req.URL.Query() + for k, v := range params { + query.Set(k, v) + } + req.URL.RawQuery = query.Encode() + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(b.Len()) + size + + client := net.NewHttpClient() + if d.UploadSliceTimeout > 0 { + client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) + } else { + client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT + } + resp, err := client.Do(req) + if err != nil { + // 检测超时或网络错误，标记需要刷新上传 URL + if isUploadURLError(err) { + log.Warnf("[baidu_netdisk] upload slice failed with network error: %v, will refresh upload URL", err) + return errors.Join(err, ErrUploadURLExpired) + } + return err + } + defer resp.Body.Close() + b.Reset() + _, err = b.ReadFrom(resp.Body) + if err != nil { + return err + } + body := b.Bytes() + respStr := string(body) + log.Debugln(respStr) + lower := strings.ToLower(respStr) + // 合并 uploadid 过期检测逻辑 + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() + if errCode != 0 || errNo != 0 { + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) + } + return nil +} + +// isUploadURLError 判断是否为需要刷新上传 URL 的错误 +// 包括：超时、连接被拒绝、连接重置、DNS 解析失败等网络错误 +func isUploadURLError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + // 超时错误 + if strings.Contains(errStr, "timeout") || + strings.Contains(errStr, "deadline exceeded") { + return true + } + // 连接错误 + if strings.Contains(errStr, "connection refused") || + strings.Contains(errStr, "connection reset") || + strings.Contains(errStr, "no such host") || + strings.Contains(errStr, "network is unreachable") { + return true + } + // EOF 错误（连接被服务器关闭） + if strings.Contains(errStr, "eof") || + strings.Contains(errStr, "broken pipe") { + return true + } + return false +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 0e27fb305..75018a708 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -207,7 +207,24 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Li return nil, err } u := fmt.Sprintf("%s&access_token=%s", resp.List[0].Dlink, d.AccessToken) - res, err := base.NoRedirectClient.R().SetHeader("User-Agent", "pan.baidu.com").Head(u) + + // Retry HEAD request with longer timeout to avoid client-side errors + // Create a client with longer timeout (base.NoRedirectClient doesn't have timeout set) + client := base.NoRedirectClient.SetTimeout(60 * time.Second) + var res *resty.Response + maxRetries := 5 + for i := 0; i < maxRetries; i++ { + res, err = client.R(). + SetHeader("User-Agent", "pan.baidu.com"). + Head(u) + if err == nil { + break + } + if i < maxRetries-1 { + log.Warnf("HEAD request failed (attempt %d/%d): %v, retrying...", i+1, maxRetries, err) + time.Sleep(time.Duration(i+1) * 2 * time.Second) // Exponential backoff: 2s, 4s, 6s, 8s + } + } if err != nil { return nil, err } From f6379279454db426a418b15c4d0f95c399f72b01 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 4 Jan 2026 16:11:15 +0800 Subject: [PATCH 34/86] feat(upload): enhance hash calculation and upload logic for various stream types feat(upload): enhance token handling and bucket creation for OSS uploads --- CLAUDE.md | 49 +++++++++++++ drivers/115_open/driver.go | 105 +++++++++++++++++++++++----- drivers/115_open/upload.go | 44 ++++++++++-- drivers/123_open/driver.go | 49 +++++++------ drivers/aliyundrive_open/upload.go | 41 +++++++---- drivers/openlist/driver.go | 106 +++++++++++++++++++++++++++-- internal/stream/stream.go | 12 +--- internal/stream/util.go | 100 +++++++++++++++++++++------ pkg/utils/hash.go | 6 ++ server/handles/fsup.go | 12 ++++ 10 files changed, 433 insertions(+), 91 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6a1e1461c..c3ffdded0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,12 @@ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +## Core Development Principles + +1. **最小代码改动原则** (Minimum code changes): Make the smallest change necessary to achieve the goal +2. **不缓存整个文件原则** (No full file caching for seekable streams): For SeekableStream, use RangeRead instead of caching entire file +3. **必要情况下可以多遍上传原则** (Multi-pass upload when necessary): If rapid upload fails, fall back to normal upload + ## Build and Development Commands ```bash @@ -166,6 +172,49 @@ Response (JSON / Proxy / Redirect) - Downloads parts in parallel - Assembles final stream +**Stream Types and Reader Management**: + +⚠️ **CRITICAL**: SeekableStream.Reader must NEVER be created early! + +- **FileStream**: One-time sequential stream (e.g., HTTP body) + - `Reader` is set at creation and consumed sequentially + - Cannot be rewound or re-read + +- **SeekableStream**: Reusable stream with RangeRead capability + - Has `rangeReader` for creating new readers on-demand + - `Reader` should ONLY be created when actually needed for sequential reading + - **DO NOT create Reader early** - use lazy initialization via `generateReader()` + +**Common Pitfall - Early Reader Creation**: +```go +// ❌ WRONG: Creating Reader early +if _, ok := rr.(*model.FileRangeReader); ok { + rc, _ := rr.RangeRead(ctx, http_range.Range{Length: -1}) + fs.Reader = rc // This will be consumed by intermediate operations! +} + +// ✅ CORRECT: Let generateReader() create it on-demand +// Reader will be created only when Read() is called +return &SeekableStream{FileStream: fs, rangeReader: rr}, nil +``` + +**Why This Matters**: +- Hash calculation uses `StreamHashFile()` which reads the file via RangeRead +- If Reader is created early, it may be at EOF when HTTP upload actually needs it +- Result: `http: ContentLength=X with Body length 0` error + +**Hash Calculation for Uploads**: +```go +// For SeekableStream: Use RangeRead to avoid consuming Reader +if _, ok := file.(*SeekableStream); ok { + hash, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + // StreamHashFile uses RangeRead internally, Reader remains unused +} + +// For FileStream: Must cache first, then calculate hash +_, hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) +``` + **Link Refresh Pattern**: ```go // In op.Link(), a refresher is automatically attached diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 1b5b43334..03d29efe6 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -226,28 +226,97 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } + sha1 := file.GetHash().GetHash(utils.SHA1) - if len(sha1) != utils.SHA1.Width { - // 流式计算SHA1 - sha1, err = stream.StreamHashFile(file, utils.SHA1, 100, &up) + sha1128k := file.GetHash().GetHash(utils.SHA1_128K) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ + FileName: file.GetName(), + FileSize: file.GetSize(), + Target: dstDir.GetID(), + FileID: strings.ToUpper(sha1), + PreID: strings.ToUpper(sha1128k), + }) if err != nil { return err } + if resp.Status == 2 { + up(100) + return nil + } + // 秒传失败，继续后续流程 } - const PreHashSize int64 = 128 * utils.KB - hashSize := PreHashSize - if file.GetSize() < PreHashSize { - hashSize = file.GetSize() - } - reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) - if err != nil { - return err - } - sha1128k, err := utils.HashReader(utils.SHA1, reader) - if err != nil { - return err + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(sha1) != utils.SHA1.Width { + sha1, err = stream.StreamHashFile(file, utils.SHA1, 100, &up) + if err != nil { + return err + } + } + // 计算 sha1_128k（如果没有预计算） + if len(sha1128k) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 如果有预计算的 hash，上面已经尝试过秒传了 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + // 秒传失败，需要缓存文件进行实际上传 + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } else { + // 没有预计算的 hash，缓存整个文件并计算 + if len(sha1) != utils.SHA1.Width { + _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + if err != nil { + return err + } + } else if file.GetFile() == nil { + // 有 SHA1 但没有缓存，需要缓存以支持后续 RangeRead + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + // 计算 sha1_128k + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } } - // 1. Init + + // 1. Init（SeekableStream 或已缓存的 FileStream） resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), @@ -273,11 +342,11 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } - reader, err = file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) + signReader, err := file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) if err != nil { return err } - signVal, err := utils.HashReader(utils.SHA1, reader) + signVal, err := utils.HashReader(utils.SHA1, signReader) if err != nil { return err } diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index d02640e2c..6af4403cf 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "io" + "strings" "time" sdk "github.com/OpenListTeam/115-sdk-go" @@ -14,8 +15,19 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" ) +// isTokenExpiredError 检测是否为OSS凭证过期错误 +func isTokenExpiredError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "SecurityTokenExpired") || + strings.Contains(errStr, "InvalidAccessKeyId") +} + func calPartSize(fileSize int64) int64 { var partSize int64 = 20 * utils.MB if fileSize > partSize { @@ -71,11 +83,16 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp // } func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { - ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) - if err != nil { - return err + // 创建OSS客户端的辅助函数 + createBucket := func(token *sdk.UploadGetTokenResp) (*oss.Bucket, error) { + ossClient, err := netutil.NewOSSClient(token.Endpoint, token.AccessKeyId, token.AccessKeySecret, oss.SecurityToken(token.SecurityToken)) + if err != nil { + return nil, err + } + return ossClient.Bucket(initResp.Bucket) } - bucket, err := ossClient.Bucket(initResp.Bucket) + + bucket, err := createBucket(tokenResp) if err != nil { return err } @@ -120,7 +137,24 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), - retry.Delay(time.Second)) + retry.Delay(time.Second), + retry.OnRetry(func(n uint, err error) { + // 如果是凭证过期错误，在重试前刷新凭证并重建bucket + if isTokenExpiredError(err) { + log.Warnf("115 OSS token expired, refreshing token...") + if newToken, refreshErr := d.client.UploadGetToken(ctx); refreshErr == nil { + tokenResp = newToken + if newBucket, bucketErr := createBucket(tokenResp); bucketErr == nil { + bucket = newBucket + log.Infof("115 OSS token refreshed successfully") + } else { + log.Errorf("Failed to create new bucket with refreshed token: %v", bucketErr) + } + } else { + log.Errorf("Failed to refresh 115 OSS token: %v", refreshErr) + } + } + })) ss.FreeSectionReader(rd) if err != nil { return err diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index d04bac59b..d125a8705 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -200,35 +200,46 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 if len(etag) >= utils.MD5.Width { - // 有etag时，先尝试秒传 createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err } - // 是否秒传 - if createResp.Data.Reuse { - // 秒传成功才会返回正确的 FileID，否则为 0 - if createResp.Data.FileID != 0 { - return File{ - FileName: file.GetName(), - Size: file.GetSize(), - FileId: createResp.Data.FileID, - Type: 2, - Etag: etag, - }, nil - } + if createResp.Data.Reuse && createResp.Data.FileID != 0 { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: createResp.Data.FileID, + Type: 2, + Etag: etag, + }, nil } - // 秒传失败，etag可能不可靠，继续流式计算真实MD5 + // 秒传失败，继续后续流程 } - // 流式MD5计算 - etag, err = stream.StreamHashFile(file, utils.MD5, 40, &up) - if err != nil { - return nil, err + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(etag) < utils.MD5.Width { + etag, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + if err != nil { + return nil, err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 秒传失败或没有 hash，缓存整个文件并计算 MD5 + _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) + if err != nil { + return nil, err + } } - // 2. 创建上传任务 + // 2. 创建上传任务（或再次尝试秒传） createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index a4a6c1de1..00c806e5f 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -163,21 +163,29 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m } count := int(math.Ceil(float64(stream.GetSize()) / float64(partSize))) createData["part_info_list"] = makePartInfos(count) + + // 检查是否是可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + // rapid upload rapidUpload := !stream.IsForceStreamUpload() && stream.GetSize() > 100*utils.KB && d.RapidUpload if rapidUpload { log.Debugf("[aliyundrive_open] start cal pre_hash") - // read 1024 bytes to calculate pre hash - reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) - if err != nil { - return nil, err - } - hash, err := utils.HashReader(utils.SHA1, reader) - if err != nil { - return nil, err + // 优先使用预计算的 pre_hash + preHash := stream.GetHash().GetHash(utils.PRE_HASH) + if len(preHash) != utils.PRE_HASH.Width { + // 没有预计算的 pre_hash，使用 RangeRead 计算 + reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) + if err != nil { + return nil, err + } + preHash, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return nil, err + } } createData["size"] = stream.GetSize() - createData["pre_hash"] = hash + createData["pre_hash"] = preHash } var createResp CreateResp _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { @@ -191,9 +199,18 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m hash := stream.GetHash().GetHash(utils.SHA1) if len(hash) != utils.SHA1.Width { - _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) - if err != nil { - return nil, err + if isSeekable { + // 可重复读取的流，使用 StreamHashFile（RangeRead），不缓存 + hash, err = streamPkg.StreamHashFile(stream, utils.SHA1, 50, &up) + if err != nil { + return nil, err + } + } else { + // 不可重复读取的流，缓存并计算 + _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) + if err != nil { + return nil, err + } } } diff --git a/drivers/openlist/driver.go b/drivers/openlist/driver.go index 79fc51185..b37d72a06 100644 --- a/drivers/openlist/driver.go +++ b/drivers/openlist/driver.go @@ -14,6 +14,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" @@ -195,6 +197,92 @@ func (d *OpenList) Remove(ctx context.Context, obj model.Obj) error { } func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + // 预计算 hash（如果不存在），使用 RangeRead 不消耗 Reader + // 这样远端驱动不需要再计算，避免 HTTP body 被重复读取 + md5Hash := s.GetHash().GetHash(utils.MD5) + sha1Hash := s.GetHash().GetHash(utils.SHA1) + sha256Hash := s.GetHash().GetHash(utils.SHA256) + sha1_128kHash := s.GetHash().GetHash(utils.SHA1_128K) + preHash := s.GetHash().GetHash(utils.PRE_HASH) + + // 计算所有缺失的 hash，确保最大兼容性 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(s, utils.MD5, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate MD5: %v", err) + md5Hash = "" + } + } + if len(sha1Hash) != utils.SHA1.Width { + var err error + sha1Hash, err = stream.StreamHashFile(s, utils.SHA1, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1: %v", err) + sha1Hash = "" + } + } + if len(sha256Hash) != utils.SHA256.Width { + var err error + sha256Hash, err = stream.StreamHashFile(s, utils.SHA256, 34, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA256: %v", err) + sha256Hash = "" + } + } + + // 计算特殊 hash（用于秒传验证） + // SHA1_128K: 前128KB的SHA1，115网盘使用 + if len(sha1_128kHash) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * 1024 // 128KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + sha1_128kHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1_128K: %v", err) + sha1_128kHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for SHA1_128K: %v", err) + } + } + + // PRE_HASH: 前1024字节的SHA1，阿里云盘使用 + if len(preHash) != utils.PRE_HASH.Width { + const PreHashSize int64 = 1024 // 1KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + preHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate PRE_HASH: %v", err) + preHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for PRE_HASH: %v", err) + } + } + + // 诊断日志：检查流的状态 + if ss, ok := s.(*stream.SeekableStream); ok { + if ss.Reader != nil { + log.Warnf("[openlist] WARNING: SeekableStream.Reader is not nil for file %s, stream may have been consumed!", s.GetName()) + } + } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, @@ -206,14 +294,20 @@ func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStream req.Header.Set("Authorization", d.Token) req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) req.Header.Set("Password", d.MetaPassword) - if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { - req.Header.Set("X-File-Md5", md5) + if len(md5Hash) > 0 { + req.Header.Set("X-File-Md5", md5Hash) + } + if len(sha1Hash) > 0 { + req.Header.Set("X-File-Sha1", sha1Hash) + } + if len(sha256Hash) > 0 { + req.Header.Set("X-File-Sha256", sha256Hash) } - if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { - req.Header.Set("X-File-Sha1", sha1) + if len(sha1_128kHash) > 0 { + req.Header.Set("X-File-Sha1-128k", sha1_128kHash) } - if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { - req.Header.Set("X-File-Sha256", sha256) + if len(preHash) > 0 { + req.Header.Set("X-File-Pre-Hash", preHash) } req.ContentLength = s.GetSize() diff --git a/internal/stream/stream.go b/internal/stream/stream.go index c29dbbec3..7eec75dd9 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -299,15 +299,9 @@ func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error if err != nil { return nil, err } - if _, ok := rr.(*model.FileRangeReader); ok { - var rc io.ReadCloser - rc, err = rr.RangeRead(fs.Ctx, http_range.Range{Length: -1}) - if err != nil { - return nil, err - } - fs.Reader = rc - fs.Add(rc) - } + // IMPORTANT: Do NOT create Reader early for FileRangeReader! + // Let generateReader() create it on-demand when actually needed for reading + // This prevents the Reader from being consumed by intermediate operations like hash calculation fs.size = size fs.Add(link) return &SeekableStream{FileStream: fs, rangeReader: rr}, nil diff --git a/internal/stream/util.go b/internal/stream/util.go index 83b20da71..1ee9f7d99 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "os" + "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -174,6 +175,45 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } +// readFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf +// file: 文件流 +// buf: 目标缓冲区 +// off: 读取的起始偏移量 +// 返回值: 实际读取的字节数和错误 +// 支持自动重试（最多3次），每次重试之间有递增延迟（3秒、6秒、9秒） +func readFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { + length := int64(len(buf)) + var lastErr error + + // 重试最多3次 + for retry := 0; retry < 3; retry++ { + reader, err := file.RangeRead(http_range.Range{Start: off, Length: length}) + if err != nil { + lastErr = fmt.Errorf("RangeRead failed at offset %d: %w", off, err) + log.Debugf("RangeRead retry %d failed: %v", retry+1, lastErr) + // 递增延迟：3秒、6秒、9秒，等待代理恢复 + time.Sleep(time.Duration(retry+1) * 3 * time.Second) + continue + } + + n, err := io.ReadFull(reader, buf) + if closer, ok := reader.(io.Closer); ok { + closer.Close() + } + + if err == nil { + return n, nil + } + + lastErr = fmt.Errorf("failed to read all data via RangeRead at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) + log.Debugf("RangeRead retry %d read failed: %v", retry+1, lastErr) + // 递增延迟：3秒、6秒、9秒，等待网络恢复 + time.Sleep(time.Duration(retry+1) * 3 * time.Second) + } + + return 0, lastErr +} + // StreamHashFile 流式计算文件哈希值，避免将整个文件加载到内存 // file: 文件流 // hashType: 哈希算法类型 @@ -197,36 +237,38 @@ func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressW hashFunc := hashType.NewFunc() size := file.GetSize() chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk + buf := make([]byte, chunkSize) var offset int64 = 0 - const maxRetries = 3 + for offset < size { readSize := chunkSize if size-offset < chunkSize { readSize = size - offset } - var lastErr error - for retry := 0; retry < maxRetries; retry++ { - reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize}) + var n int + var err error + + // 对于 SeekableStream，优先使用 RangeRead 避免消耗 Reader + // 这样后续发送时 Reader 还能正常工作 + if _, ok := file.(*SeekableStream); ok { + n, err = readFullWithRangeRead(file, buf[:readSize], offset) + } else { + // 对于 FileStream，首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） + n, err = io.ReadFull(file, buf[:readSize]) if err != nil { - lastErr = fmt.Errorf("range read for hash calculation failed: %w", err) - continue - } - _, err = io.Copy(hashFunc, reader) - if closer, ok := reader.(io.Closer); ok { - closer.Close() + // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） + log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) + n, err = readFullWithRangeRead(file, buf[:readSize], offset) } - if err == nil { - lastErr = nil - break - } - lastErr = fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) } - if lastErr != nil { - return "", lastErr + + if err != nil { + return "", fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) } - offset += readSize + hashFunc.Write(buf[:n]) + offset += int64(n) if up != nil && progressWeight > 0 { progress := progressWeight * float64(offset) / float64(size) @@ -381,12 +423,26 @@ func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeke } tempBuf := ss.bufPool.Get() buf := tempBuf[:length] + + // 首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） + // 对于 FileStream，RangeRead 会消耗底层 oriReader，所以必须先尝试顺序流读取 n, err := io.ReadFull(ss.file, buf) - ss.fileOffset += int64(n) - if int64(n) != length { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) + if err == nil { + ss.fileOffset = off + int64(n) + return &bufferSectionReader{bytes.NewReader(buf), tempBuf}, nil + } + + // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） + log.Debugf("Sequential read failed at offset %d, retrying with RangeRead: %v", off, err) + n, err = readFullWithRangeRead(ss.file, buf, off) + if err != nil { + ss.bufPool.Put(tempBuf) + return nil, fmt.Errorf("both sequential read and RangeRead failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) } - return &bufferSectionReader{bytes.NewReader(buf), buf}, nil + + // 更新 fileOffset + ss.fileOffset = off + int64(n) + return &bufferSectionReader{bytes.NewReader(buf), tempBuf}, nil } func (ss *directSectionReader) FreeSectionReader(rs io.ReadSeeker) { if sr, ok := rs.(*bufferSectionReader); ok { diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index 596e61e54..c4b4e735f 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -90,6 +90,12 @@ var ( // SHA256 indicates SHA-256 support SHA256 = RegisterHash("sha256", "SHA-256", 64, sha256.New) + + // SHA1_128K is SHA1 of first 128KB, used by 115 driver for rapid upload + SHA1_128K = RegisterHash("sha1_128k", "SHA1-128K", 40, sha1.New) + + // PRE_HASH is SHA1 of first 1024 bytes, used by Aliyundrive for rapid upload + PRE_HASH = RegisterHash("pre_hash", "PRE-HASH", 40, sha1.New) ) // HashData get hash of one hashType diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 0f46398cd..54cdb4fee 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -93,6 +93,12 @@ func FsStream(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := c.GetHeader("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) @@ -190,6 +196,12 @@ func FsForm(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := file.Header.Get("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) From ff64f84d937e6ea9a212731888c08eb73a181cce Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 4 Jan 2026 19:37:12 +0800 Subject: [PATCH 35/86] feat(link): add link refresh capability for expired download links feat(link): implement ForceRefreshLink method for refreshing download links on read failure --- drivers/baidu_netdisk/upload.go | 54 +++++--- internal/model/args.go | 8 ++ internal/op/fs.go | 22 ++++ internal/stream/stream.go | 8 ++ internal/stream/util.go | 225 +++++++++++++++++++++++++++++--- 5 files changed, 277 insertions(+), 40 deletions(-) diff --git a/drivers/baidu_netdisk/upload.go b/drivers/baidu_netdisk/upload.go index d3edec528..c160c3a9e 100644 --- a/drivers/baidu_netdisk/upload.go +++ b/drivers/baidu_netdisk/upload.go @@ -19,7 +19,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/net" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" - "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" log "github.com/sirupsen/logrus" @@ -51,6 +50,11 @@ func (d *BaiduNetdisk) calculateHashesStream( sliceMd5H2 := md5.New() sliceWritten := int64(0) + // 使用固定大小的缓冲区进行流式哈希计算 + // 这样可以利用 readFullWithRangeRead 的链接刷新逻辑 + const chunkSize = 10 * 1024 * 1024 // 10MB per chunk + buf := make([]byte, chunkSize) + for i := 0; i < count; i++ { if utils.IsCanceled(ctx) { return "", "", nil, ctx.Err() @@ -62,31 +66,39 @@ func (d *BaiduNetdisk) calculateHashesStream( length = lastBlockSize } - // 使用 RangeRead 读取数据，不会消耗流 - reader, err := stream.RangeRead(http_range.Range{Start: offset, Length: length}) - if err != nil { - return "", "", nil, err - } - // 计算分片MD5 sliceMd5Calc := md5.New() - // 同时写入多个哈希计算器 - writers := []io.Writer{fileMd5H, sliceMd5Calc} - if sliceWritten < SliceSize { - remaining := SliceSize - sliceWritten - writers = append(writers, utils.LimitWriter(sliceMd5H2, remaining)) - } + // 分块读取并计算哈希 + var sliceOffset int64 = 0 + for sliceOffset < length { + readSize := chunkSize + if length-sliceOffset < int64(chunkSize) { + readSize = int(length - sliceOffset) + } - n, err := io.Copy(io.MultiWriter(writers...), reader) - // 关闭 reader（如果是 ReadCloser） - if rc, ok := reader.(io.Closer); ok { - rc.Close() - } - if err != nil { - return "", "", nil, err + // 使用 readFullWithRangeRead 读取数据，自动处理链接刷新 + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset+sliceOffset) + if err != nil { + return "", "", nil, err + } + + // 同时写入多个哈希计算器 + fileMd5H.Write(buf[:n]) + sliceMd5Calc.Write(buf[:n]) + if sliceWritten < SliceSize { + remaining := SliceSize - sliceWritten + if int64(n) > remaining { + sliceMd5H2.Write(buf[:remaining]) + sliceWritten += remaining + } else { + sliceMd5H2.Write(buf[:n]) + sliceWritten += int64(n) + } + } + + sliceOffset += int64(n) } - sliceWritten += n blockList = append(blockList, hex.EncodeToString(sliceMd5Calc.Sum(nil))) diff --git a/internal/model/args.go b/internal/model/args.go index 073c94a63..d165908fb 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -25,6 +25,10 @@ type LinkArgs struct { Redirect bool } +// LinkRefresher is a callback function type for refreshing download links +// It returns a new Link and the associated object, or an error +type LinkRefresher func(ctx context.Context) (*Link, Obj, error) + type Link struct { URL string `json:"url"` // most common way Header http.Header `json:"header"` // needed header (for url) @@ -37,6 +41,10 @@ type Link struct { PartSize int `json:"part_size"` ContentLength int64 `json:"content_length"` // 转码视频、缩略图 + // Refresher is a callback to refresh the link when it expires during long downloads + // This field is not serialized and is optional - if nil, no refresh will be attempted + Refresher LinkRefresher `json:"-"` + utils.SyncClosers `json:"-"` // 如果SyncClosers中的资源被关闭后Link将不可用，则此值应为 true RequireReference bool `json:"-"` diff --git a/internal/op/fs.go b/internal/op/fs.go index 5116bbef5..2c91e6cf3 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -262,6 +262,28 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil { return nil, errors.Wrapf(err, "failed get link") } + + // Set up link refresher for automatic refresh on expiry during long downloads + // This enables all download scenarios to handle link expiration gracefully + if link.Refresher == nil { + storageCopy := storage + pathCopy := path + argsCopy := args + link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + log.Infof("Refreshing download link for: %s", pathCopy) + // Get fresh link directly from storage, bypassing cache + file, err := GetUnwrap(refreshCtx, storageCopy, pathCopy) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file for refresh") + } + newLink, err := storageCopy.Link(refreshCtx, file, argsCopy) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to refresh link") + } + return newLink, file, nil + } + } + ol := &objWithLink{link: link, obj: file} if link.Expiration != nil { Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 7eec75dd9..aaf310487 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -346,6 +346,14 @@ func (ss *SeekableStream) generateReader() error { return nil } +// ForceRefreshLink 实现 LinkRefresher 接口，用于在读取失败时刷新链接 +func (ss *SeekableStream) ForceRefreshLink(ctx context.Context) bool { + if rr, ok := ss.rangeReader.(*RefreshableRangeReader); ok { + return rr.ForceRefresh(ctx) + } + return false +} + func (ss *SeekableStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writer) (model.File, error) { if err := ss.generateReader(); err != nil { return nil, err diff --git a/internal/stream/util.go b/internal/stream/util.go index 1ee9f7d99..00c3bde52 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -9,6 +9,8 @@ import ( "io" "net/http" "os" + "strings" + "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -28,7 +30,157 @@ func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Ran return f(ctx, httpRange) } +// LinkRefresher 接口用于在读取数据失败时强制刷新链接 +type LinkRefresher interface { + // ForceRefreshLink 强制刷新下载链接 + // 返回 true 表示刷新成功，false 表示无法刷新 + ForceRefreshLink(ctx context.Context) bool +} + +// IsLinkExpiredError checks if the error indicates an expired download link +func IsLinkExpiredError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + + // Common expired link error keywords + expiredKeywords := []string{ + "expired", "invalid signature", "token expired", + "access denied", "forbidden", "unauthorized", + "link has expired", "url expired", "request has expired", + "signature expired", "accessdenied", "invalidtoken", + } + for _, keyword := range expiredKeywords { + if strings.Contains(errStr, keyword) { + return true + } + } + + // Check for HTTP status codes that typically indicate expired links + if statusErr, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { + code := int(statusErr) + // 401 Unauthorized, 403 Forbidden, 410 Gone are common for expired links + // 500 Internal Server Error - some providers (e.g., Baidu) return 500 when link expires + if code == 401 || code == 403 || code == 410 || code == 500 { + return true + } + } + + return false +} + +// RefreshableRangeReader wraps a RangeReader with link refresh capability +type RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // track refresh count to avoid infinite loops +} + +// NewRefreshableRangeReader creates a new RefreshableRangeReader +func NewRefreshableRangeReader(link *model.Link, size int64) *RefreshableRangeReader { + return &RefreshableRangeReader{ + link: link, + size: size, + } +} + +func (r *RefreshableRangeReader) getInnerReader() (model.RangeReaderIF, error) { + if r.innerReader != nil { + return r.innerReader, nil + } + + // Create inner reader without Refresher to avoid recursion + linkCopy := *r.link + linkCopy.Refresher = nil + + reader, err := GetRangeReaderFromLink(r.size, &linkCopy) + if err != nil { + return nil, err + } + r.innerReader = reader + return reader, nil +} + +func (r *RefreshableRangeReader) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + reader, err := r.getInnerReader() + r.mu.Unlock() + if err != nil { + return nil, err + } + + rc, err := reader.RangeRead(ctx, httpRange) + if err != nil { + // Check if we should try to refresh on initial connection error + if IsLinkExpiredError(err) && r.link.Refresher != nil { + rc, err = r.refreshAndRetry(ctx, httpRange) + } + if err != nil { + return nil, err + } + } + + return rc, nil +} + +func (r *RefreshableRangeReader) refreshAndRetry(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if err := r.doRefreshLocked(ctx); err != nil { + return nil, err + } + + reader, err := r.getInnerReader() + if err != nil { + return nil, err + } + return reader.RangeRead(ctx, httpRange) +} + +// ForceRefresh 强制刷新链接，用于读取数据失败（如读取 0 字节）的情况 +// 返回 true 表示刷新成功，false 表示无法刷新（没有 Refresher 或达到最大刷新次数） +func (r *RefreshableRangeReader) ForceRefresh(ctx context.Context) bool { + if r.link.Refresher == nil { + return false + } + + r.mu.Lock() + defer r.mu.Unlock() + + return r.doRefreshLocked(ctx) == nil +} + +// doRefreshLocked 执行实际的刷新逻辑（需要持有锁） +func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { + if r.refreshCount >= 3 { + return fmt.Errorf("max refresh attempts reached") + } + + log.Infof("Link expired, attempting to refresh...") + newLink, _, refreshErr := r.link.Refresher(ctx) + if refreshErr != nil { + return fmt.Errorf("failed to refresh link: %w", refreshErr) + } + + newLink.Refresher = r.link.Refresher + r.link = newLink + r.innerReader = nil + r.refreshCount++ + + log.Infof("Link refreshed successfully") + return nil +} + func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { + // If link has a Refresher, wrap with RefreshableRangeReader for automatic refresh on expiry + if link.Refresher != nil { + return NewRefreshableRangeReader(link, size), nil + } + if link.RangeReader != nil { if link.Concurrency < 1 && link.PartSize < 1 { return link.RangeReader, nil @@ -175,13 +327,14 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } -// readFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf +// ReadFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf // file: 文件流 // buf: 目标缓冲区 // off: 读取的起始偏移量 // 返回值: 实际读取的字节数和错误 // 支持自动重试（最多3次），每次重试之间有递增延迟（3秒、6秒、9秒） -func readFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { +// 支持链接刷新：当检测到 0 字节读取时，会自动刷新下载链接 +func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { length := int64(len(buf)) var lastErr error @@ -207,6 +360,28 @@ func readFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, lastErr = fmt.Errorf("failed to read all data via RangeRead at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) log.Debugf("RangeRead retry %d read failed: %v", retry+1, lastErr) + + // 检测是否可能是链接过期（读取 0 字节或 EOF） + if n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { + // 尝试刷新链接 + if refresher, ok := file.(LinkRefresher); ok { + // 获取 context - 从 FileStream 或 SeekableStream 中获取 + var ctx context.Context + if fs, ok := file.(*FileStream); ok { + ctx = fs.Ctx + } else if ss, ok := file.(*SeekableStream); ok { + ctx = ss.Ctx + } else { + ctx = context.Background() + } + + if refresher.ForceRefreshLink(ctx) { + log.Infof("Link refreshed after 0-byte read, retrying immediately...") + continue // 立即重试，不延迟 + } + } + } + // 递增延迟：3秒、6秒、9秒，等待网络恢复 time.Sleep(time.Duration(retry+1) * 3 * time.Second) } @@ -252,14 +427,14 @@ func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressW // 对于 SeekableStream，优先使用 RangeRead 避免消耗 Reader // 这样后续发送时 Reader 还能正常工作 if _, ok := file.(*SeekableStream); ok { - n, err = readFullWithRangeRead(file, buf[:readSize], offset) + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) } else { // 对于 FileStream，首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） n, err = io.ReadFull(file, buf[:readSize]) if err != nil { // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) - n, err = readFullWithRangeRead(file, buf[:readSize], offset) + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) } } @@ -398,8 +573,16 @@ type directSectionReader struct { bufPool *pool.Pool[[]byte] } -// 线程不安全 +// 线程不安全（依赖调用方保证串行调用） +// 对于 SeekableStream：直接跳过（无需实际读取） +// 对于 FileStream：必须顺序读取并丢弃 func (ss *directSectionReader) DiscardSection(off int64, length int64) error { + // 对于 SeekableStream，直接跳过（RangeRead 支持随机访问，不需要实际读取） + if _, ok := ss.file.(*SeekableStream); ok { + return nil + } + + // 对于 FileStream，必须顺序读取并丢弃 if off != ss.fileOffset { return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } @@ -416,31 +599,35 @@ type bufferSectionReader struct { buf []byte } -// 线程不安全 +// 线程不安全（依赖调用方保证串行调用） +// 对于 SeekableStream：使用 RangeRead，支持随机访问（续传场景可跳过已上传分片） +// 对于 FileStream：必须顺序读取 func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { - if off != ss.fileOffset { - return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) - } tempBuf := ss.bufPool.Get() buf := tempBuf[:length] - // 首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） - // 对于 FileStream，RangeRead 会消耗底层 oriReader，所以必须先尝试顺序流读取 - n, err := io.ReadFull(ss.file, buf) - if err == nil { - ss.fileOffset = off + int64(n) + // 对于 SeekableStream，直接使用 RangeRead（支持随机访问，适用于续传场景） + if _, ok := ss.file.(*SeekableStream); ok { + n, err := ReadFullWithRangeRead(ss.file, buf, off) + if err != nil { + ss.bufPool.Put(tempBuf) + return nil, fmt.Errorf("RangeRead failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) + } return &bufferSectionReader{bytes.NewReader(buf), tempBuf}, nil } - // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） - log.Debugf("Sequential read failed at offset %d, retrying with RangeRead: %v", off, err) - n, err = readFullWithRangeRead(ss.file, buf, off) + // 对于 FileStream，必须顺序读取 + if off != ss.fileOffset { + ss.bufPool.Put(tempBuf) + return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) + } + + n, err := io.ReadFull(ss.file, buf) if err != nil { ss.bufPool.Put(tempBuf) - return nil, fmt.Errorf("both sequential read and RangeRead failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) + return nil, fmt.Errorf("sequential read failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) } - // 更新 fileOffset ss.fileOffset = off + int64(n) return &bufferSectionReader{bytes.NewReader(buf), tempBuf}, nil } From 25dc69d7153480d67b0666ff435fe2fdf7cca51e Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 6 Jan 2026 02:43:16 +0800 Subject: [PATCH 36/86] =?UTF-8?q?feat(network):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=AF=B9=E6=85=A2=E9=80=9F=E7=BD=91=E7=BB=9C=E7=9A=84=E6=94=AF?= =?UTF-8?q?=E6=8C=81=EF=BC=8C=E8=B0=83=E6=95=B4=E8=B6=85=E6=97=B6=E5=92=8C?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(stream): improve thread safety and handling for SeekableStream and FileStream in directSectionReader feat(upload): 增强分片上传支持，修复超时和ETag提取逻辑 --- .gitignore | 3 +- drivers/115_open/driver.go | 31 ++++++ drivers/baidu_netdisk/meta.go | 4 +- drivers/quark_open/driver.go | 131 +++++++++++++++++++----- drivers/quark_open/meta.go | 4 +- drivers/quark_open/util.go | 94 +++++++++++++++-- internal/net/serve.go | 11 +- internal/stream/util.go | 185 +--------------------------------- 8 files changed, 239 insertions(+), 224 deletions(-) diff --git a/.gitignore b/.gitignore index 1d71f0d60..add6d56bb 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ output/ /public/dist/* /!public/dist/README.md -.VSCodeCounter \ No newline at end of file +.VSCodeCounter +nul diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 03d29efe6..f95ab429b 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -17,6 +17,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -74,13 +75,20 @@ func (d *Open115) Drop(ctx context.Context) error { } func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + start := time.Now() + log.Infof("[115] List request started for dir: %s (ID: %s)", dir.GetName(), dir.GetID()) + var res []model.Obj pageSize := int64(d.PageSize) offset := int64(0) + pageCount := 0 + for { if err := d.WaitLimit(ctx); err != nil { return nil, err } + + pageStart := time.Now() resp, err := d.client.GetFiles(ctx, &sdk.GetFilesReq{ CID: dir.GetID(), Limit: pageSize, @@ -90,7 +98,12 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) // Cur: 1, ShowDir: true, }) + pageDuration := time.Since(pageStart) + pageCount++ + log.Infof("[115] GetFiles page %d took: %v (offset=%d, limit=%d)", pageCount, pageDuration, offset, pageSize) + if err != nil { + log.Errorf("[115] GetFiles page %d failed after %v: %v", pageCount, pageDuration, err) return nil, err } res = append(res, utils.MustSliceConvert(resp.Data, func(src sdk.GetFilesResp_File) model.Obj { @@ -102,10 +115,17 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) } offset += pageSize } + + totalDuration := time.Since(start) + log.Infof("[115] List request completed in %v (%d pages, %d files)", totalDuration, pageCount, len(res)) + return res, nil } func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + start := time.Now() + log.Infof("[115] Link request started for file: %s", file.GetName()) + if err := d.WaitLimit(ctx); err != nil { return nil, err } @@ -121,14 +141,25 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, fmt.Errorf("can't convert obj") } pc := obj.Pc + + apiStart := time.Now() + log.Infof("[115] Calling DownURL API...") resp, err := d.client.DownURL(ctx, pc, ua) + apiDuration := time.Since(apiStart) + log.Infof("[115] DownURL API took: %v", apiDuration) + if err != nil { + log.Errorf("[115] DownURL API failed after %v: %v", apiDuration, err) return nil, err } u, ok := resp[obj.GetID()] if !ok { return nil, fmt.Errorf("can't get link") } + + totalDuration := time.Since(start) + log.Infof("[115] Link request completed in %v (API: %v)", totalDuration, apiDuration) + return &model.Link{ URL: u.URL.URL, Header: http.Header{ diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 3f3bed022..499fcd8a8 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -31,8 +31,8 @@ type Addition struct { const ( UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) - DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 60 // 上传分片请求默认超时时间 - UPLOAD_RETRY_COUNT = 3 + DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 180 // 上传分片请求默认超时时间（增加到3分钟以应对慢速网络） + UPLOAD_RETRY_COUNT = 5 // 增加重试次数以提高成功率 UPLOAD_RETRY_WAIT_TIME = time.Second * 1 UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 ) diff --git a/drivers/quark_open/driver.go b/drivers/quark_open/driver.go index f0b8baf09..cf1ff3cb0 100644 --- a/drivers/quark_open/driver.go +++ b/drivers/quark_open/driver.go @@ -8,6 +8,7 @@ import ( "hash" "io" "net/http" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -18,6 +19,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) type QuarkOpen struct { @@ -144,30 +146,84 @@ func (d *QuarkOpen) Remove(ctx context.Context, obj model.Obj) error { func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) - var ( - md5 hash.Hash - sha1 hash.Hash - ) - writers := []io.Writer{} - if len(md5Str) != utils.MD5.Width { - md5 = utils.MD5.NewFunc() - writers = append(writers, md5) - } - if len(sha1Str) != utils.SHA1.Width { - sha1 = utils.SHA1.NewFunc() - writers = append(writers, sha1) - } - if len(writers) > 0 { - _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) - if err != nil { - return err - } - if md5 != nil { - md5Str = hex.EncodeToString(md5.Sum(nil)) - } - if sha1 != nil { - sha1Str = hex.EncodeToString(sha1.Sum(nil)) + // 检查是否需要计算hash + needMD5 := len(md5Str) != utils.MD5.Width + needSHA1 := len(sha1Str) != utils.SHA1.Width + + if needMD5 || needSHA1 { + // 检查是否为可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 一次性计算所有hash，避免重复读取 + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + // 使用 RangeRead 分块读取文件，同时计算多个hash + multiWriter := io.MultiWriter(writers...) + size := stream.GetSize() + chunkSize := int64(10 * utils.MB) // 10MB per chunk + buf := make([]byte, chunkSize) + var offset int64 = 0 + + for offset < size { + readSize := min(chunkSize, size-offset) + + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset) + if err != nil { + return fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + + multiWriter.Write(buf[:n]) + offset += int64(n) + + // 更新进度（hash计算占用40%的进度） + up(40 * float64(offset) / float64(size)) + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } + } else { + // 不可重复读取的流（如网络流），需要缓存并计算hash + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) + if err != nil { + return err + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } } } // pre @@ -210,24 +266,43 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File if err != nil { return err } + + // 上传重试逻辑，包含URL刷新 + var etag string err = retry.Do(func() error { rd.Seek(0, io.SeekStart) - etag, err := d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) - if err != nil { - return err + var uploadErr error + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) + + // 检查是否为URL过期错误 + if uploadErr != nil && strings.Contains(uploadErr.Error(), "expire") { + log.Warnf("[quark_open] Upload URL expired for part %d, refreshing...", i) + // 刷新上传URL + newUpUrlInfo, refreshErr := d.upUrl(ctx, pre, partInfo) + if refreshErr != nil { + return fmt.Errorf("failed to refresh upload url: %w", refreshErr) + } + upUrlInfo = newUpUrlInfo + log.Infof("[quark_open] Upload URL refreshed successfully") + + // 使用新URL重试上传 + rd.Seek(0, io.SeekStart) + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) } - etags = append(etags, etag) - return nil + + return uploadErr }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) + ss.FreeSectionReader(rd) if err != nil { return fmt.Errorf("failed to upload part %d: %w", i, err) } + etags = append(etags, etag) up(95 * float64(offset+size) / float64(total)) } diff --git a/drivers/quark_open/meta.go b/drivers/quark_open/meta.go index 3527b52e9..ee1903939 100644 --- a/drivers/quark_open/meta.go +++ b/drivers/quark_open/meta.go @@ -13,8 +13,8 @@ type Addition struct { APIAddress string `json:"api_url_address" default:"https://api.oplist.org/quarkyun/renewapi"` AccessToken string `json:"access_token" required:"false" default:""` RefreshToken string `json:"refresh_token" required:"true"` - AppID string `json:"app_id" required:"true" help:"Keep it empty if you don't have one"` - SignKey string `json:"sign_key" required:"true" help:"Keep it empty if you don't have one"` + AppID string `json:"app_id" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` + SignKey string `json:"sign_key" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` } type Conf struct { diff --git a/drivers/quark_open/util.go b/drivers/quark_open/util.go index 788ca0e99..1a3058375 100644 --- a/drivers/quark_open/util.go +++ b/drivers/quark_open/util.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -283,8 +284,15 @@ func (d *QuarkOpen) getProofRange(proofSeed string, fileSize int64) (*ProofRange func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []base.Json { // 计算分片信息 - partInfo := make([]base.Json, 0) total := stream.GetSize() + + // 确保partSize合理：最小4MB，避免分片过多 + const minPartSize int64 = 4 * utils.MB + if partSize < minPartSize { + partSize = minPartSize + } + + partInfo := make([]base.Json, 0) left := total partNumber := 1 @@ -304,6 +312,7 @@ func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []ba partNumber++ } + log.Infof("[quark_open] Upload plan: file_size=%d, part_size=%d, part_count=%d", total, partSize, len(partInfo)) return partInfo } @@ -315,11 +324,17 @@ func (d *QuarkOpen) upUrl(ctx context.Context, pre UpPreResp, partInfo []base.Js } var resp UpUrlResp + log.Infof("[quark_open] Requesting upload URLs for %d parts (task_id: %s)", len(partInfo), pre.Data.TaskID) + _, err = d.request(ctx, "/open/v1/file/get_upload_urls", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { + // 如果是分片超限错误，记录详细信息 + if strings.Contains(err.Error(), "part list exceed") { + log.Errorf("[quark_open] Part list exceeded limit! Requested %d parts. Please check Quark API documentation for actual limit.", len(partInfo)) + } return upUrlInfo, err } @@ -340,13 +355,43 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("User-Agent", "Go-http-client/1.1") + // ✅ 关键修复：使用更长的超时时间（10分钟） + // 慢速网络下大文件分片上传可能需要很长时间 + client := &http.Client{ + Timeout: 10 * time.Minute, + Transport: base.HttpClient.Transport, + } + // 发送请求 - resp, err := base.HttpClient.Do(req) + resp, err := client.Do(req) if err != nil { return "", err } defer resp.Body.Close() + // 检查是否为URL过期错误（403, 410等状态码） + if resp.StatusCode == 403 || resp.StatusCode == 410 { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("upload url expired (status: %d): %s", resp.StatusCode, string(body)) + } + + // ✅ 关键修复：409 PartAlreadyExist 不是错误！ + // 夸克使用Sequential模式，超时重试时如果分片已存在，说明第一次其实成功了 + if resp.StatusCode == 409 { + body, _ := io.ReadAll(resp.Body) + // 从响应体中提取已存在分片的ETag + if strings.Contains(string(body), "PartAlreadyExist") { + // 尝试从XML响应中提取ETag + if etag := extractEtagFromXML(string(body)); etag != "" { + log.Infof("[quark_open] Part %d already exists (409), using existing ETag: %s", partNumber+1, etag) + return etag, nil + } + // 如果无法提取ETag，返回错误 + log.Warnf("[quark_open] Part %d already exists but cannot extract ETag from response: %s", partNumber+1, string(body)) + return "", fmt.Errorf("part already exists but ETag not found in response") + } + } + if resp.StatusCode != 200 { body, _ := io.ReadAll(resp.Body) return "", fmt.Errorf("up status: %d, error: %s", resp.StatusCode, string(body)) @@ -355,6 +400,23 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber return resp.Header.Get("Etag"), nil } +// extractEtagFromXML 从OSS的XML错误响应中提取ETag +// 示例: "2F796AC486BB2891E3237D8BFDE020B5" +func extractEtagFromXML(xmlBody string) string { + start := strings.Index(xmlBody, "") + if start == -1 { + return "" + } + start += len("") + end := strings.Index(xmlBody[start:], "") + if end == -1 { + return "" + } + etag := xmlBody[start : start+end] + // 移除引号 + return strings.Trim(etag, "\"") +} + func (d *QuarkOpen) upFinish(ctx context.Context, pre UpPreResp, partInfo []base.Json, etags []string) error { // 创建 part_info_list partInfoList := make([]base.Json, len(partInfo)) @@ -417,25 +479,36 @@ func (d *QuarkOpen) generateReqSign(method string, pathname string, signKey stri } func (d *QuarkOpen) refreshToken() error { - refresh, access, err := d._refreshToken() + refresh, access, appID, signKey, err := d._refreshToken() for i := 0; i < 3; i++ { if err == nil { break } else { log.Errorf("[quark_open] failed to refresh token: %s", err) } - refresh, access, err = d._refreshToken() + refresh, access, appID, signKey, err = d._refreshToken() } if err != nil { return err } log.Infof("[quark_open] token exchange: %s -> %s", d.RefreshToken, refresh) d.RefreshToken, d.AccessToken = refresh, access + + // 如果在线API返回了AppID和SignKey，保存它们（不为空时才更新） + if appID != "" && appID != d.AppID { + d.AppID = appID + log.Infof("[quark_open] AppID updated from online API: %s", appID) + } + if signKey != "" && signKey != d.SignKey { + d.SignKey = signKey + log.Infof("[quark_open] SignKey updated from online API") + } + op.MustSaveDriverStorage(d) return nil } -func (d *QuarkOpen) _refreshToken() (string, string, error) { +func (d *QuarkOpen) _refreshToken() (string, string, string, string, error) { if d.UseOnlineAPI && d.APIAddress != "" { u := d.APIAddress var resp RefreshTokenOnlineAPIResp @@ -448,19 +521,20 @@ func (d *QuarkOpen) _refreshToken() (string, string, error) { }). Get(u) if err != nil { - return "", "", err + return "", "", "", "", err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { - return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) + return "", "", "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } - return "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") + return "", "", "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } - return resp.RefreshToken, resp.AccessToken, nil + // 返回所有字段，包括AppID和SignKey + return resp.RefreshToken, resp.AccessToken, resp.AppID, resp.SignKey, nil } // TODO 本地刷新逻辑 - return "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") + return "", "", "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") } // 生成认证 Cookie diff --git a/internal/net/serve.go b/internal/net/serve.go index 6a20460b1..ee288b86a 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "mime/multipart" + stdnet "net" // 标准库net包，用于Dialer "net/http" "strconv" "strings" @@ -286,12 +287,20 @@ func NewHttpClient() *http.Client { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + // 快速连接超时：10秒建立连接，失败快速重试 + DialContext: (&stdnet.Dialer{ + Timeout: 10 * time.Second, // TCP握手超时 + KeepAlive: 30 * time.Second, // TCP keep-alive + }).DialContext, + // 响应头超时：15秒等待服务器响应头（平衡API调用与下载检测） + ResponseHeaderTimeout: 15 * time.Second, + // 允许长时间读取数据（无 IdleConnTimeout 限制） } SetProxyIfConfigured(transport) return &http.Client{ - Timeout: time.Hour * 48, + Timeout: time.Hour * 48, // 总超时保持48小时（允许大文件慢速下载） Transport: transport, } } diff --git a/internal/stream/util.go b/internal/stream/util.go index 00c3bde52..a72fb7990 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -9,8 +9,6 @@ import ( "io" "net/http" "os" - "strings" - "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -30,157 +28,7 @@ func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Ran return f(ctx, httpRange) } -// LinkRefresher 接口用于在读取数据失败时强制刷新链接 -type LinkRefresher interface { - // ForceRefreshLink 强制刷新下载链接 - // 返回 true 表示刷新成功，false 表示无法刷新 - ForceRefreshLink(ctx context.Context) bool -} - -// IsLinkExpiredError checks if the error indicates an expired download link -func IsLinkExpiredError(err error) bool { - if err == nil { - return false - } - errStr := strings.ToLower(err.Error()) - - // Common expired link error keywords - expiredKeywords := []string{ - "expired", "invalid signature", "token expired", - "access denied", "forbidden", "unauthorized", - "link has expired", "url expired", "request has expired", - "signature expired", "accessdenied", "invalidtoken", - } - for _, keyword := range expiredKeywords { - if strings.Contains(errStr, keyword) { - return true - } - } - - // Check for HTTP status codes that typically indicate expired links - if statusErr, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { - code := int(statusErr) - // 401 Unauthorized, 403 Forbidden, 410 Gone are common for expired links - // 500 Internal Server Error - some providers (e.g., Baidu) return 500 when link expires - if code == 401 || code == 403 || code == 410 || code == 500 { - return true - } - } - - return false -} - -// RefreshableRangeReader wraps a RangeReader with link refresh capability -type RefreshableRangeReader struct { - link *model.Link - size int64 - innerReader model.RangeReaderIF - mu sync.Mutex - refreshCount int // track refresh count to avoid infinite loops -} - -// NewRefreshableRangeReader creates a new RefreshableRangeReader -func NewRefreshableRangeReader(link *model.Link, size int64) *RefreshableRangeReader { - return &RefreshableRangeReader{ - link: link, - size: size, - } -} - -func (r *RefreshableRangeReader) getInnerReader() (model.RangeReaderIF, error) { - if r.innerReader != nil { - return r.innerReader, nil - } - - // Create inner reader without Refresher to avoid recursion - linkCopy := *r.link - linkCopy.Refresher = nil - - reader, err := GetRangeReaderFromLink(r.size, &linkCopy) - if err != nil { - return nil, err - } - r.innerReader = reader - return reader, nil -} - -func (r *RefreshableRangeReader) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { - r.mu.Lock() - reader, err := r.getInnerReader() - r.mu.Unlock() - if err != nil { - return nil, err - } - - rc, err := reader.RangeRead(ctx, httpRange) - if err != nil { - // Check if we should try to refresh on initial connection error - if IsLinkExpiredError(err) && r.link.Refresher != nil { - rc, err = r.refreshAndRetry(ctx, httpRange) - } - if err != nil { - return nil, err - } - } - - return rc, nil -} - -func (r *RefreshableRangeReader) refreshAndRetry(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { - r.mu.Lock() - defer r.mu.Unlock() - - if err := r.doRefreshLocked(ctx); err != nil { - return nil, err - } - - reader, err := r.getInnerReader() - if err != nil { - return nil, err - } - return reader.RangeRead(ctx, httpRange) -} - -// ForceRefresh 强制刷新链接，用于读取数据失败（如读取 0 字节）的情况 -// 返回 true 表示刷新成功，false 表示无法刷新（没有 Refresher 或达到最大刷新次数） -func (r *RefreshableRangeReader) ForceRefresh(ctx context.Context) bool { - if r.link.Refresher == nil { - return false - } - - r.mu.Lock() - defer r.mu.Unlock() - - return r.doRefreshLocked(ctx) == nil -} - -// doRefreshLocked 执行实际的刷新逻辑（需要持有锁） -func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { - if r.refreshCount >= 3 { - return fmt.Errorf("max refresh attempts reached") - } - - log.Infof("Link expired, attempting to refresh...") - newLink, _, refreshErr := r.link.Refresher(ctx) - if refreshErr != nil { - return fmt.Errorf("failed to refresh link: %w", refreshErr) - } - - newLink.Refresher = r.link.Refresher - r.link = newLink - r.innerReader = nil - r.refreshCount++ - - log.Infof("Link refreshed successfully") - return nil -} - func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { - // If link has a Refresher, wrap with RefreshableRangeReader for automatic refresh on expiry - if link.Refresher != nil { - return NewRefreshableRangeReader(link, size), nil - } - if link.RangeReader != nil { if link.Concurrency < 1 && link.PartSize < 1 { return link.RangeReader, nil @@ -327,14 +175,13 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } -// ReadFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf +// readFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf // file: 文件流 // buf: 目标缓冲区 // off: 读取的起始偏移量 // 返回值: 实际读取的字节数和错误 // 支持自动重试（最多3次），每次重试之间有递增延迟（3秒、6秒、9秒） -// 支持链接刷新：当检测到 0 字节读取时，会自动刷新下载链接 -func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { +func readFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { length := int64(len(buf)) var lastErr error @@ -360,28 +207,6 @@ func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, lastErr = fmt.Errorf("failed to read all data via RangeRead at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) log.Debugf("RangeRead retry %d read failed: %v", retry+1, lastErr) - - // 检测是否可能是链接过期（读取 0 字节或 EOF） - if n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { - // 尝试刷新链接 - if refresher, ok := file.(LinkRefresher); ok { - // 获取 context - 从 FileStream 或 SeekableStream 中获取 - var ctx context.Context - if fs, ok := file.(*FileStream); ok { - ctx = fs.Ctx - } else if ss, ok := file.(*SeekableStream); ok { - ctx = ss.Ctx - } else { - ctx = context.Background() - } - - if refresher.ForceRefreshLink(ctx) { - log.Infof("Link refreshed after 0-byte read, retrying immediately...") - continue // 立即重试，不延迟 - } - } - } - // 递增延迟：3秒、6秒、9秒，等待网络恢复 time.Sleep(time.Duration(retry+1) * 3 * time.Second) } @@ -427,14 +252,14 @@ func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressW // 对于 SeekableStream，优先使用 RangeRead 避免消耗 Reader // 这样后续发送时 Reader 还能正常工作 if _, ok := file.(*SeekableStream); ok { - n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) + n, err = readFullWithRangeRead(file, buf[:readSize], offset) } else { // 对于 FileStream，首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） n, err = io.ReadFull(file, buf[:readSize]) if err != nil { // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) - n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) + n, err = readFullWithRangeRead(file, buf[:readSize], offset) } } @@ -608,7 +433,7 @@ func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeke // 对于 SeekableStream，直接使用 RangeRead（支持随机访问，适用于续传场景） if _, ok := ss.file.(*SeekableStream); ok { - n, err := ReadFullWithRangeRead(ss.file, buf, off) + n, err := readFullWithRangeRead(ss.file, buf, off) if err != nil { ss.bufPool.Put(tempBuf) return nil, fmt.Errorf("RangeRead failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) From e91e8820118194a63ca1560a217219538f24cc1b Mon Sep 17 00:00:00 2001 From: cyk Date: Fri, 9 Jan 2026 17:10:28 +0800 Subject: [PATCH 37/86] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D500=20panic?= =?UTF-8?q?=E5=92=8CNaN=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 修复HashInfo nil pointer导致的500 panic (fsread.go) - 修复StorageDetails为nil导致的NaN显示 (storage.go, op/storage.go) - 添加DiskUsage.MarshalJSON()确保返回used_space字段 fix: 彻底修复500 panic - 初始化所有虚拟Object的HashInfo - 修复 fs.go 5个虚拟Object创建点未初始化HashInfo - 修复 storage.go 虚拟folder Object未初始化HashInfo - 确保所有代码路径都不会触发nil pointer panic fix(model): resolve DiskUsage structure conflicts after rebase Remove duplicate UsedSpace() method and MarshalJSON() that conflicted with main branch's new DiskUsage structure (commit 744dbd5e). Main's structure uses: - UsedSpace int64 (field, not method) - FreeSpace() int64 (computed from TotalSpace - UsedSpace) - MarshalJSON() returns all three fields Co-Authored-By: Claude --- internal/op/fs.go | 5 +++++ internal/op/storage.go | 5 ++++- server/handles/fsread.go | 6 +++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/op/fs.go b/internal/op/fs.go index 2c91e6cf3..3fe70a3ae 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -150,6 +150,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, IsFolder: true, Mask: model.Locked, + HashInfo: utils.NewHashInfo(nil, ""), }, nil case driver.IRootPath: return &model.Object{ @@ -158,6 +159,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, Mask: model.Locked, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), }, nil } return nil, errors.New("please implement GetRooter or IRootPath or IRootId interface") @@ -380,6 +382,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } dirCache.UpdateObject("", wrapObjName(storage, newObj)) @@ -704,6 +707,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod Modified: file.ModTime(), Ctime: file.CreateTime(), Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) @@ -772,6 +776,7 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) diff --git a/internal/op/storage.go b/internal/op/storage.go index da4c84e31..2e93bf569 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -368,7 +368,9 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, }(d) select { case r := <-resultChan: - ret.StorageDetails = r + if r != nil { + ret.StorageDetails = r + } case <-time.After(time.Second): } return ret @@ -419,6 +421,7 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver Name: name, Modified: v.GetStorage().Modified, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), } if !found { idx := len(files) diff --git a/server/handles/fsread.go b/server/handles/fsread.go index a90fc1082..8a67e4e59 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -230,6 +230,10 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { for _, obj := range objs { thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) + hashInfo := obj.GetHash().Export() + if hashInfo == nil { + hashInfo = make(map[*utils.HashType]string) + } resp = append(resp, ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), @@ -237,7 +241,7 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), + HashInfo: hashInfo, Sign: common.Sign(obj, parent, encrypt), Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), From 593bd4cab87df97f4719a7e1a73a9f3463193490 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 4 Jan 2026 12:17:22 +0800 Subject: [PATCH 38/86] fix(alias): update storage retrieval method in listRoot function --- drivers/alias/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 8e5eb8a84..b37854394 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -40,7 +40,7 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model if !withDetails || len(v) != 1 { continue } - remoteDriver, err := op.GetStorageByMountPath(v[0]) + remoteDriver, err := fs.GetStorage(v[0], &fs.GetStoragesArgs{}) if err != nil { continue } From d7dd42e47881d45258b8043b37341ffd5523aa3a Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 11 Jan 2026 21:11:42 +0800 Subject: [PATCH 39/86] =?UTF-8?q?fix(copy=5Fmove):=20=E5=B0=86=E9=A2=84?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E5=AD=90=E7=9B=AE=E5=BD=95=E7=9A=84=E6=B7=B1?= =?UTF-8?q?=E5=BA=A6=E4=BB=8E2=E7=BA=A7=E8=B0=83=E6=95=B4=E4=B8=BA1?= =?UTF-8?q?=E7=BA=A7=EF=BC=8C=E4=BB=A5=E9=81=BF=E5=85=8D=E6=B7=B1=E5=BA=A6?= =?UTF-8?q?=E9=80=92=E5=BD=92=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/sftp/types.go | 4 +- internal/fs/copy_move.go | 68 +++++++++++++ internal/op/fs.go | 11 ++- internal/stream/util.go | 207 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 274 insertions(+), 16 deletions(-) diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go index 00a32f001..a57076e08 100644 --- a/drivers/sftp/types.go +++ b/drivers/sftp/types.go @@ -48,8 +48,8 @@ func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { Size: _f.Size(), Modified: _f.ModTime(), IsFolder: _f.IsDir(), - Path: target, + Path: path, // Use symlink's own path, not target path } - log.Debugf("[sftp] obj: %+v, is symlink: %v", obj, symlink) + log.Debugf("[sftp] obj: %+v, is symlink: %v, target: %s", obj, symlink, target) return obj, nil } diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index e78fc9be8..77c2015b5 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -17,6 +17,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type taskType uint8 @@ -192,6 +193,21 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) + // Pre-create the destination directory first + t.Status = "ensuring destination directory exists" + if err := op.MakeDir(t.Ctx(), t.DstStorage, dstActualPath); err != nil { + log.Warnf("[copy_move] failed to ensure destination dir [%s]: %v, will continue", dstActualPath, err) + // Continue anyway - the directory might exist but Get failed due to cache issues + } + + // Pre-create subdirectories (up to 1 level deep) to avoid deep recursion issues + // Balances between reducing API calls and maintaining fault tolerance + t.Status = "pre-creating subdirectories" + if err := t.preCreateDirectoryTree(objs, dstActualPath, 1); err != nil { + log.Warnf("[copy_move] failed to pre-create directory tree: %v, will continue", err) + // Continue anyway - individual directories will be created on-demand + } + existedObjs := make(map[string]bool) if t.TaskType == merge { dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) @@ -263,6 +279,58 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } +// preCreateDirectoryTree recursively scans source directory tree and pre-creates +// directories on destination up to maxDepth levels to avoid deep MakeDir recursion issues. +// maxDepth=0 means only current level, maxDepth=1 means current+1 level, etc. +func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath string, maxDepth int) error { + // First pass: create immediate subdirectories + var subdirs []model.Obj + for _, obj := range objs { + // Check for cancellation + if err := t.Ctx().Err(); err != nil { + return err + } + + if obj.IsDir() { + subdirPath := stdpath.Join(dstBasePath, obj.GetName()) + if err := op.MakeDir(t.Ctx(), t.DstStorage, subdirPath); err != nil { + log.Debugf("[copy_move] failed to pre-create dir [%s]: %v", subdirPath, err) + // Continue with other directories + } + subdirs = append(subdirs, obj) + } + } + + // Stop recursion if max depth reached + if maxDepth <= 0 { + return nil + } + + // Second pass: recursively scan and create nested subdirectories + for _, subdir := range subdirs { + if err := t.Ctx().Err(); err != nil { + return err + } + + // List contents of this subdirectory + subdirSrcPath := stdpath.Join(t.SrcActualPath, subdir.GetName()) + subdirDstPath := stdpath.Join(dstBasePath, subdir.GetName()) + + subObjs, err := op.List(t.Ctx(), t.SrcStorage, subdirSrcPath, model.ListArgs{}) + if err != nil { + log.Debugf("[copy_move] failed to list subdir [%s] for pre-creation: %v", subdirSrcPath, err) + continue // Skip this subdirectory, will handle when processing + } + + // Recursively create subdirectories with decreased depth + if err := t.preCreateDirectoryTree(subObjs, subdirDstPath, maxDepth-1); err != nil { + return err + } + } + + return nil +} + var ( CopyTaskManager *tache.Manager[*FileTransferTask] MoveTaskManager *tache.Manager[*FileTransferTask] diff --git a/internal/op/fs.go b/internal/op/fs.go index 3fe70a3ae..29a31ad63 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -350,9 +350,16 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) } parentDir, err := GetUnwrap(ctx, storage, parentPath) - // this should not happen if err != nil { - return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + if errs.IsObjectNotFound(err) { + // Retry once after a short delay (handles cloud storage API sync delay) + log.Debugf("[op] parent dir [%s] not found immediately after creation, retrying...", parentPath) + time.Sleep(100 * time.Millisecond) + parentDir, err = GetUnwrap(ctx, storage, parentPath) + } + if err != nil { + return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } } if model.ObjHasMask(parentDir, model.NoWrite) { return nil, errors.WithStack(errs.PermissionDenied) diff --git a/internal/stream/util.go b/internal/stream/util.go index a72fb7990..b24ad2417 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -9,6 +9,8 @@ import ( "io" "net/http" "os" + "strings" + "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -22,13 +24,171 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + // 链接刷新相关常量 + MAX_LINK_REFRESH_COUNT = 50 // 下载链接最大刷新次数（支持长时间传输） + + // RangeRead 重试相关常量 + MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead 最大重试次数（从3增加到5） +) + type RangeReaderFunc func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return f(ctx, httpRange) } +// LinkRefresher 接口用于在读取数据失败时强制刷新链接 +type LinkRefresher interface { + // ForceRefreshLink 强制刷新下载链接 + // 返回 true 表示刷新成功，false 表示无法刷新 + ForceRefreshLink(ctx context.Context) bool +} + +// IsLinkExpiredError checks if the error indicates an expired download link +func IsLinkExpiredError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + + // Common expired link error keywords + expiredKeywords := []string{ + "expired", "invalid signature", "token expired", + "access denied", "forbidden", "unauthorized", + "link has expired", "url expired", "request has expired", + "signature expired", "accessdenied", "invalidtoken", + } + for _, keyword := range expiredKeywords { + if strings.Contains(errStr, keyword) { + return true + } + } + + // Check for HTTP status codes that typically indicate expired links + if statusErr, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { + code := int(statusErr) + // 401 Unauthorized, 403 Forbidden, 410 Gone are common for expired links + // 500 Internal Server Error - some providers (e.g., Baidu) return 500 when link expires + if code == 401 || code == 403 || code == 410 || code == 500 { + return true + } + } + + return false +} + +// RefreshableRangeReader wraps a RangeReader with link refresh capability +type RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // track refresh count to avoid infinite loops +} + +// NewRefreshableRangeReader creates a new RefreshableRangeReader +func NewRefreshableRangeReader(link *model.Link, size int64) *RefreshableRangeReader { + return &RefreshableRangeReader{ + link: link, + size: size, + } +} + +func (r *RefreshableRangeReader) getInnerReader() (model.RangeReaderIF, error) { + if r.innerReader != nil { + return r.innerReader, nil + } + + // Create inner reader without Refresher to avoid recursion + linkCopy := *r.link + linkCopy.Refresher = nil + + reader, err := GetRangeReaderFromLink(r.size, &linkCopy) + if err != nil { + return nil, err + } + r.innerReader = reader + return reader, nil +} + +func (r *RefreshableRangeReader) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + reader, err := r.getInnerReader() + r.mu.Unlock() + if err != nil { + return nil, err + } + + rc, err := reader.RangeRead(ctx, httpRange) + if err != nil { + // Check if we should try to refresh on initial connection error + if IsLinkExpiredError(err) && r.link.Refresher != nil { + rc, err = r.refreshAndRetry(ctx, httpRange) + } + if err != nil { + return nil, err + } + } + + return rc, nil +} + +func (r *RefreshableRangeReader) refreshAndRetry(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if err := r.doRefreshLocked(ctx); err != nil { + return nil, err + } + + reader, err := r.getInnerReader() + if err != nil { + return nil, err + } + return reader.RangeRead(ctx, httpRange) +} + +// ForceRefresh 强制刷新链接，用于读取数据失败（如读取 0 字节）的情况 +// 返回 true 表示刷新成功，false 表示无法刷新（没有 Refresher 或达到最大刷新次数） +func (r *RefreshableRangeReader) ForceRefresh(ctx context.Context) bool { + if r.link.Refresher == nil { + return false + } + + r.mu.Lock() + defer r.mu.Unlock() + + return r.doRefreshLocked(ctx) == nil +} + +// doRefreshLocked 执行实际的刷新逻辑（需要持有锁） +func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { + if r.refreshCount >= MAX_LINK_REFRESH_COUNT { + return fmt.Errorf("max refresh attempts (%d) reached", MAX_LINK_REFRESH_COUNT) + } + + log.Infof("Link expired, attempting to refresh...") + newLink, _, refreshErr := r.link.Refresher(ctx) + if refreshErr != nil { + return fmt.Errorf("failed to refresh link: %w", refreshErr) + } + + newLink.Refresher = r.link.Refresher + r.link = newLink + r.innerReader = nil + r.refreshCount++ + + log.Infof("Link refreshed successfully") + return nil +} + func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { + // If link has a Refresher, wrap with RefreshableRangeReader for automatic refresh on expiry + if link.Refresher != nil { + return NewRefreshableRangeReader(link, size), nil + } + if link.RangeReader != nil { if link.Concurrency < 1 && link.PartSize < 1 { return link.RangeReader, nil @@ -175,24 +335,25 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } -// readFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf +// ReadFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf // file: 文件流 // buf: 目标缓冲区 // off: 读取的起始偏移量 // 返回值: 实际读取的字节数和错误 -// 支持自动重试（最多3次），每次重试之间有递增延迟（3秒、6秒、9秒） -func readFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { +// 支持自动重试（最多5次），快速重试策略（1秒、2秒、3秒、4秒、5秒） +// 支持链接刷新：当检测到 0 字节读取时，会自动刷新下载链接 +func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { length := int64(len(buf)) var lastErr error - // 重试最多3次 - for retry := 0; retry < 3; retry++ { + // 重试最多 MAX_RANGE_READ_RETRY_COUNT 次 + for retry := 0; retry < MAX_RANGE_READ_RETRY_COUNT; retry++ { reader, err := file.RangeRead(http_range.Range{Start: off, Length: length}) if err != nil { lastErr = fmt.Errorf("RangeRead failed at offset %d: %w", off, err) log.Debugf("RangeRead retry %d failed: %v", retry+1, lastErr) - // 递增延迟：3秒、6秒、9秒，等待代理恢复 - time.Sleep(time.Duration(retry+1) * 3 * time.Second) + // 快速重试：1秒、2秒、3秒、4秒、5秒（连接失败快速重试） + time.Sleep(time.Duration(retry+1) * time.Second) continue } @@ -207,8 +368,30 @@ func readFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, lastErr = fmt.Errorf("failed to read all data via RangeRead at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) log.Debugf("RangeRead retry %d read failed: %v", retry+1, lastErr) - // 递增延迟：3秒、6秒、9秒，等待网络恢复 - time.Sleep(time.Duration(retry+1) * 3 * time.Second) + + // 检测是否可能是链接过期（读取 0 字节或 EOF） + if n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { + // 尝试刷新链接 + if refresher, ok := file.(LinkRefresher); ok { + // 获取 context - 从 FileStream 或 SeekableStream 中获取 + var ctx context.Context + if fs, ok := file.(*FileStream); ok { + ctx = fs.Ctx + } else if ss, ok := file.(*SeekableStream); ok { + ctx = ss.Ctx + } else { + ctx = context.Background() + } + + if refresher.ForceRefreshLink(ctx) { + log.Infof("Link refreshed after 0-byte read, retrying immediately...") + continue // 立即重试，不延迟 + } + } + } + + // 快速重试：1秒、2秒、3秒、4秒、5秒（读取失败快速重试） + time.Sleep(time.Duration(retry+1) * time.Second) } return 0, lastErr @@ -252,14 +435,14 @@ func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressW // 对于 SeekableStream，优先使用 RangeRead 避免消耗 Reader // 这样后续发送时 Reader 还能正常工作 if _, ok := file.(*SeekableStream); ok { - n, err = readFullWithRangeRead(file, buf[:readSize], offset) + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) } else { // 对于 FileStream，首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） n, err = io.ReadFull(file, buf[:readSize]) if err != nil { // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) - n, err = readFullWithRangeRead(file, buf[:readSize], offset) + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) } } @@ -433,7 +616,7 @@ func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeke // 对于 SeekableStream，直接使用 RangeRead（支持随机访问，适用于续传场景） if _, ok := ss.file.(*SeekableStream); ok { - n, err := readFullWithRangeRead(ss.file, buf, off) + n, err := ReadFullWithRangeRead(ss.file, buf, off) if err != nil { ss.bufPool.Put(tempBuf) return nil, fmt.Errorf("RangeRead failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) From 8d8ed272ab39765c0c6f24c9262e6bab861ca40c Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 12 Jan 2026 18:49:42 +0800 Subject: [PATCH 40/86] =?UTF-8?q?fix(driver):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=A4=B8=E5=85=8B=E5=88=86=E7=89=87=E5=A4=A7=E5=B0=8F=E8=B0=83?= =?UTF-8?q?=E6=95=B4=E9=80=BB=E8=BE=91=EF=BC=8C=E6=94=AF=E6=8C=81=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E6=9C=BA=E5=88=B6=E4=BB=A5=E5=BA=94=E5=AF=B9=E8=B6=85?= =?UTF-8?q?=E9=99=90=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/quark_open/driver.go | 59 ++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/drivers/quark_open/driver.go b/drivers/quark_open/driver.go index cf1ff3cb0..181c282f2 100644 --- a/drivers/quark_open/driver.go +++ b/drivers/quark_open/driver.go @@ -237,16 +237,57 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return nil } - // get part info - partInfo := d._getPartInfo(stream, pre.Data.PartSize) - // get upload url info - upUrlInfo, err := d.upUrl(ctx, pre, partInfo) - if err != nil { + // 带重试的分片大小调整逻辑：如果检测到 "part list exceed" 错误，自动翻倍分片大小 + var upUrlInfo UpUrlInfo + var partInfo []base.Json + currentPartSize := pre.Data.PartSize + const maxRetries = 5 + const maxPartSize = 1024 * utils.MB // 1GB 上限 + + for attempt := 0; attempt < maxRetries; attempt++ { + // 计算分片信息 + partInfo = d._getPartInfo(stream, currentPartSize) + + // 尝试获取上传 URL + upUrlInfo, err = d.upUrl(ctx, pre, partInfo) + if err == nil { + // 成功获取上传 URL + log.Infof("[quark_open] Successfully obtained upload URLs with part size: %d MB (%d parts)", + currentPartSize/(1024*1024), len(partInfo)) + break + } + + // 检查是否为分片超限错误 + if strings.Contains(err.Error(), "exceed") { + if attempt < maxRetries-1 { + // 还有重试机会，翻倍分片大小 + newPartSize := currentPartSize * 2 + + // 检查是否超过上限 + if newPartSize > maxPartSize { + return fmt.Errorf("part list exceeded and cannot increase part size (current: %d MB, max: %d MB). File may be too large for Quark API", + currentPartSize/(1024*1024), maxPartSize/(1024*1024)) + } + + log.Warnf("[quark_open] Part list exceeded (attempt %d/%d, %d parts). Retrying with doubled part size: %d MB -> %d MB", + attempt+1, maxRetries, len(partInfo), + currentPartSize/(1024*1024), newPartSize/(1024*1024)) + + currentPartSize = newPartSize + continue // 重试 + } else { + // 已达到最大重试次数 + return fmt.Errorf("part list exceeded after %d retries. Last attempt: part size %d MB, %d parts", + maxRetries, currentPartSize/(1024*1024), len(partInfo)) + } + } + + // 其他错误，直接返回 return err } - // part up - ss, err := streamPkg.NewStreamSectionReader(stream, int(pre.Data.PartSize), &up) + // part up - 使用调整后的 currentPartSize + ss, err := streamPkg.NewStreamSectionReader(stream, int(currentPartSize), &up) if err != nil { return err } @@ -260,8 +301,8 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return ctx.Err() } - offset := int64(i) * pre.Data.PartSize - size := min(pre.Data.PartSize, total-offset) + offset := int64(i) * currentPartSize + size := min(currentPartSize, total-offset) rd, err := ss.GetSectionReader(offset, size) if err != nil { return err From 7f69f08103201ce34a49ba78a8cbdcaadca5e734 Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 13 Jan 2026 11:50:30 +0800 Subject: [PATCH 41/86] =?UTF-8?q?refactor(stream):=20=E7=A7=BB=E9=99=A4?= =?UTF-8?q?=E8=BF=87=E6=97=B6=E7=9A=84=E9=93=BE=E6=8E=A5=E5=88=B7=E6=96=B0?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=8C=E6=B7=BB=E5=8A=A0=E8=87=AA=E6=84=88?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E5=99=A8=E4=BB=A5=E5=A4=84=E7=90=860?= =?UTF-8?q?=E5=AD=97=E8=8A=82=E8=AF=BB=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/stream/stream.go | 8 --- internal/stream/util.go | 133 ++++++++++++++++++++++++++------------ 2 files changed, 90 insertions(+), 51 deletions(-) diff --git a/internal/stream/stream.go b/internal/stream/stream.go index aaf310487..7eec75dd9 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -346,14 +346,6 @@ func (ss *SeekableStream) generateReader() error { return nil } -// ForceRefreshLink 实现 LinkRefresher 接口，用于在读取失败时刷新链接 -func (ss *SeekableStream) ForceRefreshLink(ctx context.Context) bool { - if rr, ok := ss.rangeReader.(*RefreshableRangeReader); ok { - return rr.ForceRefresh(ctx) - } - return false -} - func (ss *SeekableStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writer) (model.File, error) { if err := ss.generateReader(); err != nil { return nil, err diff --git a/internal/stream/util.go b/internal/stream/util.go index b24ad2417..6a9599a71 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -38,13 +38,6 @@ func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Ran return f(ctx, httpRange) } -// LinkRefresher 接口用于在读取数据失败时强制刷新链接 -type LinkRefresher interface { - // ForceRefreshLink 强制刷新下载链接 - // 返回 true 表示刷新成功，false 表示无法刷新 - ForceRefreshLink(ctx context.Context) bool -} - // IsLinkExpiredError checks if the error indicates an expired download link func IsLinkExpiredError(err error) bool { if err == nil { @@ -131,7 +124,16 @@ func (r *RefreshableRangeReader) RangeRead(ctx context.Context, httpRange http_r } } - return rc, nil + // Wrap the ReadCloser with self-healing capability to detect 0-byte reads + // This handles cases where cloud providers return 200 OK but empty body for expired links + return &selfHealingReadCloser{ + ReadCloser: rc, + refresher: r, + ctx: ctx, + httpRange: httpRange, + firstRead: false, + closed: false, + }, nil } func (r *RefreshableRangeReader) refreshAndRetry(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { @@ -149,19 +151,6 @@ func (r *RefreshableRangeReader) refreshAndRetry(ctx context.Context, httpRange return reader.RangeRead(ctx, httpRange) } -// ForceRefresh 强制刷新链接，用于读取数据失败（如读取 0 字节）的情况 -// 返回 true 表示刷新成功，false 表示无法刷新（没有 Refresher 或达到最大刷新次数） -func (r *RefreshableRangeReader) ForceRefresh(ctx context.Context) bool { - if r.link.Refresher == nil { - return false - } - - r.mu.Lock() - defer r.mu.Unlock() - - return r.doRefreshLocked(ctx) == nil -} - // doRefreshLocked 执行实际的刷新逻辑（需要持有锁） func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { if r.refreshCount >= MAX_LINK_REFRESH_COUNT { @@ -183,6 +172,84 @@ func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { return nil } +// selfHealingReadCloser wraps an io.ReadCloser and automatically refreshes the link +// if it detects 0-byte reads (common with expired links from some cloud providers) +type selfHealingReadCloser struct { + io.ReadCloser + refresher *RefreshableRangeReader + ctx context.Context + httpRange http_range.Range + firstRead bool + closed bool + mu sync.Mutex +} + +func (s *selfHealingReadCloser) Read(p []byte) (n int, err error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return 0, errors.New("read from closed reader") + } + + n, err = s.ReadCloser.Read(p) + + // Detect 0-byte read on first attempt (indicates link may be expired but returned 200 OK) + if !s.firstRead && n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { + s.firstRead = true + log.Warnf("Detected 0-byte read on first attempt, attempting to refresh link...") + + // Try to refresh the link + s.refresher.mu.Lock() + refreshErr := s.refresher.doRefreshLocked(s.ctx) + s.refresher.mu.Unlock() + + if refreshErr != nil { + log.Errorf("Failed to refresh link after 0-byte read: %v", refreshErr) + return n, err + } + + // Close old connection + s.ReadCloser.Close() + + // Get new reader and retry + s.refresher.mu.Lock() + reader, getErr := s.refresher.getInnerReader() + s.refresher.mu.Unlock() + + if getErr != nil { + log.Errorf("Failed to get inner reader after refresh: %v", getErr) + return n, err + } + + newRc, rangeErr := reader.RangeRead(s.ctx, s.httpRange) + if rangeErr != nil { + log.Errorf("Failed to create new range reader after refresh: %v", rangeErr) + return n, err + } + + s.ReadCloser = newRc + log.Infof("Successfully refreshed link and reconnected after 0-byte read") + + // Retry read with new connection + return s.ReadCloser.Read(p) + } + + s.firstRead = true + return n, err +} + +func (s *selfHealingReadCloser) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return nil + } + s.closed = true + return s.ReadCloser.Close() +} + func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { // If link has a Refresher, wrap with RefreshableRangeReader for automatic refresh on expiry if link.Refresher != nil { @@ -341,7 +408,7 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT // off: 读取的起始偏移量 // 返回值: 实际读取的字节数和错误 // 支持自动重试（最多5次），快速重试策略（1秒、2秒、3秒、4秒、5秒） -// 支持链接刷新：当检测到 0 字节读取时，会自动刷新下载链接 +// 注意：链接刷新现在由 RefreshableRangeReader 内部的 selfHealingReadCloser 自动处理 func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { length := int64(len(buf)) var lastErr error @@ -369,28 +436,8 @@ func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, lastErr = fmt.Errorf("failed to read all data via RangeRead at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) log.Debugf("RangeRead retry %d read failed: %v", retry+1, lastErr) - // 检测是否可能是链接过期（读取 0 字节或 EOF） - if n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { - // 尝试刷新链接 - if refresher, ok := file.(LinkRefresher); ok { - // 获取 context - 从 FileStream 或 SeekableStream 中获取 - var ctx context.Context - if fs, ok := file.(*FileStream); ok { - ctx = fs.Ctx - } else if ss, ok := file.(*SeekableStream); ok { - ctx = ss.Ctx - } else { - ctx = context.Background() - } - - if refresher.ForceRefreshLink(ctx) { - log.Infof("Link refreshed after 0-byte read, retrying immediately...") - continue // 立即重试，不延迟 - } - } - } - // 快速重试：1秒、2秒、3秒、4秒、5秒（读取失败快速重试） + // 注意：0字节读取导致的链接过期现在由 selfHealingReadCloser 自动处理 time.Sleep(time.Duration(retry+1) * time.Second) } From 34469231be47ca37c1359c70a1d4abaac12bdfe9 Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 13 Jan 2026 22:23:59 +0800 Subject: [PATCH 42/86] =?UTF-8?q?fix(google=5Fdrive):=20=E6=9B=B4=E6=96=B0?= =?UTF-8?q?Put=E6=96=B9=E6=B3=95=E4=BB=A5=E6=94=AF=E6=8C=81=E5=8F=AF?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E8=AF=BB=E5=8F=96=E6=B5=81=E5=92=8C=E4=B8=8D?= =?UTF-8?q?=E5=8F=AF=E9=87=8D=E5=A4=8D=E8=AF=BB=E5=8F=96=E6=B5=81=E7=9A=84?= =?UTF-8?q?MD5=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test_docker.yml | 3 +- drivers/google_drive/driver.go | 69 ++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index a3ca52258..0ecf05075 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -6,7 +6,8 @@ on: - main pull_request: branches: - - fix # 👈 允许你的 fix 分支触发 + - copy + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 94ef854f2..2496db95f 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -10,6 +10,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -111,8 +113,44 @@ func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - obj := stream.GetExist() +func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + // 1. 准备MD5（用于完整性校验） + md5Hash := file.GetHash().GetHash(utils.MD5) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(file, utils.MD5, 10, &up) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验（Google Drive会自动校验） + } + } else { + // 不可重复读取的流（如 HTTP body） + if len(md5Hash) != utils.MD5.Width { + // 缓存整个文件并计算 MD5 + var err error + _, md5Hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验 + } else if file.GetFile() == nil { + // 有 MD5 但没有缓存，需要缓存以支持后续 RangeRead + _, err := file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + } + + // 2. 初始化可恢复上传会话 + obj := file.GetExist() var ( e Error url string @@ -125,7 +163,7 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi data = base.Json{} } else { data = base.Json{ - "name": stream.GetName(), + "name": file.GetName(), "parents": []string{dstDir.GetID()}, } url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" @@ -133,8 +171,8 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi req := base.NoRedirectClient.R(). SetHeaders(map[string]string{ "Authorization": "Bearer " + d.AccessToken, - "X-Upload-Content-Type": stream.GetMimetype(), - "X-Upload-Content-Length": strconv.FormatInt(stream.GetSize(), 10), + "X-Upload-Content-Type": file.GetMimetype(), + "X-Upload-Content-Length": strconv.FormatInt(file.GetSize(), 10), }). SetError(&e).SetBody(data).SetContext(ctx) if obj != nil { @@ -151,20 +189,29 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi if err != nil { return err } - return d.Put(ctx, dstDir, stream, up) + return d.Put(ctx, dstDir, file, up) } return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } + + // 3. 上传文件内容 putUrl := res.Header().Get("location") - if stream.GetSize() < d.ChunkSize*1024*1024 { + if file.GetSize() < d.ChunkSize*1024*1024 { + // 小文件上传：使用 RangeRead 读取整个文件（避免消费已计算hash的stream） + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { - req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). - SetBody(driver.NewLimitedUploadStream(ctx, stream)) + req.SetHeader("Content-Length", strconv.FormatInt(file.GetSize(), 10)). + SetBody(driver.NewLimitedUploadStream(ctx, reader)) }, nil) + return err } else { - err = d.chunkUpload(ctx, stream, putUrl, up) + // 大文件分片上传 + return d.chunkUpload(ctx, file, putUrl, up) } - return err } func (d *GoogleDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { From 0f467a3e767456b662bed745430aa52048c398cb Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 14 Jan 2026 00:45:12 +0800 Subject: [PATCH 43/86] ci: use Ironboxplus/OpenList-Frontend for beta builds --- .github/workflows/test_docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 0ecf05075..6a3375b33 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -22,6 +22,8 @@ env: RELEASE_PLATFORMS: 'linux/amd64' # 👇 关键修改：强制允许推送，不用管是不是 push 事件 IMAGE_PUSH: 'true' + # 👇 使用自己的前端仓库 + FRONTEND_REPO: 'Ironboxplus/OpenList-Frontend' IMAGE_TAGS_BETA: | type=ref,event=pr type=raw,value=beta-retry From bf1711b50ea1088191da9196729af56569ad820c Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 14 Jan 2026 00:49:03 +0800 Subject: [PATCH 44/86] perf: optimize CI build speed with smart caching - Add Go module cache - Add frontend download cache with commit SHA tracking - Add Docker layer cache (registry-based) - Cache will invalidate when frontend repo updates --- .github/workflows/test_docker.yml | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 6a3375b33..8c1955298 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -35,9 +35,31 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + + - name: Setup Go + uses: actions/setup-go@v5 with: go-version: '1.25.0' + cache: true + cache-dependency-path: go.sum + + # 获取前端仓库的最新commit SHA + - name: Get Frontend Commit SHA + id: frontend-sha + run: | + FRONTEND_SHA=$(curl -s https://api.github.com/repos/${{ env.FRONTEND_REPO }}/commits/main | jq -r '.sha') + echo "sha=$FRONTEND_SHA" >> $GITHUB_OUTPUT + echo "Frontend repo latest commit: $FRONTEND_SHA" + + # 缓存前端下载 - key包含前端仓库的commit SHA + - name: Cache Frontend + id: cache-frontend + uses: actions/cache@v4 + with: + path: public/dist + key: frontend-${{ env.FRONTEND_REPO }}-${{ steps.frontend-sha.outputs.sha }} + restore-keys: | + frontend-${{ env.FRONTEND_REPO }}- # 即使只构建 x64，我们也需要 musl 工具链（因为 BuildDockerMultiplatform 默认会检查它） - name: Cache Musl @@ -46,6 +68,7 @@ jobs: with: path: build/musl-libs key: docker-musl-libs-v2 + - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' run: bash build.sh prepare docker-multiplatform @@ -57,7 +80,7 @@ jobs: run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} + FRONTEND_REPO: ${{ env.FRONTEND_REPO }} - name: Upload artifacts uses: actions/upload-artifact@v4 @@ -77,7 +100,7 @@ jobs: packages: write strategy: matrix: - # 你可以选择只构建 latest，或者保留全部变体 + # 构建所有变体 image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" @@ -137,3 +160,5 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.image }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.image }},mode=max From 2336cdc55506e197ccccde0ddcbaea77a9c306c9 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 14 Jan 2026 08:52:09 +0800 Subject: [PATCH 45/86] =?UTF-8?q?fix(stream):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E8=BF=87=E6=9C=9F=E6=A3=80=E6=B5=8B=E9=80=BB?= =?UTF-8?q?=E8=BE=91=EF=BC=8C=E9=81=BF=E5=85=8D=E5=B0=86=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E5=8F=96=E6=B6=88=E8=A7=86=E4=B8=BA=E9=93=BE=E6=8E=A5?= =?UTF-8?q?=E8=BF=87=E6=9C=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/stream/util.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/stream/util.go b/internal/stream/util.go index 6a9599a71..299f1345d 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -43,6 +43,13 @@ func IsLinkExpiredError(err error) bool { if err == nil { return false } + + // Don't treat context cancellation as link expiration + // This happens when user pauses/seeks video or cancels download + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + errStr := strings.ToLower(err.Error()) // Common expired link error keywords @@ -62,8 +69,8 @@ func IsLinkExpiredError(err error) bool { if statusErr, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { code := int(statusErr) // 401 Unauthorized, 403 Forbidden, 410 Gone are common for expired links - // 500 Internal Server Error - some providers (e.g., Baidu) return 500 when link expires - if code == 401 || code == 403 || code == 410 || code == 500 { + // Note: Removed 500 to avoid false positives from temporary network errors + if code == 401 || code == 403 || code == 410 { return true } } @@ -158,7 +165,9 @@ func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { } log.Infof("Link expired, attempting to refresh...") - newLink, _, refreshErr := r.link.Refresher(ctx) + // Use independent context for refresh to prevent cancellation from affecting link refresh + refreshCtx := context.WithoutCancel(ctx) + newLink, _, refreshErr := r.link.Refresher(refreshCtx) if refreshErr != nil { return fmt.Errorf("failed to refresh link: %w", refreshErr) } From 224ba4d2cc9d052640e561eb44e6edff762e7b5b Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 14 Jan 2026 08:58:41 +0800 Subject: [PATCH 46/86] =?UTF-8?q?refactor(workflow):=20=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=89=8D=E7=AB=AF=E4=BB=93=E5=BA=93=EF=BC=8C?= =?UTF-8?q?=E7=A7=BB=E9=99=A4=E8=87=AA=E5=AE=9A=E4=B9=89=E5=89=8D=E7=AB=AF?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test_docker.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 8c1955298..d3f4a8fff 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -22,8 +22,8 @@ env: RELEASE_PLATFORMS: 'linux/amd64' # 👇 关键修改：强制允许推送，不用管是不是 push 事件 IMAGE_PUSH: 'true' - # 👇 使用自己的前端仓库 - FRONTEND_REPO: 'Ironboxplus/OpenList-Frontend' + # 👇 使用默认的前端仓库 (OpenListTeam/OpenList-Frontend) + # FRONTEND_REPO: 'Ironboxplus/OpenList-Frontend' IMAGE_TAGS_BETA: | type=ref,event=pr type=raw,value=beta-retry @@ -47,8 +47,15 @@ jobs: - name: Get Frontend Commit SHA id: frontend-sha run: | - FRONTEND_SHA=$(curl -s https://api.github.com/repos/${{ env.FRONTEND_REPO }}/commits/main | jq -r '.sha') + FRONTEND_REPO="${{ env.FRONTEND_REPO }}" + # 如果未设置FRONTEND_REPO，使用默认值 + if [ -z "$FRONTEND_REPO" ]; then + FRONTEND_REPO="OpenListTeam/OpenList-Frontend" + fi + FRONTEND_SHA=$(curl -s https://api.github.com/repos/$FRONTEND_REPO/commits/main | jq -r '.sha') echo "sha=$FRONTEND_SHA" >> $GITHUB_OUTPUT + echo "repo=$FRONTEND_REPO" >> $GITHUB_OUTPUT + echo "Frontend repo: $FRONTEND_REPO" echo "Frontend repo latest commit: $FRONTEND_SHA" # 缓存前端下载 - key包含前端仓库的commit SHA @@ -57,9 +64,9 @@ jobs: uses: actions/cache@v4 with: path: public/dist - key: frontend-${{ env.FRONTEND_REPO }}-${{ steps.frontend-sha.outputs.sha }} + key: frontend-${{ steps.frontend-sha.outputs.repo }}-${{ steps.frontend-sha.outputs.sha }} restore-keys: | - frontend-${{ env.FRONTEND_REPO }}- + frontend-${{ steps.frontend-sha.outputs.repo }}- # 即使只构建 x64，我们也需要 musl 工具链（因为 BuildDockerMultiplatform 默认会检查它） - name: Cache Musl @@ -80,7 +87,7 @@ jobs: run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FRONTEND_REPO: ${{ env.FRONTEND_REPO }} + # FRONTEND_REPO 使用 build.sh 默认值 (OpenListTeam/OpenList-Frontend) - name: Upload artifacts uses: actions/upload-artifact@v4 From 4b40a1ece83cc9b82306c8992fd3d3b1e7c49135 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 14 Jan 2026 13:12:26 +0800 Subject: [PATCH 47/86] =?UTF-8?q?feat(google=5Fdrive):=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=A4=84=E7=90=86=E9=87=8D=E5=A4=8D=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=90=8D=E7=9A=84=E5=8A=9F=E8=83=BD=EF=BC=8C=E7=A1=AE=E4=BF=9D?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=90=8D=E5=94=AF=E4=B8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/google_drive/driver.go | 29 ++++++++++++++++++- drivers/google_drive/util.go | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 2496db95f..23c44d4a1 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strconv" + "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -14,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) type GoogleDrive struct { @@ -69,12 +71,37 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkA } func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + // Check if folder already exists to avoid duplicates + // Google Drive allows multiple files with same name, but we want unique folders + var existingFiles Files + // Escape single quotes in dirName to prevent query injection + escapedDirName := strings.ReplaceAll(dirName, "'", "\\'") + query := map[string]string{ + "q": fmt.Sprintf("name='%s' and '%s' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false", escapedDirName, parentDir.GetID()), + "fields": "files(id)", + } + _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &existingFiles) + + // If query succeeded and folder exists, return success (idempotent) + if err == nil && len(existingFiles.Files) > 0 { + log.Debugf("[google_drive] Folder '%s' already exists in parent %s, skipping creation", dirName, parentDir.GetID()) + return nil + } + // If query failed, return error to prevent duplicate creation + // Google Drive allows multiple files with same name, so we must ensure check succeeds + if err != nil { + return fmt.Errorf("failed to check existing folder '%s': %w", dirName, err) + } + + // Create new folder (only when confirmed folder doesn't exist) data := base.Json{ "name": dirName, "parents": []string{parentDir.GetID()}, "mimeType": "application/vnd.google-apps.folder", } - _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { + _, err = d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 042abafa4..1fc68c335 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -296,9 +296,60 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { res = append(res, resp.Files...) } + + // Handle duplicate filenames by adding suffixes like (1), (2), etc. + // Google Drive allows multiple files with the same name in one folder, + // but OpenList uses path-based file system which requires unique names + res = handleDuplicateNames(res) + return res, nil } +// handleDuplicateNames adds suffixes to duplicate filenames to make them unique +// For example: file.txt, file (1).txt, file (2).txt +func handleDuplicateNames(files []File) []File { + if len(files) <= 1 { + return files + } + + // Track how many files with each name we've seen + nameCount := make(map[string]int) + + // First pass: count occurrences of each name + for _, file := range files { + nameCount[file.Name]++ + } + + // Second pass: add suffixes to duplicates + nameIndex := make(map[string]int) + for i := range files { + name := files[i].Name + if nameCount[name] > 1 { + index := nameIndex[name] + nameIndex[name]++ + + if index > 0 { + // Add suffix for all except the first occurrence + // Split name into base and extension + ext := "" + base := name + for j := len(name) - 1; j >= 0; j-- { + if name[j] == '.' { + ext = name[j:] + base = name[:j] + break + } + } + + // Add (1), (2), etc. suffix + files[i].Name = fmt.Sprintf("%s (%d)%s", base, index, ext) + } + } + } + + return files +} + // getTargetFileInfo gets target file details for shortcuts func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) { var targetFile File From 25c5a8141ee4b41f05a92459472730f1f4e2de2f Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 17 Jan 2026 17:24:35 +0800 Subject: [PATCH 48/86] =?UTF-8?q?feat(quark=5Fopen):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E9=80=9F=E7=8E=87=E9=99=90=E5=88=B6=E5=92=8C=E9=87=8D=E8=AF=95?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=8C=E4=BC=98=E5=8C=96=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/quark_open/driver.go | 72 ++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/drivers/quark_open/driver.go b/drivers/quark_open/driver.go index 181c282f2..26a4288cc 100644 --- a/drivers/quark_open/driver.go +++ b/drivers/quark_open/driver.go @@ -20,15 +20,22 @@ import ( "github.com/avast/retry-go" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) type QuarkOpen struct { model.Storage Addition - config driver.Config - conf Conf + config driver.Config + conf Conf + limiter *rate.Limiter } +// 速率限制常量：夸克开放平台限流，保守设置 +const ( + quarkRateLimit = 2.0 // 每秒2个请求，避免限流 +) + func (d *QuarkOpen) Config() driver.Config { return d.config } @@ -38,6 +45,9 @@ func (d *QuarkOpen) GetAddition() driver.Additional { } func (d *QuarkOpen) Init(ctx context.Context) error { + // 初始化速率限制器 + d.limiter = rate.NewLimiter(rate.Limit(quarkRateLimit), 1) + var resp UserInfoResp _, err := d.request(ctx, "/open/v1/user/info", http.MethodGet, nil, &resp) @@ -54,11 +64,22 @@ func (d *QuarkOpen) Init(ctx context.Context) error { return err } +// waitLimit 等待速率限制 +func (d *QuarkOpen) waitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + func (d *QuarkOpen) Drop(ctx context.Context) error { return nil } func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } files, err := d.GetFiles(ctx, dir.GetID()) if err != nil { return nil, err @@ -69,6 +90,9 @@ func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs } func (d *QuarkOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } data := base.Json{ "fid": file.GetID(), } @@ -145,6 +169,9 @@ func (d *QuarkOpen) Remove(ctx context.Context, obj model.Obj) error { } func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + if err := d.waitLimit(ctx); err != nil { + return err + } md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) // 检查是否需要计算hash @@ -226,8 +253,32 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File } } } - // pre - pre, err := d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + // pre - 带有 proof fail 重试逻辑 + var pre UpPreResp + var err error + err = retry.Do(func() error { + var preErr error + pre, preErr = d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + if preErr != nil { + // 检查是否为 proof fail 错误 + if strings.Contains(preErr.Error(), "proof") || strings.Contains(preErr.Error(), "43010") { + log.Warnf("[quark_open] Proof verification failed, retrying: %v", preErr) + return preErr // 返回错误触发重试 + } + // 检查是否为限流错误 + if strings.Contains(preErr.Error(), "限流") || strings.Contains(preErr.Error(), "rate") { + log.Warnf("[quark_open] Rate limited, waiting before retry: %v", preErr) + time.Sleep(2 * time.Second) // 额外等待 + return preErr + } + } + return preErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) if err != nil { return err } @@ -237,6 +288,19 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return nil } + // 空文件特殊处理：跳过分片上传，直接调用 upFinish + // 由于夸克 API 对空文件处理不稳定，尝试完成上传，失败则直接成功返回 + if stream.GetSize() == 0 { + log.Infof("[quark_open] Empty file detected, attempting direct finish (task_id: %s)", pre.Data.TaskID) + err = d.upFinish(ctx, pre, []base.Json{}, []string{}) + if err != nil { + // 空文件 upFinish 失败，可能是 API 不支持，直接视为成功 + log.Warnf("[quark_open] Empty file upFinish failed: %v, treating as success", err) + } + up(100) + return nil + } + // 带重试的分片大小调整逻辑：如果检测到 "part list exceed" 错误，自动翻倍分片大小 var upUrlInfo UpUrlInfo var partInfo []base.Json From 6f6c07ae527538d8dfb233e40776c5034cc7c7f8 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 17 Jan 2026 20:50:29 +0800 Subject: [PATCH 49/86] =?UTF-8?q?feat(google=5Fdrive):=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E6=96=87=E4=BB=B6=E5=A4=B9=E5=88=9B=E5=BB=BA=E7=9A=84?= =?UTF-8?q?=E9=94=81=E6=9C=BA=E5=88=B6=E5=92=8C=E9=87=8D=E8=AF=95=E9=80=BB?= =?UTF-8?q?=E8=BE=91=EF=BC=8C=E7=A1=AE=E4=BF=9D=E6=96=87=E4=BB=B6=E5=A4=B9?= =?UTF-8?q?=E5=94=AF=E4=B8=80=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/google_drive/driver.go | 66 +++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 23c44d4a1..153f85408 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -6,6 +6,8 @@ import ( "net/http" "strconv" "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -14,10 +16,15 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) +// mkdirLocks prevents race conditions when creating folders with the same name +// Google Drive allows duplicate folder names, so we need application-level locking +var mkdirLocks sync.Map // map[string]*sync.Mutex - key is parentID + "/" + dirName + type GoogleDrive struct { model.Storage Addition @@ -71,18 +78,34 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkA } func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - // Check if folder already exists to avoid duplicates - // Google Drive allows multiple files with same name, but we want unique folders - var existingFiles Files - // Escape single quotes in dirName to prevent query injection + // Use per-folder lock to prevent concurrent creation of same folder + // This is critical because Google Drive allows duplicate folder names + lockKey := parentDir.GetID() + "/" + dirName + lockVal, _ := mkdirLocks.LoadOrStore(lockKey, &sync.Mutex{}) + lock := lockVal.(*sync.Mutex) + lock.Lock() + defer lock.Unlock() + + // Check if folder already exists with retry to handle API eventual consistency escapedDirName := strings.ReplaceAll(dirName, "'", "\\'") query := map[string]string{ "q": fmt.Sprintf("name='%s' and '%s' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false", escapedDirName, parentDir.GetID()), "fields": "files(id)", } - _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { - req.SetQueryParams(query) - }, &existingFiles) + + var existingFiles Files + err := retry.Do(func() error { + var checkErr error + _, checkErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &existingFiles) + return checkErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(200*time.Millisecond), + ) // If query succeeded and folder exists, return success (idempotent) if err == nil && len(existingFiles.Files) > 0 { @@ -90,7 +113,6 @@ func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName return nil } // If query failed, return error to prevent duplicate creation - // Google Drive allows multiple files with same name, so we must ensure check succeeds if err != nil { return fmt.Errorf("failed to check existing folder '%s': %w", dirName, err) } @@ -101,10 +123,30 @@ func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName "parents": []string{parentDir.GetID()}, "mimeType": "application/vnd.google-apps.folder", } - _, err = d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { - req.SetBody(data) - }, nil) - return err + + var createErr error + err = retry.Do(func() error { + _, createErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + return createErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) + + if err != nil { + return err + } + + // Wait briefly for API eventual consistency before releasing lock + // This helps prevent race conditions where a concurrent request + // checks for folder existence before the newly created folder is visible + time.Sleep(100 * time.Millisecond) + + return nil } func (d *GoogleDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { From c05068c0990c290e2d00cb48d7e832aca9b7c320 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 18 Jan 2026 11:44:24 +0800 Subject: [PATCH 50/86] =?UTF-8?q?refactor(link):=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E7=BC=93=E5=AD=98=E9=80=BB=E8=BE=91=EF=BC=8C?= =?UTF-8?q?=E7=A7=BB=E9=99=A4=E5=AF=B9=20SyncClosers=20=E7=9A=84=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=EF=BC=8C=E7=AE=80=E5=8C=96=E8=BF=87=E6=9C=9F=E5=88=A4?= =?UTF-8?q?=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/op/fs.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/internal/op/fs.go b/internal/op/fs.go index 29a31ad63..9711d416e 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -245,10 +245,9 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li } key := Key(storage, path) if ol, exists := Cache.linkCache.GetType(key, typeKey); exists { - if ol.link.Expiration != nil || - ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { - return ol.link, ol.obj, nil - } + // 缓存命中：直接返回缓存的链接 + // 链接过期由 RefreshableRangeReader 在 HTTP 请求失败时检测并刷新 + return ol.link, ol.obj, nil } fn := func() (*objWithLink, error) { @@ -290,19 +289,18 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if link.Expiration != nil { Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) } else { - Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) + // 不使用 SyncClosers 作为过期判断，使用默认 TTL + // 链接真正过期时由 RefreshableRangeReader 检测并刷新 + Cache.linkCache.SetType(key, typeKey, ol) } return ol, nil } - for { - ol, err, _ := linkG.Do(key+"/"+typeKey, fn) - if err != nil { - return nil, nil, err - } - if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { - return ol.link, ol.obj, nil - } + // 直接执行获取链接，不再依赖 SyncClosers 引用计数 + ol, err, _ := linkG.Do(key+"/"+typeKey, fn) + if err != nil { + return nil, nil, err } + return ol.link, ol.obj, nil } // Other api From 23972f3059508ae857bba346bfb349e4d4ec8d88 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 18 Jan 2026 12:05:49 +0800 Subject: [PATCH 51/86] =?UTF-8?q?fix(stream):=20=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E8=BF=87=E6=9C=9F=E9=93=BE=E6=8E=A5=E6=A3=80=E6=9F=A5=E9=80=BB?= =?UTF-8?q?=E8=BE=91=EF=BC=8C=E6=94=AF=E6=8C=81=E6=89=80=E6=9C=894xx?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AF=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/stream/util.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/stream/util.go b/internal/stream/util.go index 299f1345d..d54776399 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -68,9 +68,9 @@ func IsLinkExpiredError(err error) bool { // Check for HTTP status codes that typically indicate expired links if statusErr, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { code := int(statusErr) - // 401 Unauthorized, 403 Forbidden, 410 Gone are common for expired links - // Note: Removed 500 to avoid false positives from temporary network errors - if code == 401 || code == 403 || code == 410 { + // All 4xx client errors may indicate expired/invalid links + // 400 Bad Request, 401 Unauthorized, 403 Forbidden, 404 Not Found, 410 Gone, etc. + if code >= 400 && code < 500 { return true } } From 86c7a05d40c846941428675751219bc9b306192c Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 27 Jan 2026 16:44:45 +0800 Subject: [PATCH 52/86] =?UTF-8?q?fix(115=5Fopen):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E9=93=BE=E6=8E=A5=E9=94=99=E8=AF=AF=E5=A4=84?= =?UTF-8?q?=E7=90=86=EF=BC=8C=E9=80=9A=E8=BF=87=E6=A3=80=E6=B5=8B=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E7=A0=8110008=E8=87=AA=E5=8A=A8=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=B9=B6=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- COMPATIBILITY_REPORT.md | 204 ++++++++++++++ internal/offline_download/115_open/client.go | 47 +++- .../offline_download/115_open/client_test.go | 248 ++++++++++++++++++ 3 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 COMPATIBILITY_REPORT.md create mode 100644 internal/offline_download/115_open/client_test.go diff --git a/COMPATIBILITY_REPORT.md b/COMPATIBILITY_REPORT.md new file mode 100644 index 000000000..0b8be3b52 --- /dev/null +++ b/COMPATIBILITY_REPORT.md @@ -0,0 +1,204 @@ +# Rebase兼容性分析报告 + +## 提交概览 +共引入 **21个commits**，主要涉及以下模块： + +### 核心功能改动 + +#### 1. **链接刷新机制** (`internal/stream/util.go`) +**Commits**: +- `4c33ffa4` feat(link): add link refresh capability for expired download links +- `f38fe180` fix(stream): 修复链接过期检测逻辑，避免将上下文取消视为链接过期 +- `7cf362c6` fix(stream): 更新过期链接检查逻辑，支持所有4xx客户端错误 +- `03fbaf1c` refactor(stream): 移除过时的链接刷新逻辑，添加自愈读取器以处理0字节读取 + +**核心代码**: +```go +// 新增常量 +MAX_LINK_REFRESH_COUNT = 50 // 链接最大刷新次数 +MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead重试次数（从3提升到5） + +// 新增函数 +IsLinkExpiredError(err error) bool // 判断是否为链接过期错误 + +// 新增结构 +RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // 防止无限循环 +} + +selfHealingReadCloser struct { + // 检测0字节读取，自动刷新链接 +} +``` + +**功能说明**: +1. **链接过期检测**: 识别多种云盘的过期错误（expired, token expired, access denied, 4xx状态码等） +2. **自动刷新**: 检测到过期时自动调用Refresher获取新链接，最多刷新50次 +3. **自愈机制**: 处理某些云盘返回200但内容为空的情况（0字节读取检测） +4. **并发安全**: 使用sync.Mutex保护共享状态 +5. **Context隔离**: 刷新时使用WithoutCancel避免用户取消操作影响刷新 + +**潜在风险**: +- ✅ Context.WithoutCancel需要Go 1.21+ +- ✅ 并发场景下的锁竞争 +- ✅ refreshCount可能在某些场景下不递增导致无限循环 + +--- + +#### 2. **目录预创建优化** (`internal/fs/copy_move.go`) +**Commit**: `ce0da112` fix(copy_move): 将预创建子目录的深度从2级调整为1级 + +**核心代码**: +```go +func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath string, maxDepth int) error { + // 第一轮：创建直接子目录 + for _, obj := range objs { + if obj.IsDir() { + subdirPath := stdpath.Join(dstBasePath, obj.GetName()) + op.MakeDir(t.Ctx(), t.DstStorage, subdirPath) + subdirs = append(subdirs, obj) + } + } + + // 停止递归条件 + if maxDepth <= 0 { + return nil + } + + // 第二轮：递归创建嵌套目录 + for _, subdir := range subdirs { + subObjs := op.List(...) + preCreateDirectoryTree(subObjs, subdirDstPath, maxDepth-1) + } +} +``` + +**功能说明**: +1. **深度控制**: 默认maxDepth=1，只预创建2级目录（当前+子级） +2. **防止深度递归**: 避免在大型项目中递归过深导致栈溢出或性能问题 +3. **错误容忍**: MakeDir失败时继续处理其他目录 +4. **Context感知**: 每次循环检查ctx.Err()支持取消操作 + +**潜在风险**: +- ✅ op.MakeDir和op.List调用需要存储初始化 +- ✅ 大量目录时的性能问题 +- ✅ Context取消时的资源清理 + +--- + +#### 3. **网络优化** (`drivers/`, `internal/net/`) +**Commits**: +- `b9dafa65` feat(network): 增加对慢速网络的支持，调整超时和重试机制 +- `bce47884` fix(driver): 增加夸克分片大小调整逻辑，支持重试机制 +- `0b8471f6` feat(quark_open): 添加速率限制和重试逻辑 + +**功能说明**: +1. 提升RangeRead重试次数: 3 → 5 +2. 调整网络超时参数 +3. 添加分片上传重试逻辑 + +--- + +#### 4. **驱动修复** +**Commits**: +- `da2812c0` fix(google_drive): 更新Put方法以支持可重复读取流和不可重复读取流的MD5校验 +- `5a6bad90` feat(google_drive): 添加文件夹创建的锁机制和重试逻辑 +- `a54b2388` feat(google_drive): 添加处理重复文件名的功能 +- `9ef22ec9` fix(driver): fix file copy failure to 123pan due to incorrect etag +- `0ead87ef` fix(alias): update storage retrieval method in listRoot function +- `311f6246` fix: 修复500 panic和NaN问题 + +--- + +## 兼容性评估 + +### ✅ 编译兼容性 +- 构建成功，无语法错误 +- 依赖版本无冲突 + +### ✅ API兼容性 +- 新增函数不破坏现有接口 +- RefreshableRangeReader实现model.RangeReaderIF接口 +- 向后兼容旧代码 + +### ⚠️ 运行时兼容性 +**需要验证的场景**: +1. **并发安全**: RefreshableRangeReader的并发读取 +2. **资源泄漏**: Context取消时goroutine是否正确退出 +3. **边界条件**: + - refreshCount达到50次的行为 + - 0字节读取检测的准确性 + - maxDepth=0时的目录创建 +4. **错误处理**: + - nil Refresher时的处理 + - 链接刷新失败时的回退机制 +5. **性能**: + - 大文件下载时的刷新开销 + - 深层目录结构的预创建性能 + +--- + +## 测试需求 + +### 必须测试的场景 + +#### Stream包测试 +1. **IsLinkExpiredError准确性** + - 各种云盘的过期错误格式 + - Context取消不应判断为过期 + - HTTP 4xx/5xx的区分 + +2. **RefreshableRangeReader可靠性** + - 正常读取流程 + - 自动刷新触发和成功 + - 达到最大刷新次数 + - 并发读取安全性 + - Context取消的正确处理 + +3. **selfHealingReadCloser** + - 0字节读取检测 + - 刷新重试机制 + - 资源正确关闭 + +#### FS包测试 +1. **preCreateDirectoryTree** + - 深度控制正确性(0, 1, 2级) + - 大量目录的性能 + - Context取消的响应 + - 错误容忍性 + +--- + +## 风险等级: **中等** + +**原因**: +- ✅ 新功能设计合理，有明确的边界和错误处理 +- ⚠️ 并发场景需要充分测试 +- ⚠️ 链接刷新逻辑复杂，需要验证各种边界情况 +- ⚠️ 依赖op包的函数需要正确的初始化 + +--- + +## 推送建议: **通过测试后可推送** + +**前置条件**: +1. 完成全面的单元测试（见下方测试代码） +2. 验证并发安全性 +3. 确认Context取消不会导致资源泄漏 +4. 性能测试通过（大文件、深层目录） + +**建议测试命令**: +```bash +# 单元测试 +go test ./internal/stream ./internal/fs -v -count=1 -race + +# 压力测试 +go test ./internal/stream -run Stress -v -count=10 + +# 完整测试套件 +go test ./... -short -count=1 +``` diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index d12e02ec5..adbff344f 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -3,6 +3,7 @@ package _115_open import ( "context" "fmt" + "strings" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -70,10 +71,54 @@ func (o *Open115) AddURL(args *tool.AddUrlArgs) (string, error) { } hashs, err := driver115Open.OfflineDownload(ctx, []string{args.Url}, parentDir) - if err != nil || len(hashs) < 1 { + + // 检查是否是重复链接错误 (code: 10008) + if err != nil { + // 尝试从错误信息中判断是否是重复链接 + errStr := err.Error() + isDuplicateError := false + + // 检查是否包含"重复"、"已存在"等关键词，或错误码10008 + if strings.Contains(errStr, "10008") || + strings.Contains(errStr, "重复") || + strings.Contains(errStr, "已存在") || + strings.Contains(errStr, "duplicate") { + isDuplicateError = true + } + + if isDuplicateError { + // 尝试查找并删除已存在的相同URL任务，然后重试 + taskList, listErr := driver115Open.OfflineList(ctx) + if listErr == nil && taskList != nil { + // 查找匹配的任务 + for _, task := range taskList.Tasks { + if task.URL == args.Url { + // 找到重复任务，删除它 + deleteErr := driver115Open.DeleteOfflineTask(ctx, task.InfoHash, false) + if deleteErr == nil { + // 删除成功，重新尝试添加 + hashs, err = driver115Open.OfflineDownload(ctx, []string{args.Url}, parentDir) + if err != nil { + return "", fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + if len(hashs) > 0 { + return hashs[0], nil + } + } + break + } + } + } + } + + // 如果不是重复错误或处理失败，返回原始错误 return "", fmt.Errorf("failed to add offline download task: %w", err) } + if len(hashs) < 1 { + return "", fmt.Errorf("failed to add offline download task: no task hash returned") + } + return hashs[0], nil } diff --git a/internal/offline_download/115_open/client_test.go b/internal/offline_download/115_open/client_test.go new file mode 100644 index 000000000..b675e65fe --- /dev/null +++ b/internal/offline_download/115_open/client_test.go @@ -0,0 +1,248 @@ +package _115_open + +import ( + "context" + "fmt" + "strings" + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" + _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" +) + +// Mock implementation of Open115 driver for testing +type mockOpen115 struct { + _115_open.Open115 + offlineDownloadFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + offlineListFunc func(ctx context.Context) (*sdk.OfflineTaskListResp, error) + deleteOfflineFunc func(ctx context.Context, infoHash string, deleteFiles bool) error +} + +func (m *mockOpen115) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + if m.offlineDownloadFunc != nil { + return m.offlineDownloadFunc(ctx, uris, dstDir) + } + return nil, fmt.Errorf("not implemented") +} + +func (m *mockOpen115) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + if m.offlineListFunc != nil { + return m.offlineListFunc(ctx) + } + return nil, fmt.Errorf("not implemented") +} + +func (m *mockOpen115) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { + if m.deleteOfflineFunc != nil { + return m.deleteOfflineFunc(ctx, infoHash, deleteFiles) + } + return fmt.Errorf("not implemented") +} + +// TestAddURL_Success tests successful URL addition +func TestAddURL_Success(t *testing.T) { + t.Skip("需要真实的storage环境，跳过此测试") +} + +// TestAddURL_DuplicateHandling tests the duplicate URL handling logic +func TestAddURL_DuplicateHandling(t *testing.T) { + t.Skip("需要真实的storage环境，跳过此测试") +} + +// TestDuplicateLinkRetryLogic tests the logic without actual API calls +func TestDuplicateLinkRetryLogic(t *testing.T) { + testURL := "https://example.com/test.torrent" + testHash := "test_hash_123" + + t.Run("首次添加成功", func(t *testing.T) { + // 模拟首次添加成功的场景 + callCount := 0 + mock := &mockOpen115{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return []string{testHash}, nil + } + return nil, fmt.Errorf("unexpected call") + }, + } + + hashes, err := mock.OfflineDownload(context.Background(), []string{testURL}, nil) + if err != nil { + t.Errorf("首次添加失败: %v", err) + } + if len(hashes) != 1 || hashes[0] != testHash { + t.Errorf("期望hash=%s, 实际=%v", testHash, hashes) + } + if callCount != 1 { + t.Errorf("期望调用1次, 实际调用%d次", callCount) + } + }) + + t.Run("检测到重复错误并自动删除重试", func(t *testing.T) { + // 模拟重复链接错误的场景 + callCount := 0 + deleteCount := 0 + + mock := &mockOpen115{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + // 首次调用返回重复错误 + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } else if callCount == 2 { + // 删除后重试，返回成功 + return []string{testHash}, nil + } + return nil, fmt.Errorf("unexpected call count: %d", callCount) + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + // 返回包含重复任务的列表 + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + { + InfoHash: "old_hash_456", + URL: testURL, + Status: 1, // 下载中 + }, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != "old_hash_456" { + t.Errorf("期望删除hash=old_hash_456, 实际=%s", infoHash) + } + if deleteFiles { + t.Error("不应该删除源文件") + } + return nil + }, + } + + // 模拟完整的错误处理逻辑 + ctx := context.Background() + + // 第一次调用返回重复错误 + _, err := mock.OfflineDownload(ctx, []string{testURL}, nil) + if err == nil { + t.Error("第一次应该返回错误") + } + + // 检查是否是重复错误 + errStr := err.Error() + if !strings.Contains(errStr, "10008") && !strings.Contains(errStr, "重复") { + t.Errorf("应该是重复错误，实际错误: %v", err) + } + + // 获取任务列表 + taskList, err := mock.OfflineList(ctx) + if err != nil { + t.Errorf("获取任务列表失败: %v", err) + } + + // 查找并删除重复任务 + found := false + for _, task := range taskList.Tasks { + if task.URL == testURL { + err := mock.DeleteOfflineTask(ctx, task.InfoHash, false) + if err != nil { + t.Errorf("删除任务失败: %v", err) + } + found = true + break + } + } + + if !found { + t.Error("未找到重复任务") + } + + // 重试添加 + hashes, err := mock.OfflineDownload(ctx, []string{testURL}, nil) + if err != nil { + t.Errorf("重试添加失败: %v", err) + } + if len(hashes) != 1 || hashes[0] != testHash { + t.Errorf("期望hash=%s, 实际=%v", testHash, hashes) + } + + if callCount != 2 { + t.Errorf("期望调用OfflineDownload 2次, 实际%d次", callCount) + } + if deleteCount != 1 { + t.Errorf("期望调用DeleteOfflineTask 1次, 实际%d次", deleteCount) + } + }) + + t.Run("重复链接但删除失败", func(t *testing.T) { + mock := &mockOpen115{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + // 始终返回重复错误 + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + { + InfoHash: "old_hash_789", + URL: testURL, + }, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return fmt.Errorf("删除失败：权限不足") + }, + } + + // 删除失败时应该返回错误 + ctx := context.Background() + _, err := mock.OfflineDownload(ctx, []string{testURL}, nil) + if err == nil { + t.Error("应该返回错误") + } + }) +} + +// TestOpen115_Name tests the Name method +func TestOpen115_Name(t *testing.T) { + o := &Open115{} + name := o.Name() + expected := "115 Open" + if name != expected { + t.Errorf("期望名称=%s, 实际=%s", expected, name) + } +} + +// TestOpen115_Items tests the Items method +func TestOpen115_Items(t *testing.T) { + o := &Open115{} + items := o.Items() + if items != nil { + t.Error("Items应该返回nil") + } +} + +// TestOpen115_Run tests the Run method +func TestOpen115_Run(t *testing.T) { + o := &Open115{} + err := o.Run(&tool.DownloadTask{}) + if err == nil { + t.Error("Run应该返回NotSupport错误") + } +} + +// TestOpen115_Init tests the Init method +func TestOpen115_Init(t *testing.T) { + o := &Open115{} + msg, err := o.Init() + if err != nil { + t.Errorf("Init失败: %v", err) + } + if msg != "ok" { + t.Errorf("期望消息='ok', 实际=%s", msg) + } +} From 40a4514f42c0053ca6b8422e8e820a8e26127873 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 1 Feb 2026 22:13:40 +0800 Subject: [PATCH 53/86] =?UTF-8?q?feat(offline=5Fdownload):=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E4=BB=BB=E5=8A=A1=E5=88=97=E8=A1=A8=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=8C=E6=94=AF=E6=8C=81=E5=A4=9A=E9=A1=B5?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E8=8E=B7=E5=8F=96=E5=B9=B6=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E7=8A=B6=E6=80=81=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/115_open/driver.go | 11 +++++++++++ internal/offline_download/115_open/client.go | 4 ++-- internal/offline_download/tool/download.go | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index f95ab429b..9033365a5 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -420,10 +420,21 @@ func (d *Open115) DeleteOfflineTask(ctx context.Context, infoHash string, delete } func (d *Open115) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + // 获取第一页 resp, err := d.client.OfflineTaskList(ctx, 1) if err != nil { return nil, err } + // 如果有多页，获取所有页面的任务 + if resp.PageCount > 1 { + for page := 2; page <= resp.PageCount; page++ { + pageResp, err := d.client.OfflineTaskList(ctx, int64(page)) + if err != nil { + return nil, err + } + resp.Tasks = append(resp.Tasks, pageResp.Tasks...) + } + } return resp, nil } diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index adbff344f..e8d33326b 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -174,8 +174,8 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { return s, nil } } - s.Err = fmt.Errorf("the task has been deleted") - return nil, nil + // 任务不在列表中，可能已完成或被删除 + return nil, fmt.Errorf("task %s not found in offline list", task.GID) } var _ tool.Tool = (*Open115)(nil) diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 5ee6ef4ff..e033cccba 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -147,7 +147,7 @@ func (t *DownloadTask) Update() (bool, error) { if err != nil { t.callStatusRetried++ log.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) - if t.callStatusRetried > 5 { + if t.callStatusRetried > 10 { return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) } return false, nil From d996474d7a5b561988426e7facee342676c32804 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 9 Feb 2026 22:17:16 +0800 Subject: [PATCH 54/86] =?UTF-8?q?fix(google=5Fdrive):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=20MakeDir=20=E6=96=B9=E6=B3=95=E4=B8=AD=E7=9A=84=E7=AD=89?= =?UTF-8?q?=E5=BE=85=E6=97=B6=E9=97=B4=EF=BC=8C=E4=BB=A5=E5=A4=84=E7=90=86?= =?UTF-8?q?=20Google=20Drive=20API=20=E7=9A=84=E5=90=8C=E6=AD=A5=E5=BB=B6?= =?UTF-8?q?=E8=BF=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/google_drive/driver.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 153f85408..61c182bfb 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -141,10 +141,11 @@ func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName return err } - // Wait briefly for API eventual consistency before releasing lock + // Wait for API eventual consistency before releasing lock // This helps prevent race conditions where a concurrent request // checks for folder existence before the newly created folder is visible - time.Sleep(100 * time.Millisecond) + // 500ms is needed because Google Drive API has significant sync delay + time.Sleep(500 * time.Millisecond) return nil } From 70ddb51d5cb3b184809d106f4e4661552e657826 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 18 Feb 2026 11:09:29 +0800 Subject: [PATCH 55/86] =?UTF-8?q?fix(link):=20=E4=BF=AE=E5=A4=8D=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E6=96=87=E4=BB=B6=20'file=20already=20closed'=20?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 恢复链接缓存命中时的 SyncClosers 引用计数检查： - 缓存命中时若文件句柄已关闭，删除缓存条目并重新获取 - RequireReference 为 true 时用 SetTypeWithExpirable 绑定文件句柄生命周期 - 无需引用计数的链接仍使用默认 TTL，保持多客户端复用行为 - 恢复 for 循环处理 singleflight 返回已关闭句柄的竞态情况 --- internal/op/fs.go | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/internal/op/fs.go b/internal/op/fs.go index 9711d416e..534e73697 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -245,9 +245,12 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li } key := Key(storage, path) if ol, exists := Cache.linkCache.GetType(key, typeKey); exists { - // 缓存命中：直接返回缓存的链接 - // 链接过期由 RefreshableRangeReader 在 HTTP 请求失败时检测并刷新 - return ol.link, ol.obj, nil + if ol.link.Expiration != nil || + ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil + } + // SyncClosers 已关闭（文件句柄已关闭），删除缓存条目，重新获取 + Cache.linkCache.DeleteKey(key) } fn := func() (*objWithLink, error) { @@ -288,19 +291,24 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li ol := &objWithLink{link: link, obj: file} if link.Expiration != nil { Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) + } else if link.RequireReference { + // 本地文件等需要引用计数的链接，缓存与文件句柄生命周期绑定 + Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) } else { - // 不使用 SyncClosers 作为过期判断，使用默认 TTL - // 链接真正过期时由 RefreshableRangeReader 检测并刷新 + // 不需要引用计数（如云盘链接无过期时间），使用默认 TTL，多客户端复用 Cache.linkCache.SetType(key, typeKey, ol) } return ol, nil } - // 直接执行获取链接，不再依赖 SyncClosers 引用计数 - ol, err, _ := linkG.Do(key+"/"+typeKey, fn) - if err != nil { - return nil, nil, err + for { + ol, err, _ := linkG.Do(key+"/"+typeKey, fn) + if err != nil { + return nil, nil, err + } + if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil + } } - return ol.link, ol.obj, nil } // Other api From 43c186a1e826c6b4552a55cbc6e6ae1d575cd202 Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 25 Feb 2026 12:31:25 +0800 Subject: [PATCH 56/86] fix(fs): fix srcBasePath bug and remove redundant sleep in preCreateDirectoryTree - Add srcBasePath parameter to preCreateDirectoryTree to properly track the current source directory during recursion. The old code used t.SrcActualPath (the top-level path) for all recursion levels, causing incorrect op.List paths when maxDepth > 0. This was latent with maxDepth=1 (second pass returned early at maxDepth=0 before the bug triggered) but would break if maxDepth is raised. - Remove the 50ms time.Sleep added per MakeDir call. Drivers that enforce QPS limits (e.g. 115, 115_open) already call d.WaitLimit(ctx) -- a token-bucket rate.Limiter with burst=1 -- inside their own MakeDir implementation, so op.MakeDir naturally blocks at the correct per-driver rate. An unconditional sleep would penalise all other drivers (S3, WebDAV, etc.) with no benefit. --- internal/fs/copy_move.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index 77c2015b5..58fbc0534 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -203,7 +203,7 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer // Pre-create subdirectories (up to 1 level deep) to avoid deep recursion issues // Balances between reducing API calls and maintaining fault tolerance t.Status = "pre-creating subdirectories" - if err := t.preCreateDirectoryTree(objs, dstActualPath, 1); err != nil { + if err := t.preCreateDirectoryTree(objs, t.SrcActualPath, dstActualPath, 1); err != nil { log.Warnf("[copy_move] failed to pre-create directory tree: %v, will continue", err) // Continue anyway - individual directories will be created on-demand } @@ -282,7 +282,9 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer // preCreateDirectoryTree recursively scans source directory tree and pre-creates // directories on destination up to maxDepth levels to avoid deep MakeDir recursion issues. // maxDepth=0 means only current level, maxDepth=1 means current+1 level, etc. -func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath string, maxDepth int) error { +// srcBasePath is the current source directory being scanned (must be passed explicitly to +// support correct path building during recursion; do NOT use t.SrcActualPath inside). +func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, srcBasePath, dstBasePath string, maxDepth int) error { // First pass: create immediate subdirectories var subdirs []model.Obj for _, obj := range objs { @@ -298,6 +300,9 @@ func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath // Continue with other directories } subdirs = append(subdirs, obj) + // No explicit sleep here: drivers that have QPS limits (e.g. 115, BaiduNetDisk) + // implement WaitLimit via a token-bucket rate.Limiter and call it inside their + // MakeDir, so op.MakeDir already blocks at the correct per-driver rate. } } @@ -312,8 +317,9 @@ func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath return err } - // List contents of this subdirectory - subdirSrcPath := stdpath.Join(t.SrcActualPath, subdir.GetName()) + // Build paths relative to srcBasePath (NOT t.SrcActualPath) so that + // deeper recursion levels resolve to the correct source paths. + subdirSrcPath := stdpath.Join(srcBasePath, subdir.GetName()) subdirDstPath := stdpath.Join(dstBasePath, subdir.GetName()) subObjs, err := op.List(t.Ctx(), t.SrcStorage, subdirSrcPath, model.ListArgs{}) @@ -323,7 +329,7 @@ func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath } // Recursively create subdirectories with decreased depth - if err := t.preCreateDirectoryTree(subObjs, subdirDstPath, maxDepth-1); err != nil { + if err := t.preCreateDirectoryTree(subObjs, subdirSrcPath, subdirDstPath, maxDepth-1); err != nil { return err } } From fdee9f09156c26d928b51fa77c357273e29db73d Mon Sep 17 00:00:00 2001 From: cyk Date: Wed, 25 Feb 2026 12:40:48 +0800 Subject: [PATCH 57/86] test(fs): add unit tests for preCreateDirTreeFn; refactor for testability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the core recursion logic from preCreateDirectoryTree into a pure helper preCreateDirTreeFn that accepts makeDir and listSrc as injected function parameters. The method wrapper passes the real op.MakeDir / op.List closures unchanged, so production behavior is identical. This makes the function unit-testable without a real storage driver or database. Added 11 tests covering: - empty / file-only objs → no MakeDir calls - flat dirs at maxDepth=0 → correct dst paths, no List calls - srcBasePath regression: recursive List must use subdirSrcPath not the fixed t.SrcActualPath (the original bug that was latent at maxDepth=1) - maxDepth boundary: recursion stops exactly at the configured depth - context cancellation (immediate and mid-recursion) - MakeDir error non-fatal: remaining dirs still processed - List error non-fatal: other subdirs still recursed - mixed file+dir objects: only dirs trigger MakeDir - context timeout --- internal/fs/copy_move.go | 45 +++-- internal/fs/copy_move_test.go | 317 ++++++++++++++++++++++++++++++++++ 2 files changed, 351 insertions(+), 11 deletions(-) create mode 100644 internal/fs/copy_move_test.go diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index 58fbc0534..5ae92524e 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -207,7 +207,6 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer log.Warnf("[copy_move] failed to pre-create directory tree: %v, will continue", err) // Continue anyway - individual directories will be created on-demand } - existedObjs := make(map[string]bool) if t.TaskType == merge { dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) @@ -279,23 +278,47 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } -// preCreateDirectoryTree recursively scans source directory tree and pre-creates -// directories on destination up to maxDepth levels to avoid deep MakeDir recursion issues. -// maxDepth=0 means only current level, maxDepth=1 means current+1 level, etc. -// srcBasePath is the current source directory being scanned (must be passed explicitly to -// support correct path building during recursion; do NOT use t.SrcActualPath inside). +// preCreateDirectoryTree is a thin method wrapper that resolves the storage-bound +// makeDir / listSrc functions and delegates to the pure preCreateDirTreeFn helper. func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, srcBasePath, dstBasePath string, maxDepth int) error { + makeDir := func(ctx context.Context, path string) error { + return op.MakeDir(ctx, t.DstStorage, path) + } + listSrc := func(ctx context.Context, path string) ([]model.Obj, error) { + return op.List(ctx, t.SrcStorage, path, model.ListArgs{}) + } + return preCreateDirTreeFn(t.Ctx(), objs, srcBasePath, dstBasePath, maxDepth, makeDir, listSrc) +} + +// preCreateDirTreeFn recursively scans source directory tree and pre-creates +// directories on destination up to maxDepth levels to avoid deep MakeDir recursion issues. +// +// - maxDepth=0 – only create dirs in the current objs list (no recursion) +// - maxDepth=1 – also recurse one level deeper, etc. +// - srcBasePath – current source directory path; passed explicitly through all +// recursion levels so that subdirSrcPath is always correct (do NOT use +// t.SrcActualPath, which is fixed at the top-level path). +// +// makeDir and listSrc are injected to enable testing without a real storage driver. +func preCreateDirTreeFn( + ctx context.Context, + objs []model.Obj, + srcBasePath, dstBasePath string, + maxDepth int, + makeDir func(context.Context, string) error, + listSrc func(context.Context, string) ([]model.Obj, error), +) error { // First pass: create immediate subdirectories var subdirs []model.Obj for _, obj := range objs { // Check for cancellation - if err := t.Ctx().Err(); err != nil { + if err := ctx.Err(); err != nil { return err } if obj.IsDir() { subdirPath := stdpath.Join(dstBasePath, obj.GetName()) - if err := op.MakeDir(t.Ctx(), t.DstStorage, subdirPath); err != nil { + if err := makeDir(ctx, subdirPath); err != nil { log.Debugf("[copy_move] failed to pre-create dir [%s]: %v", subdirPath, err) // Continue with other directories } @@ -313,7 +336,7 @@ func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, srcBasePath, // Second pass: recursively scan and create nested subdirectories for _, subdir := range subdirs { - if err := t.Ctx().Err(); err != nil { + if err := ctx.Err(); err != nil { return err } @@ -322,14 +345,14 @@ func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, srcBasePath, subdirSrcPath := stdpath.Join(srcBasePath, subdir.GetName()) subdirDstPath := stdpath.Join(dstBasePath, subdir.GetName()) - subObjs, err := op.List(t.Ctx(), t.SrcStorage, subdirSrcPath, model.ListArgs{}) + subObjs, err := listSrc(ctx, subdirSrcPath) if err != nil { log.Debugf("[copy_move] failed to list subdir [%s] for pre-creation: %v", subdirSrcPath, err) continue // Skip this subdirectory, will handle when processing } // Recursively create subdirectories with decreased depth - if err := t.preCreateDirectoryTree(subObjs, subdirSrcPath, subdirDstPath, maxDepth-1); err != nil { + if err := preCreateDirTreeFn(ctx, subObjs, subdirSrcPath, subdirDstPath, maxDepth-1, makeDir, listSrc); err != nil { return err } } diff --git a/internal/fs/copy_move_test.go b/internal/fs/copy_move_test.go new file mode 100644 index 000000000..5c9f4b539 --- /dev/null +++ b/internal/fs/copy_move_test.go @@ -0,0 +1,317 @@ +package fs + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// ---------- helpers ---------- + +func dirObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: true} +} + +func fileObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: false} +} + +// callRecorder records every path passed to makeDir and listSrc. +type callRecorder struct { + mu sync.Mutex + mkdirs []string + lists []string + // listReturns maps srcPath → objects to return (nil = empty) + listReturns map[string][]model.Obj + // mkdirErr maps dstPath → error to return + mkdirErr map[string]error +} + +func newRecorder() *callRecorder { + return &callRecorder{ + listReturns: make(map[string][]model.Obj), + mkdirErr: make(map[string]error), + } +} + +func (r *callRecorder) makeDir(_ context.Context, path string) error { + r.mu.Lock() + r.mkdirs = append(r.mkdirs, path) + err := r.mkdirErr[path] + r.mu.Unlock() + return err +} + +func (r *callRecorder) listSrc(_ context.Context, path string) ([]model.Obj, error) { + r.mu.Lock() + r.lists = append(r.lists, path) + objs := r.listReturns[path] + r.mu.Unlock() + return objs, nil +} + +func (r *callRecorder) hasMkdir(path string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range r.mkdirs { + if p == path { + return true + } + } + return false +} + +func (r *callRecorder) hasList(path string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range r.lists { + if p == path { + return true + } + } + return false +} + +// ---------- tests ---------- + +// TestPreCreateDirTreeFn_EmptyObjs: no objects → no calls at all. +func TestPreCreateDirTreeFn_EmptyObjs(t *testing.T) { + rec := newRecorder() + err := preCreateDirTreeFn(context.Background(), nil, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("expected 0 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + } + if len(rec.lists) != 0 { + t.Errorf("expected 0 List calls, got %d: %v", len(rec.lists), rec.lists) + } +} + +// TestPreCreateDirTreeFn_OnlyFiles: file objects only → zero MakeDir calls. +func TestPreCreateDirTreeFn_OnlyFiles(t *testing.T) { + objs := []model.Obj{fileObj("a.txt"), fileObj("b.txt")} + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("expected 0 MakeDir calls, got %d", len(rec.mkdirs)) + } +} + +// TestPreCreateDirTreeFn_FlatDirs_MaxDepth0: dirs present, maxDepth=0 → MakeDir +// called for each dir with correct dstPath, NO listSrc calls. +func TestPreCreateDirTreeFn_FlatDirs_MaxDepth0(t *testing.T) { + objs := []model.Obj{dirObj("subA"), fileObj("file.txt"), dirObj("subB")} + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst/parent", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !rec.hasMkdir("/dst/parent/subA") { + t.Error("expected MakeDir(/dst/parent/subA)") + } + if !rec.hasMkdir("/dst/parent/subB") { + t.Error("expected MakeDir(/dst/parent/subB)") + } + if rec.hasMkdir("/dst/parent/file.txt") { + t.Error("MakeDir must NOT be called for a file") + } + if len(rec.lists) != 0 { + t.Errorf("maxDepth=0 must not trigger any List calls, got: %v", rec.lists) + } +} + +// TestPreCreateDirTreeFn_Recursion_CorrectSrcPath is the regression test for the +// srcBasePath bug: with maxDepth=1 the recursive List must use the SUBDIR src path, +// not the original top-level srcBasePath. +func TestPreCreateDirTreeFn_Recursion_CorrectSrcPath(t *testing.T) { + // /src/parent contains [subA(dir), subB(dir)] + // /src/parent/subA contains [subA1(dir)] + // /src/parent/subB contains [] + topObjs := []model.Obj{dirObj("subA"), dirObj("subB")} + rec := newRecorder() + rec.listReturns["/src/parent/subA"] = []model.Obj{dirObj("subA1")} + rec.listReturns["/src/parent/subB"] = []model.Obj{} + + if err := preCreateDirTreeFn(context.Background(), topObjs, "/src/parent", "/dst/parent", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // ── first level dirs must be created + if !rec.hasMkdir("/dst/parent/subA") { + t.Error("expected MakeDir(/dst/parent/subA)") + } + if !rec.hasMkdir("/dst/parent/subB") { + t.Error("expected MakeDir(/dst/parent/subB)") + } + + // ── listSrc must use subdirSrcPath (NOT the whole /src/parent again) + if !rec.hasList("/src/parent/subA") { + t.Error("listSrc must be called with /src/parent/subA, got:", rec.lists) + } + if !rec.hasList("/src/parent/subB") { + t.Error("listSrc must be called with /src/parent/subB, got:", rec.lists) + } + // The original bug would have called listSrc("/src/parent/subA") as + // stdpath.Join(t.SrcActualPath, "subA") where t.SrcActualPath=="/src/parent", + // but in a deeper recursive call (e.g. maxDepth=2) it would have used + // the top-level path incorrectly; verify the nested mkdir used the right dst. + if !rec.hasMkdir("/dst/parent/subA/subA1") { + t.Error("expected MakeDir(/dst/parent/subA/subA1), got mkdirs:", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion: with maxDepth=1 recursion +// goes exactly one level. The nested list returns another dir, but since maxDepth +// reaches 0 that deeper dir must NOT be listed further. +func TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion(t *testing.T) { + topObjs := []model.Obj{dirObj("sub")} + rec := newRecorder() + // sub contains deeper, deeper contains deepest + rec.listReturns["/src/sub"] = []model.Obj{dirObj("deeper")} + rec.listReturns["/src/sub/deeper"] = []model.Obj{dirObj("deepest")} // should NOT be listed + + if err := preCreateDirTreeFn(context.Background(), topObjs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !rec.hasMkdir("/dst/sub") { + t.Error("expected /dst/sub to be created") + } + if !rec.hasMkdir("/dst/sub/deeper") { + t.Error("expected /dst/sub/deeper to be created (within maxDepth=1)") + } + // deepest must NOT be created (would require maxDepth=2) + if rec.hasMkdir("/dst/sub/deeper/deepest") { + t.Error("/dst/sub/deeper/deepest must NOT be created at maxDepth=1") + } + // /src/sub/deeper must NOT be listed (we've hit maxDepth=0 at that point) + if rec.hasList("/src/sub/deeper") { + t.Error("/src/sub/deeper must NOT be listed when maxDepth reaches 0") + } +} + +// TestPreCreateDirTreeFn_ContextCancelled: context cancelled before processing → +// returns ctx.Err, makes zero or partial calls. +func TestPreCreateDirTreeFn_ContextCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // already cancelled + + objs := []model.Obj{dirObj("sub")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled, got: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("no MakeDir should be called after cancellation, got: %v", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_ContextCancelledDuringRecursion: context is cancelled +// during the second-pass recursion loop. +func TestPreCreateDirTreeFn_ContextCancelledDuringRecursion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + // Two dirs; cancel after the first List in recursion + callCount := 0 + listSrc := func(c context.Context, path string) ([]model.Obj, error) { + callCount++ + cancel() // cancel on first list call + return nil, nil + } + objs := []model.Obj{dirObj("sub1"), dirObj("sub2")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, listSrc) + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled after cancellation during recursion, got: %v", err) + } + if callCount > 1 { + t.Errorf("listSrc should have been called at most once before ctx.Err fired, got %d", callCount) + } +} + +// TestPreCreateDirTreeFn_MakeDirErrorNonFatal: a MakeDir failure on one dir must +// not stop processing of subsequent dirs. +func TestPreCreateDirTreeFn_MakeDirErrorNonFatal(t *testing.T) { + objs := []model.Obj{dirObj("subA"), dirObj("subB"), dirObj("subC")} + rec := newRecorder() + rec.mkdirErr["/dst/subA"] = errors.New("quota exceeded") + + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("error should not propagate from MakeDir failure: %v", err) + } + // All three must have been attempted despite the error on subA + for _, p := range []string{"/dst/subA", "/dst/subB", "/dst/subC"} { + if !rec.hasMkdir(p) { + t.Errorf("expected MakeDir(%s) to be called", p) + } + } +} + +// TestPreCreateDirTreeFn_ListErrorNonFatal: a List error for one subdir during +// recursion skips that subdir but continues with the rest. +func TestPreCreateDirTreeFn_ListErrorNonFatal(t *testing.T) { + objs := []model.Obj{dirObj("subA"), dirObj("subB")} + listCallCount := 0 + listSrc := func(_ context.Context, path string) ([]model.Obj, error) { + listCallCount++ + if path == "/src/subA" { + return nil, errors.New("I/O error") + } + return []model.Obj{dirObj("nested")}, nil + } + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, listSrc); err != nil { + t.Fatalf("List error must not be fatal: %v", err) + } + // subB's nested dir should still be processed despite subA's List failure + if !rec.hasMkdir("/dst/subB/nested") { + t.Error("expected /dst/subB/nested to be created despite subA list error, mkdirs:", rec.mkdirs) + } + if listCallCount != 2 { + t.Errorf("both subdirs must be attempted for listing, got %d calls", listCallCount) + } +} + +// TestPreCreateDirTreeFn_MixedObjs: mixed files and dirs; only dirs are processed. +func TestPreCreateDirTreeFn_MixedObjs(t *testing.T) { + objs := []model.Obj{ + fileObj("readme.md"), + dirObj("assets"), + fileObj("main.go"), + dirObj("pkg"), + } + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 2 { + t.Errorf("expected exactly 2 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + } + if !rec.hasMkdir("/dst/assets") || !rec.hasMkdir("/dst/pkg") { + t.Errorf("unexpected mkdirs: %v", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_Timeout: context with a very short deadline cancels execution. +func TestPreCreateDirTreeFn_Timeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + time.Sleep(5 * time.Millisecond) // ensure deadline has passed + + objs := []model.Obj{dirObj("sub")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("expected DeadlineExceeded, got: %v", err) + } +} From 409d01ac95b002667763c3d03172dd9cf50fda7e Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 7 Mar 2026 21:56:40 +0800 Subject: [PATCH 58/86] =?UTF-8?q?fix(google=5Fdrive):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E6=9C=BA=E5=88=B6=E4=BB=A5=E5=A4=84=E7=90=86?= =?UTF-8?q?=E5=B0=8F=E6=96=87=E4=BB=B6=E4=B8=8A=E4=BC=A0=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/google_drive/driver.go | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 61c182bfb..5860946e0 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -3,6 +3,7 @@ package google_drive import ( "context" "fmt" + "io" "net/http" "strconv" "strings" @@ -268,15 +269,26 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, file model.File putUrl := res.Header().Get("location") if file.GetSize() < d.ChunkSize*1024*1024 { // 小文件上传：使用 RangeRead 读取整个文件（避免消费已计算hash的stream） - reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) - if err != nil { - return err - } + err = retry.Do(func() error { + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } - _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { - req.SetHeader("Content-Length", strconv.FormatInt(file.GetSize(), 10)). - SetBody(driver.NewLimitedUploadStream(ctx, reader)) - }, nil) + _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { + req.SetHeader("Content-Length", strconv.FormatInt(file.GetSize(), 10)). + SetBody(driver.NewLimitedUploadStream(ctx, reader)) + }, nil) + return err + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) return err } else { // 大文件分片上传 From 34799fe35f94eb31245bed5de6eb4ff3baf5a8d2 Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 9 Mar 2026 17:07:21 +0800 Subject: [PATCH 59/86] fix(offline_download): cleanup 115 open completed tasks --- drivers/115_open/driver.go | 25 +- internal/offline_download/115_open/client.go | 500 ++++++++++++-- .../offline_download/115_open/client_test.go | 608 ++++++++++++++---- internal/offline_download/tool/download.go | 10 +- .../offline_download/tool/download_test.go | 90 +++ 5 files changed, 1045 insertions(+), 188 deletions(-) create mode 100644 internal/offline_download/tool/download_test.go diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 9033365a5..d09198031 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -2,6 +2,7 @@ package _115_open import ( "context" + "encoding/json" "fmt" "net/http" "strconv" @@ -206,7 +207,7 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) return nil, err } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ - FileID: srcObj.GetID(), + FileID: srcObj.GetID(), FileName: newName, }) if err != nil { @@ -415,6 +416,28 @@ func (d *Open115) OfflineDownload(ctx context.Context, uris []string, dstDir mod return d.client.AddOfflineTaskURIs(ctx, uris, dstDir.GetID()) } +func (d *Open115) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + var envelope sdk.Resp[[]sdk.AddOfflineTaskURIsResp] + response, err := d.client.AuthRequestRaw(ctx, sdk.ApiAddOffline, http.MethodPost, nil, sdk.ReqWithForm(sdk.Form{ + "urls": strings.Join(uris, "\n"), + "wp_path_id": dstDir.GetID(), + })) + if response != nil { + _ = json.Unmarshal(response.Bytes(), &envelope) + } + hashes := make([]string, 0, len(envelope.Data)) + for _, item := range envelope.Data { + if item.State && item.InfoHash != "" { + hashes = append(hashes, item.InfoHash) + } + } + rawResponse := "" + if response != nil { + rawResponse = response.String() + } + return hashes, envelope.Data, rawResponse, err +} + func (d *Open115) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { return d.client.DeleteOfflineTask(ctx, infoHash, deleteFiles) } diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index e8d33326b..69bad122d 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -2,9 +2,16 @@ package _115_open import ( "context" + "encoding/base32" + "encoding/hex" + "errors" "fmt" + "net/url" + "strconv" "strings" + "time" + sdk "github.com/OpenListTeam/115-sdk-go" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -13,11 +20,22 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" + log "github.com/sirupsen/logrus" ) type Open115 struct { } +type offlineTaskClient interface { + OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) + DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error +} + +type offlineTaskDetailClient interface { + OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) +} + func (o *Open115) Name() string { return "115 Open" } @@ -69,50 +87,12 @@ func (o *Open115) AddURL(args *tool.AddUrlArgs) (string, error) { if err != nil { return "", err } + log.Infof("[115_open] AddURL start: temp_dir=%q actual_path=%q parent_id=%q parent_name=%q url=%q", args.TempDir, actualPath, parentDir.GetID(), parentDir.GetName(), args.Url) + logOfflineURLDetails("[115_open] AddURL input", args.Url) - hashs, err := driver115Open.OfflineDownload(ctx, []string{args.Url}, parentDir) - - // 检查是否是重复链接错误 (code: 10008) + hashs, err := addOfflineDownloadTask(ctx, driver115Open, args.Url, parentDir) if err != nil { - // 尝试从错误信息中判断是否是重复链接 - errStr := err.Error() - isDuplicateError := false - - // 检查是否包含"重复"、"已存在"等关键词，或错误码10008 - if strings.Contains(errStr, "10008") || - strings.Contains(errStr, "重复") || - strings.Contains(errStr, "已存在") || - strings.Contains(errStr, "duplicate") { - isDuplicateError = true - } - - if isDuplicateError { - // 尝试查找并删除已存在的相同URL任务，然后重试 - taskList, listErr := driver115Open.OfflineList(ctx) - if listErr == nil && taskList != nil { - // 查找匹配的任务 - for _, task := range taskList.Tasks { - if task.URL == args.Url { - // 找到重复任务，删除它 - deleteErr := driver115Open.DeleteOfflineTask(ctx, task.InfoHash, false) - if deleteErr == nil { - // 删除成功，重新尝试添加 - hashs, err = driver115Open.OfflineDownload(ctx, []string{args.Url}, parentDir) - if err != nil { - return "", fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) - } - if len(hashs) > 0 { - return hashs[0], nil - } - } - break - } - } - } - } - - // 如果不是重复错误或处理失败，返回原始错误 - return "", fmt.Errorf("failed to add offline download task: %w", err) + return "", err } if len(hashs) < 1 { @@ -122,6 +102,436 @@ func (o *Open115) AddURL(args *tool.AddUrlArgs) (string, error) { return hashs[0], nil } +func addOfflineDownloadTask(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, error) { + parentID, parentName := "", "" + if parentDir != nil { + parentID = parentDir.GetID() + parentName = parentDir.GetName() + } + log.Infof("[115_open] addOfflineDownloadTask: parent_id=%q parent_name=%q url=%q", parentID, parentName, url) + logOfflineURLDetails("[115_open] addOfflineDownloadTask target", url) + if err := preCleanDuplicateOfflineTasks(ctx, client, url); err != nil { + return nil, err + } + hashs, addItems, rawResp, err := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] addOfflineDownloadTask first attempt result: hashes=%v err=%v add_items=%d", hashs, err, len(addItems)) + if err == nil { + return hashs, nil + } + if !isDuplicateOfflineTaskError(err) { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] duplicate offline task detected, trying cleanup before retry") + if rawResp != "" { + log.Infof("[115_open] duplicate add response: %s", rawResp) + } + for _, item := range addItems { + log.Infof("[115_open] duplicate add item: state=%v code=%d info_hash=%q url=%q", item.State, item.Code, item.InfoHash, item.URL) + logOfflineURLDetails("[115_open] duplicate add item url", item.URL) + if item.InfoHash == "" { + log.Infof("[115_open] skipping add-response duplicate item: empty info_hash") + continue + } + if item.URL != "" && !offlineTaskURLMatches(item.URL, url) { + log.Infof("[115_open] skipping add-response duplicate item: url mismatch") + continue + } + log.Infof("[115_open] deleting duplicate task directly from add response: info_hash=%s url=%s", item.InfoHash, item.URL) + if deleteErr := client.DeleteOfflineTask(ctx, item.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete duplicate task from add response failed: info_hash=%s err=%v", item.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task from add response: %w", deleteErr) + } + log.Infof("[115_open] delete duplicate task from add response success: info_hash=%s", item.InfoHash) + waitForOfflineTaskRemoval(ctx, client, item.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after add-response delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after add-response delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] matched duplicate offline task: info_hash=%s, name=%s", task.InfoHash, task.Name) + log.Infof("[115_open] deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + log.Infof("[115_open] delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after matched delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after matched delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + log.Warnf("[115_open] duplicate offline task detected but no matching task found in offline list") + return nil, fmt.Errorf("failed to add offline download task: %w", err) +} + +func preCleanDuplicateOfflineTasks(ctx context.Context, client offlineTaskClient, url string) error { + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + log.Warnf("[115_open] pre-add offline list failed: err=%v", listErr) + return nil + } + log.Infof("[115_open] pre-add offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + deleted := 0 + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] pre-add duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] pre-add duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] pre-add deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] pre-add delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + deleted++ + log.Infof("[115_open] pre-add delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + } + if deleted == 0 { + log.Infof("[115_open] pre-add duplicate scan found no matches") + } + return nil +} + +func offlineDownloadWithDetails(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if detailClient, ok := client.(offlineTaskDetailClient); ok { + return detailClient.OfflineDownloadWithDetails(ctx, []string{url}, parentDir) + } + hashs, err := client.OfflineDownload(ctx, []string{url}, parentDir) + return hashs, nil, "", err +} + +func isDuplicateOfflineTaskError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + return strings.Contains(errStr, "10008") || + strings.Contains(errStr, "重复") || + strings.Contains(errStr, "已存在") || + strings.Contains(errStr, "duplicate") +} + +func offlineTaskURLMatches(taskURL string, rawURL string) bool { + taskVariants := normalizedOfflineTaskURLVariants(taskURL) + rawVariants := normalizedOfflineTaskURLVariants(rawURL) + for candidate := range taskVariants { + if _, ok := rawVariants[candidate]; ok { + return true + } + } + return false +} + +func offlineTaskMatches(task sdk.OfflineTask, rawURL string) bool { + matched, _ := offlineTaskMatchReason(task, rawURL) + return matched +} + +func offlineTaskMatchReason(task sdk.OfflineTask, rawURL string) (bool, string) { + if offlineTaskURLMatches(task.URL, rawURL) { + return true, "url variants matched" + } + if httpURLMatches(task.URL, rawURL) { + return true, "http url host+path matched" + } + rawMagnet := parseMagnetBTIH(rawURL) + if rawMagnet != "" { + taskHash := normalizeInfoHash(task.InfoHash) + if taskHash != "" && taskHash == rawMagnet { + return true, "task info_hash matched raw magnet" + } + taskURLHash := parseMagnetBTIH(task.URL) + if taskURLHash != "" && taskURLHash == rawMagnet { + return true, "task url magnet hash matched" + } + return false, fmt.Sprintf("task magnet hash mismatch: task_info_hash=%q task_url_hash=%q raw_hash=%q", taskHash, taskURLHash, rawMagnet) + } + taskED2K, rawED2K := parseED2KLink(task.URL), parseED2KLink(rawURL) + if taskED2K != nil && rawED2K != nil { + if taskED2K.Hash == rawED2K.Hash { + if taskED2K.Size == rawED2K.Size { + return true, "task url ed2k hash matched" + } + return true, "task url ed2k hash matched despite size mismatch" + } + return false, fmt.Sprintf("task url ed2k mismatch: task=%s raw=%s", taskED2K.String(), rawED2K.String()) + } + if rawED2K == nil { + return false, "raw url is not ed2k and url variants did not match" + } + if normalizeOfflineTaskURL(task.InfoHash) == rawED2K.Hash { + if task.Size == rawED2K.Size { + return true, "task info_hash matched raw ed2k" + } + return true, "task info_hash matched raw ed2k despite size mismatch" + } + taskName := normalizeOfflineTaskURL(task.Name) + if taskName == normalizeOfflineTaskURL(rawED2K.Name) && task.Size == rawED2K.Size { + return true, "task name and size matched raw ed2k" + } + return false, fmt.Sprintf("task name/hash/size mismatch: task_name=%q raw_name=%q task_info_hash=%q raw_hash=%q task_size=%d raw_size=%d", taskName, normalizeOfflineTaskURL(rawED2K.Name), normalizeOfflineTaskURL(task.InfoHash), rawED2K.Hash, task.Size, rawED2K.Size) +} + +func normalizedOfflineTaskURLVariants(raw string) map[string]struct{} { + variants := map[string]struct{}{} + queue := []string{raw} + for len(queue) > 0 { + current := normalizeOfflineTaskURL(queue[0]) + queue = queue[1:] + if current == "" { + continue + } + if _, ok := variants[current]; ok { + continue + } + variants[current] = struct{}{} + if decoded, err := url.QueryUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + if decoded, err := url.PathUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + } + return variants +} + +func httpURLMatches(taskURL, rawURL string) bool { + taskNormalized := normalizeHTTPURL(taskURL) + rawNormalized := normalizeHTTPURL(rawURL) + if taskNormalized == "" || rawNormalized == "" { + return false + } + return taskNormalized == rawNormalized +} + +func normalizeHTTPURL(raw string) string { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed == nil { + return "" + } + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return "" + } + host := strings.ToLower(parsed.Host) + if host == "" { + return "" + } + path := strings.ToLower(parsed.Path) + path = strings.TrimSuffix(path, "/") + if path == "" { + path = "/" + } + return fmt.Sprintf("%s://%s%s", scheme, host, path) +} + +func normalizeOfflineTaskURL(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimSuffix(normalized, "/") + return strings.ToLower(normalized) +} + +type ed2kLink struct { + Name string + Size int64 + Hash string +} + +func parseED2KLink(raw string) *ed2kLink { + normalized := strings.TrimSpace(raw) + if !strings.HasPrefix(strings.ToLower(normalized), "ed2k://|file|") { + return nil + } + parts := strings.Split(normalized, "|") + if len(parts) < 6 { + return nil + } + name, err := url.PathUnescape(parts[2]) + if err != nil { + name = parts[2] + } + size, err := strconv.ParseInt(parts[3], 10, 64) + if err != nil { + return nil + } + return &ed2kLink{ + Name: normalizeOfflineTaskURL(name), + Size: size, + Hash: normalizeOfflineTaskURL(parts[4]), + } +} + +func parseMagnetBTIH(raw string) string { + if raw == "" { + return "" + } + lower := strings.ToLower(raw) + idx := strings.Index(lower, "btih:") + if idx == -1 { + return "" + } + candidate := raw[idx+len("btih:"):] + if candidate == "" { + return "" + } + for i, ch := range candidate { + if ch == '&' || ch == '#' || ch == '/' { + candidate = candidate[:i] + break + } + } + candidate = strings.TrimSpace(candidate) + if candidate == "" { + return "" + } + if decoded, err := url.QueryUnescape(candidate); err == nil { + candidate = decoded + } + return normalizeInfoHash(candidate) +} + +func normalizeInfoHash(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimPrefix(strings.ToLower(normalized), "urn:btih:") + if normalized == "" { + return "" + } + if len(normalized) == 40 && isHexString(normalized) { + return normalized + } + if len(normalized) == 32 && isBase32String(normalized) { + decoded, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(normalized)) + if err == nil && len(decoded) == 20 { + return hex.EncodeToString(decoded) + } + } + return normalized +} + +func isHexString(value string) bool { + for _, ch := range value { + if (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') { + continue + } + return false + } + return true +} + +func isBase32String(value string) bool { + for _, ch := range value { + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '2' && ch <= '7') { + continue + } + return false + } + return true +} + +func (e *ed2kLink) Equal(other *ed2kLink) bool { + if e == nil || other == nil { + return false + } + return e.Name == other.Name && e.Size == other.Size && e.Hash == other.Hash +} + +func (e *ed2kLink) String() string { + if e == nil { + return "" + } + return fmt.Sprintf("name=%q size=%d hash=%q", e.Name, e.Size, e.Hash) +} + +func logOfflineURLDetails(prefix string, raw string) { + if raw == "" { + log.Infof("%s details: raw is empty", prefix) + return + } + variants := normalizedOfflineTaskURLVariants(raw) + log.Infof("%s details: raw=%q normalized_variants=%v", prefix, raw, mapKeys(variants)) + if parsedMagnet := parseMagnetBTIH(raw); parsedMagnet != "" { + log.Infof("%s details: parsed_magnet_hash=%s", prefix, parsedMagnet) + } + if parsed := parseED2KLink(raw); parsed != nil { + log.Infof("%s details: parsed_ed2k=%s", prefix, parsed.String()) + } +} + +func mapKeys(values map[string]struct{}) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + return keys +} + +func waitForOfflineTaskRemoval(ctx context.Context, client offlineTaskClient, infoHash string) { + const maxChecks = 3 + for attempt := 1; attempt <= maxChecks; attempt++ { + taskList, err := client.OfflineList(ctx) + if err != nil { + log.Warnf("[115_open] post-delete check failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } + stillExists := false + taskStatus := -999 + taskName := "" + for _, task := range taskList.Tasks { + if normalizeOfflineTaskURL(task.InfoHash) != normalizeOfflineTaskURL(infoHash) { + continue + } + stillExists = true + taskStatus = task.Status + taskName = task.Name + break + } + log.Infof("[115_open] post-delete check: info_hash=%s attempt=%d exists=%v status=%d name=%q task_count=%d", infoHash, attempt, stillExists, taskStatus, taskName, len(taskList.Tasks)) + if !stillExists { + return + } + if attempt < maxChecks { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } + } +} + func (o *Open115) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { @@ -169,13 +579,15 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = errors.New(t.GetStatus()) } return s, nil } } // 任务不在列表中，可能已完成或被删除 - return nil, fmt.Errorf("task %s not found in offline list", task.GID) + s.Progress = 100 + s.Completed = true + return s, nil } var _ tool.Tool = (*Open115)(nil) diff --git a/internal/offline_download/115_open/client_test.go b/internal/offline_download/115_open/client_test.go index b675e65fe..29305a57e 100644 --- a/internal/offline_download/115_open/client_test.go +++ b/internal/offline_download/115_open/client_test.go @@ -7,242 +7,576 @@ import ( "testing" sdk "github.com/OpenListTeam/115-sdk-go" - _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" ) -// Mock implementation of Open115 driver for testing -type mockOpen115 struct { - _115_open.Open115 - offlineDownloadFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) - offlineListFunc func(ctx context.Context) (*sdk.OfflineTaskListResp, error) - deleteOfflineFunc func(ctx context.Context, infoHash string, deleteFiles bool) error +type mockOfflineTaskClient struct { + offlineDownloadFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + offlineDownloadWithDetailsFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) + offlineListFunc func(ctx context.Context) (*sdk.OfflineTaskListResp, error) + deleteOfflineFunc func(ctx context.Context, infoHash string, deleteFiles bool) error } -func (m *mockOpen115) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { - if m.offlineDownloadFunc != nil { - return m.offlineDownloadFunc(ctx, uris, dstDir) - } - return nil, fmt.Errorf("not implemented") +func (m *mockOfflineTaskClient) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return m.offlineDownloadFunc(ctx, uris, dstDir) } -func (m *mockOpen115) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { - if m.offlineListFunc != nil { - return m.offlineListFunc(ctx) +func (m *mockOfflineTaskClient) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if m.offlineDownloadWithDetailsFunc == nil { + hashes, err := m.OfflineDownload(ctx, uris, dstDir) + return hashes, nil, "", err } - return nil, fmt.Errorf("not implemented") + return m.offlineDownloadWithDetailsFunc(ctx, uris, dstDir) +} + +func (m *mockOfflineTaskClient) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return m.offlineListFunc(ctx) } -func (m *mockOpen115) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { - if m.deleteOfflineFunc != nil { - return m.deleteOfflineFunc(ctx, infoHash, deleteFiles) +func (m *mockOfflineTaskClient) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { + return m.deleteOfflineFunc(ctx, infoHash, deleteFiles) +} + +func TestIsDuplicateOfflineTaskError(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "code 10008", err: fmt.Errorf("code: 10008"), want: true}, + {name: "chinese duplicate", err: fmt.Errorf("任务重复"), want: true}, + {name: "already exists", err: fmt.Errorf("任务已存在"), want: true}, + {name: "english duplicate", err: fmt.Errorf("duplicate task"), want: true}, + {name: "other", err: fmt.Errorf("network timeout"), want: false}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := isDuplicateOfflineTaskError(tc.err); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) } - return fmt.Errorf("not implemented") } -// TestAddURL_Success tests successful URL addition -func TestAddURL_Success(t *testing.T) { - t.Skip("需要真实的storage环境，跳过此测试") +func TestOfflineTaskURLMatches(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + taskURL string + rawURL string + want bool + }{ + { + name: "exact match", + taskURL: "ed2k://|file|test.avi|123|ABC|/", + rawURL: "ed2k://|file|test.avi|123|ABC|/", + want: true, + }, + { + name: "percent encoded file name", + taskURL: "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + rawURL: "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + want: true, + }, + { + name: "case and trailing slash normalized", + taskURL: "ED2K://|FILE|TEST.AVI|123|ABC|", + rawURL: "ed2k://|file|test.avi|123|abc|/", + want: true, + }, + { + name: "different link", + taskURL: "ed2k://|file|a.avi|123|ABC|/", + rawURL: "ed2k://|file|b.avi|123|ABC|/", + want: false, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := offlineTaskURLMatches(tc.taskURL, tc.rawURL); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } } -// TestAddURL_DuplicateHandling tests the duplicate URL handling logic -func TestAddURL_DuplicateHandling(t *testing.T) { - t.Skip("需要真实的storage环境，跳过此测试") +func TestOfflineTaskMatches(t *testing.T) { + t.Parallel() + + t.Run("match ed2k by parsed fields when task url differs", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + InfoHash: "server-task-hash", + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected task to match by ed2k parsed fields") + } + }) + + t.Run("do not match different ed2k size", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1, + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if offlineTaskMatches(task, rawURL) { + t.Fatal("expected task not to match") + } + }) + + t.Run("magnet still matches by url", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: rawURL, + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by url") + } + }) + + t.Run("match magnet by btih despite noisy tracker", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test&tr=%3C!DOCTYPE%20html%3E", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by btih") + } + }) + + t.Run("match http by host and path", func(t *testing.T) { + t.Parallel() + + rawURL := "https://example.com/files/test.mp4" + task := sdk.OfflineTask{ + URL: "https://EXAMPLE.com/files/test.mp4?token=abc", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected http task to match by host and path") + } + }) } -// TestDuplicateLinkRetryLogic tests the logic without actual API calls -func TestDuplicateLinkRetryLogic(t *testing.T) { - testURL := "https://example.com/test.torrent" - testHash := "test_hash_123" +func TestAddOfflineDownloadTask(t *testing.T) { + t.Parallel() + + const ( + testURL = "https://example.com/test.torrent" + firstHash = "hash-1" + staleHash = "hash-stale" + deleteError = "delete failed" + ) + + t.Run("success on first try", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry", func(t *testing.T) { + t.Parallel() - t.Run("首次添加成功", func(t *testing.T) { - // 模拟首次添加成功的场景 callCount := 0 - mock := &mockOpen115{ + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { callCount++ if callCount == 1 { - return []string{testHash}, nil + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: testURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") } - return nil, fmt.Errorf("unexpected call") + return nil }, } - hashes, err := mock.OfflineDownload(context.Background(), []string{testURL}, nil) + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) if err != nil { - t.Errorf("首次添加失败: %v", err) + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) } - if len(hashes) != 1 || hashes[0] != testHash { - t.Errorf("期望hash=%s, 实际=%v", testHash, hashes) + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) } - if callCount != 1 { - t.Errorf("期望调用1次, 实际调用%d次", callCount) + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) } }) - t.Run("检测到重复错误并自动删除重试", func(t *testing.T) { - // 模拟重复链接错误的场景 + t.Run("delete duplicate magnet and retry", func(t *testing.T) { + t.Parallel() + callCount := 0 deleteCount := 0 - - mock := &mockOpen115{ + listCount := 0 + magnetURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + client := &mockOfflineTaskClient{ offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { callCount++ if callCount == 1 { - // 首次调用返回重复错误 - return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") - } else if callCount == 2 { - // 删除后重试，返回成功 - return []string{testHash}, nil + return nil, fmt.Errorf("code: 10008, message: 任务已存在") } - return nil, fmt.Errorf("unexpected call count: %d", callCount) + return []string{firstHash}, nil }, offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { - // 返回包含重复任务的列表 + listCount++ return &sdk.OfflineTaskListResp{ Tasks: []sdk.OfflineTask{ - { - InfoHash: "old_hash_456", - URL: testURL, - Status: 1, // 下载中 - }, + {InfoHash: staleHash, URL: magnetURL}, }, }, nil }, deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { deleteCount++ - if infoHash != "old_hash_456" { - t.Errorf("期望删除hash=old_hash_456, 实际=%s", infoHash) + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) } if deleteFiles { - t.Error("不应该删除源文件") + t.Fatal("deleteFiles should be false") } return nil }, } - // 模拟完整的错误处理逻辑 - ctx := context.Background() - - // 第一次调用返回重复错误 - _, err := mock.OfflineDownload(ctx, []string{testURL}, nil) - if err == nil { - t.Error("第一次应该返回错误") + hashes, err := addOfflineDownloadTask(context.Background(), client, magnetURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) - // 检查是否是重复错误 - errStr := err.Error() - if !strings.Contains(errStr, "10008") && !strings.Contains(errStr, "重复") { - t.Errorf("应该是重复错误，实际错误: %v", err) + t.Run("delete duplicate and retry with decoded ed2k url", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + decodedURL := "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: decodedURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, } - // 获取任务列表 - taskList, err := mock.OfflineList(ctx) + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) if err != nil { - t.Errorf("获取任务列表失败: %v", err) + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry with empty task url but matching name and size", func(t *testing.T) { + t.Parallel() - // 查找并删除重复任务 - found := false - for _, task := range taskList.Tasks { - if task.URL == testURL { - err := mock.DeleteOfflineTask(ctx, task.InfoHash, false) - if err != nil { - t.Errorf("删除任务失败: %v", err) + callCount := 0 + deleteCount := 0 + listCount := 0 + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") } - found = true - break - } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + { + InfoHash: staleHash, + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + }, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, } - if !found { - t.Error("未找到重复任务") + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate directly from add response info hash", func(t *testing.T) { + t.Parallel() - // 重试添加 - hashes, err := mock.OfflineDownload(ctx, []string{testURL}, nil) + callCount := 0 + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadWithDetailsFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + callCount++ + if callCount == 1 { + return nil, []sdk.AddOfflineTaskURIsResp{ + {InfoHash: staleHash, URL: testURL}, + }, `{"state":false,"code":10008,"message":"任务已存在","data":[{"info_hash":"hash-stale","url":"` + testURL + `"}]}`, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil, "", nil + }, + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("unexpected fallback OfflineDownload call") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) if err != nil { - t.Errorf("重试添加失败: %v", err) + t.Fatalf("unexpected error: %v", err) } - if len(hashes) != 1 || hashes[0] != testHash { - t.Errorf("期望hash=%s, 实际=%v", testHash, hashes) + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) } - if callCount != 2 { - t.Errorf("期望调用OfflineDownload 2次, 实际%d次", callCount) + t.Fatalf("want 2 download attempts, got %d", callCount) } if deleteCount != 1 { - t.Errorf("期望调用DeleteOfflineTask 1次, 实际%d次", deleteCount) + t.Fatalf("want 1 delete attempt, got %d", deleteCount) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) } }) - t.Run("重复链接但删除失败", func(t *testing.T) { - mock := &mockOpen115{ + t.Run("duplicate delete failure", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { - // 始终返回重复错误 - return nil, fmt.Errorf("code: 10008, message: 任务已存在") + return nil, fmt.Errorf("duplicate task") }, offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ return &sdk.OfflineTaskListResp{ Tasks: []sdk.OfflineTask{ - { - InfoHash: "old_hash_789", - URL: testURL, - }, + {InfoHash: staleHash, URL: testURL}, }, }, nil }, deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { - return fmt.Errorf("删除失败：权限不足") + return fmt.Errorf(deleteError) }, } - // 删除失败时应该返回错误 - ctx := context.Background() - _, err := mock.OfflineDownload(ctx, []string{testURL}, nil) + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) if err == nil { - t.Error("应该返回错误") + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), deleteError) { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) } }) -} -// TestOpen115_Name tests the Name method -func TestOpen115_Name(t *testing.T) { - o := &Open115{} - name := o.Name() - expected := "115 Open" - if name != expected { - t.Errorf("期望名称=%s, 实际=%s", expected, name) - } -} + t.Run("non duplicate error is returned", func(t *testing.T) { + t.Parallel() -// TestOpen115_Items tests the Items method -func TestOpen115_Items(t *testing.T) { - o := &Open115{} - items := o.Items() - if items != nil { - t.Error("Items应该返回nil") - } -} + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("network timeout") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } -// TestOpen115_Run tests the Run method -func TestOpen115_Run(t *testing.T) { - o := &Open115{} - err := o.Run(&tool.DownloadTask{}) - if err == nil { - t.Error("Run应该返回NotSupport错误") - } + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "network timeout") { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) } -// TestOpen115_Init tests the Init method -func TestOpen115_Init(t *testing.T) { +func TestOpen115BasicMethods(t *testing.T) { + t.Parallel() + o := &Open115{} + + if o.Name() != "115 Open" { + t.Fatalf("unexpected name: %s", o.Name()) + } + if o.Items() != nil { + t.Fatal("Items should return nil") + } msg, err := o.Init() if err != nil { - t.Errorf("Init失败: %v", err) + t.Fatalf("unexpected init error: %v", err) } if msg != "ok" { - t.Errorf("期望消息='ok', 实际=%s", msg) + t.Fatalf("unexpected init message: %s", msg) } } diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index e033cccba..33d91176d 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -32,6 +32,8 @@ type DownloadTask struct { callStatusRetried int } +var completedOfflineTaskCleanupDelay = time.Second + func (t *DownloadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) @@ -97,18 +99,14 @@ outer: if t.tool.Name() == "ThunderX" { return nil } - if t.tool.Name() == "115 Cloud" { - // hack for 115 - <-time.After(time.Second * 1) + if t.tool.Name() == "115 Cloud" || t.tool.Name() == "115 Open" { + <-time.After(completedOfflineTaskCleanupDelay) err := t.tool.Remove(t) if err != nil { log.Errorln(err.Error()) } return nil } - if t.tool.Name() == "115 Open" { - return nil - } if t.tool.Name() == "123 Open" { return nil } diff --git a/internal/offline_download/tool/download_test.go b/internal/offline_download/tool/download_test.go new file mode 100644 index 000000000..5303d35aa --- /dev/null +++ b/internal/offline_download/tool/download_test.go @@ -0,0 +1,90 @@ +package tool + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + task2 "github.com/OpenListTeam/OpenList/v4/internal/task" +) + +type mockTool struct { + name string + addURLFunc func(args *AddUrlArgs) (string, error) + removeFunc func(task *DownloadTask) error + statusFunc func(task *DownloadTask) (*Status, error) + runFunc func(task *DownloadTask) error +} + +func (m *mockTool) Name() string { return m.name } + +func (m *mockTool) Items() []model.SettingItem { return nil } + +func (m *mockTool) Init() (string, error) { return "ok", nil } + +func (m *mockTool) IsReady() bool { return true } + +func (m *mockTool) AddURL(args *AddUrlArgs) (string, error) { + return m.addURLFunc(args) +} + +func (m *mockTool) Remove(task *DownloadTask) error { + return m.removeFunc(task) +} + +func (m *mockTool) Status(task *DownloadTask) (*Status, error) { + return m.statusFunc(task) +} + +func (m *mockTool) Run(task *DownloadTask) error { + return m.runFunc(task) +} + +func TestDownloadTaskRun_RemovesCompleted115OpenRecord(t *testing.T) { + previousDelay := completedOfflineTaskCleanupDelay + completedOfflineTaskCleanupDelay = 0 + defer func() { + completedOfflineTaskCleanupDelay = previousDelay + }() + + removeCount := 0 + tool := &mockTool{ + name: "115 Open", + addURLFunc: func(args *AddUrlArgs) (string, error) { + return "gid-1", nil + }, + removeFunc: func(task *DownloadTask) error { + removeCount++ + if task.GID != "gid-1" { + t.Fatalf("unexpected gid: %s", task.GID) + } + return nil + }, + statusFunc: func(task *DownloadTask) (*Status, error) { + return &Status{ + Completed: true, + Status: "completed", + }, nil + }, + runFunc: func(task *DownloadTask) error { + return errs.NotSupport + }, + } + + task := &DownloadTask{ + TaskExtension: task2.TaskExtension{}, + Url: "https://example.com/test.torrent", + DstDirPath: "/115", + TempDir: "/115", + tool: tool, + } + task.SetCtx(context.Background()) + + if err := task.Run(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if removeCount != 1 { + t.Fatalf("want 1 cleanup remove, got %d", removeCount) + } +} From 1f5221fd773c07499bc2943bdb1535976dd2ce03 Mon Sep 17 00:00:00 2001 From: cyk Date: Thu, 12 Mar 2026 19:54:06 +0800 Subject: [PATCH 60/86] Refactor code structure for improved readability and maintainability --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2fc141f2e..d497a5fcd 100644 --- a/go.mod +++ b/go.mod @@ -313,4 +313,4 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -// replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.4 diff --git a/go.sum b/go.sum index 0f69ce117..dcb62bc93 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= +github.com/Ironboxplus/115-sdk-go v0.2.4 h1:QUa55UmuNQr+TUZI+EfYPiTkUO8VE19PRWTIxBcEpfE= +github.com/Ironboxplus/115-sdk-go v0.2.4/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= @@ -29,8 +31,6 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/OpenListTeam/115-sdk-go v0.2.3 h1:nDNz0GxgliW+nT2Ds486k/rp/GgJj7Ngznc98ZBUwZo= -github.com/OpenListTeam/115-sdk-go v0.2.3/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= From 4c5ee9665bfeca59a42d85fee5ebb85c134ec1ab Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 17 Mar 2026 13:12:31 +0800 Subject: [PATCH 61/86] =?UTF-8?q?feat(stream):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E8=87=AA=E6=84=88=E8=AF=BB=E5=8F=96=E5=99=A8=E5=92=8C=E5=8D=95?= =?UTF-8?q?=E5=85=83=E6=B5=8B=E8=AF=95=E4=BB=A5=E5=A4=84=E7=90=86=E4=B8=AD?= =?UTF-8?q?=E6=96=AD=E7=9A=84=E6=B5=81=E9=87=8D=E8=BF=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/stream/util.go | 115 ++++++++++++++++++------- internal/stream/util_test.go | 160 +++++++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 32 deletions(-) create mode 100644 internal/stream/util_test.go diff --git a/internal/stream/util.go b/internal/stream/util.go index d54776399..e98341ede 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -182,13 +182,14 @@ func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { } // selfHealingReadCloser wraps an io.ReadCloser and automatically refreshes the link -// if it detects 0-byte reads (common with expired links from some cloud providers) +// if the upstream reader dies before the requested range is fully delivered. type selfHealingReadCloser struct { io.ReadCloser refresher *RefreshableRangeReader ctx context.Context httpRange http_range.Range firstRead bool + bytesRead int64 closed bool mu sync.Mutex } @@ -202,50 +203,100 @@ func (s *selfHealingReadCloser) Read(p []byte) (n int, err error) { } n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + wasFirstRead := !s.firstRead + s.firstRead = true // Detect 0-byte read on first attempt (indicates link may be expired but returned 200 OK) - if !s.firstRead && n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { - s.firstRead = true + if s.shouldReconnectAfterRead(wasFirstRead, n, err) { + if reconnectErr := s.reconnectFromCurrentOffsetLocked(); reconnectErr != nil { + log.Errorf("Failed to refresh link after interrupted read: %v", reconnectErr) + return n, err + } + + if n > 0 { + return n, nil + } + + n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + return n, err + } + + return n, err +} + +func (s *selfHealingReadCloser) shouldReconnectAfterRead(wasFirstRead bool, n int, err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + if s.remainingBytes() <= 0 { + return false + } + + if wasFirstRead && n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { log.Warnf("Detected 0-byte read on first attempt, attempting to refresh link...") + return true + } - // Try to refresh the link - s.refresher.mu.Lock() - refreshErr := s.refresher.doRefreshLocked(s.ctx) - s.refresher.mu.Unlock() + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + log.Warnf("Detected interrupted read after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } - if refreshErr != nil { - log.Errorf("Failed to refresh link after 0-byte read: %v", refreshErr) - return n, err - } + errStr := strings.ToLower(err.Error()) + if strings.Contains(errStr, "connection reset by peer") { + log.Warnf("Detected upstream connection reset after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } - // Close old connection - s.ReadCloser.Close() + return false +} + +func (s *selfHealingReadCloser) reconnectFromCurrentOffsetLocked() error { + nextRange := s.httpRange + nextRange.Start += s.bytesRead + if nextRange.Length >= 0 { + nextRange.Length -= s.bytesRead + } - // Get new reader and retry - s.refresher.mu.Lock() - reader, getErr := s.refresher.getInnerReader() + s.refresher.mu.Lock() + refreshErr := s.refresher.doRefreshLocked(s.ctx) + if refreshErr != nil { s.refresher.mu.Unlock() + return refreshErr + } - if getErr != nil { - log.Errorf("Failed to get inner reader after refresh: %v", getErr) - return n, err - } + reader, getErr := s.refresher.getInnerReader() + s.refresher.mu.Unlock() + if getErr != nil { + return getErr + } - newRc, rangeErr := reader.RangeRead(s.ctx, s.httpRange) - if rangeErr != nil { - log.Errorf("Failed to create new range reader after refresh: %v", rangeErr) - return n, err - } + newRc, rangeErr := reader.RangeRead(s.ctx, nextRange) + if rangeErr != nil { + return rangeErr + } - s.ReadCloser = newRc - log.Infof("Successfully refreshed link and reconnected after 0-byte read") + _ = s.ReadCloser.Close() + s.ReadCloser = newRc + log.Infof("Successfully refreshed link and reconnected from offset %d", nextRange.Start) + return nil +} - // Retry read with new connection - return s.ReadCloser.Read(p) +func (s *selfHealingReadCloser) remainingBytes() int64 { + length := s.httpRange.Length + if length < 0 || s.httpRange.Start+length > s.refresher.size { + length = s.refresher.size - s.httpRange.Start } - - s.firstRead = true - return n, err + remaining := length - s.bytesRead + if remaining < 0 { + return 0 + } + return remaining } func (s *selfHealingReadCloser) Close() error { diff --git a/internal/stream/util_test.go b/internal/stream/util_test.go new file mode 100644 index 000000000..6dc4c8a09 --- /dev/null +++ b/internal/stream/util_test.go @@ -0,0 +1,160 @@ +package stream + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: int64(len(data))}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != int64(len(data)-5) { + t.Fatalf("resumed range length = %d, want %d", resumedRanges[0].Length, len(data)-5) + } +} + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset_UnboundedRange(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: -1}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != -1 { + t.Fatalf("resumed range length = %d, want -1", resumedRanges[0].Length) + } +} + +type flakyReadCloser struct { + data []byte + failAfter int + failErr error + failed bool +} + +func newFlakyReadCloser(data []byte, failAfter int, failErr error) *flakyReadCloser { + return &flakyReadCloser{ + data: data, + failAfter: failAfter, + failErr: failErr, + } +} + +func (f *flakyReadCloser) Read(p []byte) (int, error) { + if f.failed { + return 0, io.EOF + } + if f.failAfter >= len(f.data) { + f.failed = true + n := copy(p, f.data) + return n, io.EOF + } + + n := copy(p, f.data[:f.failAfter]) + f.failed = true + return n, f.failErr +} + +func (f *flakyReadCloser) Close() error { + return nil +} + +func sliceForRange(data []byte, httpRange http_range.Range) []byte { + start := int(httpRange.Start) + length := int(httpRange.Length) + if httpRange.Length < 0 || httpRange.Start+httpRange.Length > int64(len(data)) { + length = len(data) - start + } + return data[start : start+length] +} From 398779f4f02a52617bebc787c587599fd2e70f2c Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 17 Mar 2026 14:24:29 +0800 Subject: [PATCH 62/86] =?UTF-8?q?fix:=20=E6=9B=B4=E6=96=B0=20WebVersion=20?= =?UTF-8?q?=E4=B8=BA=20latest=EF=BC=8C=E5=B9=B6=E5=9C=A8=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=E8=84=9A=E6=9C=AC=E4=B8=AD=E6=B7=BB=E5=8A=A0=E5=AF=B9=20WEB=5F?= =?UTF-8?q?VERSION=20=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F=E7=9A=84?= =?UTF-8?q?=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/beta_release.yml | 4 +--- .github/workflows/build.yml | 5 ++++- .github/workflows/test_docker.yml | 1 + build.sh | 10 +++++++++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 97312f52e..98615b0e0 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -128,9 +128,7 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag - github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling - env: - GOFLAGS: ${{ matrix.goflags }} + github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=latest - name: Compress run: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2393b84b..d89f7efff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: build: strategy: @@ -53,7 +56,7 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag - github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=latest output: openlist$ext - name: Upload artifact diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index d3f4a8fff..7542dcf11 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -87,6 +87,7 @@ jobs: run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WEB_VERSION: latest # FRONTEND_REPO 使用 build.sh 默认值 (OpenListTeam/OpenList-Frontend) - name: Upload artifacts diff --git a/build.sh b/build.sh index c26d7c557..afbe44ae1 100644 --- a/build.sh +++ b/build.sh @@ -31,6 +31,10 @@ else webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/$frontendRepo/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi +if [ -n "$WEB_VERSION" ]; then + webVersion="$WEB_VERSION" +fi + echo "backend version: $version" echo "frontend version: $webVersion" if [ "$useLite" = true ]; then @@ -615,7 +619,11 @@ if [ "$buildType" = "dev" ]; then fi elif [ "$buildType" = "release" -o "$buildType" = "beta" ]; then if [ "$buildType" = "beta" ]; then - FetchWebRolling + if [ "$WEB_VERSION" = "latest" ]; then + FetchWebRelease + else + FetchWebRolling + fi else FetchWebRelease fi From ed2129c2d29aa60b6c55547f21c55b7bd9d81cd4 Mon Sep 17 00:00:00 2001 From: cyk Date: Tue, 17 Mar 2026 14:41:42 +0800 Subject: [PATCH 63/86] =?UTF-8?q?feat(offline=5Fdownload):=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E4=BB=BB=E5=8A=A1=E9=99=90=E5=88=B6=E7=AD=89=E5=BE=85?= =?UTF-8?q?=E6=9C=BA=E5=88=B6=E4=BB=A5=E4=BC=98=E5=8C=96=E7=A6=BB=E7=BA=BF?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E4=BB=BB=E5=8A=A1=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/offline_download/115_open/client.go | 34 +++++++++++++++++++ .../offline_download/115_open/client_test.go | 34 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index 69bad122d..56669a674 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -36,6 +36,18 @@ type offlineTaskDetailClient interface { OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) } +type offlineTaskLimiter interface { + WaitLimit(ctx context.Context) error +} + +func waitOfflineTaskLimit(ctx context.Context, client offlineTaskClient) error { + limiter, ok := client.(offlineTaskLimiter) + if !ok { + return nil + } + return limiter.WaitLimit(ctx) +} + func (o *Open115) Name() string { return "115 Open" } @@ -137,6 +149,9 @@ func addOfflineDownloadTask(ctx context.Context, client offlineTaskClient, url s continue } log.Infof("[115_open] deleting duplicate task directly from add response: info_hash=%s url=%s", item.InfoHash, item.URL) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } if deleteErr := client.DeleteOfflineTask(ctx, item.InfoHash, false); deleteErr != nil { log.Errorf("[115_open] delete duplicate task from add response failed: info_hash=%s err=%v", item.InfoHash, deleteErr) return nil, fmt.Errorf("failed to delete duplicate offline download task from add response: %w", deleteErr) @@ -154,6 +169,9 @@ func addOfflineDownloadTask(ctx context.Context, client offlineTaskClient, url s } return hashs, nil } + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } taskList, listErr := client.OfflineList(ctx) if listErr != nil || taskList == nil { return nil, fmt.Errorf("failed to add offline download task: %w", err) @@ -168,6 +186,9 @@ func addOfflineDownloadTask(ctx context.Context, client offlineTaskClient, url s } log.Infof("[115_open] matched duplicate offline task: info_hash=%s, name=%s", task.InfoHash, task.Name) log.Infof("[115_open] deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { log.Errorf("[115_open] delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) return nil, fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) @@ -190,6 +211,9 @@ func addOfflineDownloadTask(ctx context.Context, client offlineTaskClient, url s } func preCleanDuplicateOfflineTasks(ctx context.Context, client offlineTaskClient, url string) error { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } taskList, listErr := client.OfflineList(ctx) if listErr != nil || taskList == nil { log.Warnf("[115_open] pre-add offline list failed: err=%v", listErr) @@ -205,6 +229,9 @@ func preCleanDuplicateOfflineTasks(ctx context.Context, client offlineTaskClient continue } log.Infof("[115_open] pre-add deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { log.Errorf("[115_open] pre-add delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) return fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) @@ -220,6 +247,9 @@ func preCleanDuplicateOfflineTasks(ctx context.Context, client offlineTaskClient } func offlineDownloadWithDetails(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, nil, "", err + } if detailClient, ok := client.(offlineTaskDetailClient); ok { return detailClient.OfflineDownloadWithDetails(ctx, []string{url}, parentDir) } @@ -501,6 +531,10 @@ func mapKeys(values map[string]struct{}) []string { func waitForOfflineTaskRemoval(ctx context.Context, client offlineTaskClient, infoHash string) { const maxChecks = 3 for attempt := 1; attempt <= maxChecks; attempt++ { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + log.Warnf("[115_open] post-delete wait limit failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } taskList, err := client.OfflineList(ctx) if err != nil { log.Warnf("[115_open] post-delete check failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) diff --git a/internal/offline_download/115_open/client_test.go b/internal/offline_download/115_open/client_test.go index 29305a57e..2e8b94778 100644 --- a/internal/offline_download/115_open/client_test.go +++ b/internal/offline_download/115_open/client_test.go @@ -15,6 +15,8 @@ type mockOfflineTaskClient struct { offlineDownloadWithDetailsFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) offlineListFunc func(ctx context.Context) (*sdk.OfflineTaskListResp, error) deleteOfflineFunc func(ctx context.Context, infoHash string, deleteFiles bool) error + waitLimitFunc func(ctx context.Context) error + waitLimitCalls int } func (m *mockOfflineTaskClient) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { @@ -37,6 +39,14 @@ func (m *mockOfflineTaskClient) DeleteOfflineTask(ctx context.Context, infoHash return m.deleteOfflineFunc(ctx, infoHash, deleteFiles) } +func (m *mockOfflineTaskClient) WaitLimit(ctx context.Context) error { + m.waitLimitCalls++ + if m.waitLimitFunc != nil { + return m.waitLimitFunc(ctx) + } + return nil +} + func TestIsDuplicateOfflineTaskError(t *testing.T) { t.Parallel() @@ -225,6 +235,30 @@ func TestAddOfflineDownloadTask(t *testing.T) { } }) + t.Run("wait limit applied for add flow", func(t *testing.T) { + t.Parallel() + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return nil + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if client.waitLimitCalls < 2 { + t.Fatalf("want wait limit calls >= 2, got %d", client.waitLimitCalls) + } + }) + t.Run("delete duplicate and retry", func(t *testing.T) { t.Parallel() From a58cc89fbca3bba43880a4f284530e5c3f2fb4a8 Mon Sep 17 00:00:00 2001 From: cyk Date: Fri, 20 Mar 2026 14:47:05 +0800 Subject: [PATCH 64/86] feat(115_open): expose proxy_range option --- drivers/115_open/meta.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go index ed908e2e6..efb477bd2 100644 --- a/drivers/115_open/meta.go +++ b/drivers/115_open/meta.go @@ -18,9 +18,10 @@ type Addition struct { } var config = driver.Config{ - Name: "115 Open", - DefaultRoot: "0", - LinkCacheMode: driver.LinkCacheUA, + Name: "115 Open", + DefaultRoot: "0", + ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheUA, } func init() { From cb26d17951b5e4b7844af0e092e5a3d873bac533 Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 22 Mar 2026 13:11:34 +0800 Subject: [PATCH 65/86] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E6=B0=B8?= =?UTF-8?q?=E4=B9=85=E5=88=A0=E9=99=A4=E5=8A=9F=E8=83=BD=E5=8F=8A=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/115_open/driver.go | 113 +++++++++++++- drivers/115_open/driver_test.go | 262 ++++++++++++++++++++++++++++++++ drivers/115_open/meta.go | 1 + 3 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 drivers/115_open/driver_test.go diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index d09198031..52f1a458b 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -243,13 +243,124 @@ func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { if !ok { return fmt.Errorf("can't convert obj") } - _, err := d.client.DelFile(ctx, &sdk.DelFileReq{ + resp, err := d.client.DelFile(ctx, &sdk.DelFileReq{ FileIDs: _obj.GetID(), ParentID: _obj.Pid, }) if err != nil { return err } + if d.RemoveWay != "delete" { + return nil + } + return d.removePermanently(ctx, _obj, resp) +} + +func (d *Open115) removePermanently(ctx context.Context, obj *Obj, deleteResp []string) error { + var directDeleteErr error + for _, tid := range deleteResp { + tid = strings.TrimSpace(tid) + if tid == "" { + continue + } + if err := d.deleteRecycleBinEntry(ctx, tid); err == nil { + return nil + } else if directDeleteErr == nil { + directDeleteErr = err + } + } + + recycleEntry, err := d.findRecycleBinEntry(ctx, obj) + if err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin candidate: %w; fallback lookup failed: %v", directDeleteErr, err) + } + return err + } + if err := d.deleteRecycleBinEntry(ctx, recycleEntry.ID); err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin entry %s after candidate delete error %v: %w", recycleEntry.ID, directDeleteErr, err) + } + return err + } + return nil +} + +func (d *Open115) deleteRecycleBinEntry(ctx context.Context, tid string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + _, err := d.client.RbDelete(ctx, tid) + return err +} + +func (d *Open115) findRecycleBinEntry(ctx context.Context, obj *Obj) (*sdk.RbListResp_FileInfo, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 200 + } else if pageSize > 1150 { + pageSize = 1150 + } + + offset := int64(0) + for { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + resp, err := d.client.RbList(ctx, pageSize, offset) + if err != nil { + return nil, err + } + if entry := matchRecycleBinEntry(obj, resp.Files); entry != nil { + return entry, nil + } + + count, err := strconv.ParseInt(resp.Count, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse recycle bin count %q: %w", resp.Count, err) + } + offset += pageSize + if offset >= count || len(resp.Files) == 0 { + break + } + } + + return nil, fmt.Errorf("recycle bin entry not found for object id=%s name=%s parent=%s", obj.GetID(), obj.GetName(), obj.Pid) +} + +func matchRecycleBinEntry(obj *Obj, files map[string]sdk.RbListResp_FileInfo) *sdk.RbListResp_FileInfo { + if len(files) == 0 { + return nil + } + if entry, ok := files[obj.GetID()]; ok { + matched := entry + return &matched + } + + size := strconv.FormatInt(obj.GetSize(), 10) + for _, entry := range files { + if entry.ID == obj.GetID() { + matched := entry + return &matched + } + if obj.IsDir() { + if entry.FileName == obj.GetName() && entry.CID == obj.Pid { + matched := entry + return &matched + } + continue + } + if obj.Sha1 != "" && entry.SHA1 != "" && strings.EqualFold(entry.SHA1, obj.Sha1) { + if entry.FileName == obj.GetName() || entry.CID == obj.Pid { + matched := entry + return &matched + } + } + if entry.FileName == obj.GetName() && entry.CID == obj.Pid && entry.FileSize == size { + matched := entry + return &matched + } + } return nil } diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go new file mode 100644 index 000000000..568e882e1 --- /dev/null +++ b/drivers/115_open/driver_test.go @@ -0,0 +1,262 @@ +package _115_open + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type recordedRequest struct { + Path string + Form url.Values +} + +type rewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (t *rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + cloned := req.Clone(req.Context()) + cloned.URL.Scheme = t.target.Scheme + cloned.URL.Host = t.target.Host + return t.base.RoundTrip(cloned) +} + +func TestOpen115RemoveTrashUsesDelFileOnly(t *testing.T) { + driver, requests := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { + writeSDKSuccess(t, w, []string{"rb-123"}) + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete") + assertFormValue(t, requests()[0].Form, "file_ids", "file-1") + assertFormValue(t, requests()[0].Form, "parent_id", "dir-1") +} + +func TestOpen115RemoveDeleteUsesDelFileResponseIDWhenAvailable(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/del": + writeSDKSuccess(t, w, []string{"rb-123"}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del") + assertFormValue(t, requests()[1].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteFallsBackToRecycleBinLookup(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "file_size": "123", + "cid": "dir-1", + "sha1": "sha-demo", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteReturnsErrorWhenRecycleEntryMissing(t *testing.T) { + driver, _ := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + err := driver.Remove(context.Background(), obj) + if err == nil { + t.Fatalf("expected Remove to fail when recycle-bin entry is missing") + } + if !strings.Contains(err.Error(), "recycle bin entry not found") { + t.Fatalf("expected recycle-bin lookup error, got: %v", err) + } +} + +func TestOpen115DriverInfoIncludesRemoveWay(t *testing.T) { + info, ok := op.GetDriverInfoMap()["115 Open"] + if !ok { + t.Fatalf("115 Open driver info was not registered") + } + + for _, item := range info.Additional { + if item.Name != "remove_way" { + continue + } + if item.Type != "select" { + t.Fatalf("unexpected remove_way type: %q", item.Type) + } + if item.Options != "trash,delete" { + t.Fatalf("unexpected remove_way options: %q", item.Options) + } + if item.Default != "trash" { + t.Fatalf("unexpected remove_way default: %q", item.Default) + } + if !item.Required { + t.Fatalf("expected remove_way to be required") + } + return + } + + t.Fatalf("remove_way item not found in 115 Open driver info") +} + +func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { + t.Helper() + + var ( + mu sync.Mutex + requests []recordedRequest + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Fatalf("ParseForm failed: %v", err) + } + mu.Lock() + requests = append(requests, recordedRequest{ + Path: r.URL.Path, + Form: cloneValues(r.Form), + }) + mu.Unlock() + responder(w, r) + })) + t.Cleanup(server.Close) + + target, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("Parse server URL failed: %v", err) + } + + client := sdk.New(sdk.WithAccessToken("test-token")) + client.SetHttpClient(&http.Client{ + Transport: &rewriteTransport{ + target: target, + base: http.DefaultTransport, + }, + }) + + return &Open115{ + Addition: Addition{ + RemoveWay: removeWay, + PageSize: 1, + }, + client: client, + }, func() []recordedRequest { + mu.Lock() + defer mu.Unlock() + return append([]recordedRequest(nil), requests...) + } +} + +func writeSDKSuccess(t *testing.T, w http.ResponseWriter, data any) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": true, + "data": data, + }) +} + +func writeSDKError(t *testing.T, w http.ResponseWriter, code int64, message string) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": false, + "code": code, + "message": message, + }) +} + +func writeSDKResponse(t *testing.T, w http.ResponseWriter, payload map[string]any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(payload); err != nil { + t.Fatalf("Encode response failed: %v", err) + } +} + +func assertRequestPaths(t *testing.T, requests []recordedRequest, want ...string) { + t.Helper() + got := make([]string, 0, len(requests)) + for _, req := range requests { + got = append(got, req.Path) + } + if !slices.Equal(got, want) { + t.Fatalf("unexpected request paths: got %v want %v", got, want) + } +} + +func assertFormValue(t *testing.T, form url.Values, key, want string) { + t.Helper() + if got := form.Get(key); got != want { + t.Fatalf("unexpected form value for %s: got %q want %q", key, got, want) + } +} + +func cloneValues(src url.Values) url.Values { + dst := make(url.Values, len(src)) + for key, values := range src { + dst[key] = append([]string(nil), values...) + } + return dst +} diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go index efb477bd2..0479ac1fd 100644 --- a/drivers/115_open/meta.go +++ b/drivers/115_open/meta.go @@ -11,6 +11,7 @@ type Addition struct { // define other OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete" default:"trash"` LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` PageSize int64 `json:"page_size" type:"number" default:"200" help:"list api per page size of 115open driver"` AccessToken string `json:"access_token" required:"true"` From fc06cf0e15ff44b5995eaf29d1020f96b7d321bd Mon Sep 17 00:00:00 2001 From: cyk Date: Mon, 23 Mar 2026 13:26:51 +0800 Subject: [PATCH 66/86] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E5=89=8D?= =?UTF-8?q?=E7=AB=AF=E7=BC=93=E5=AD=98=E6=9C=BA=E5=88=B6=E4=BB=A5=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E6=9E=84=E5=BB=BA=E8=BF=87=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test_docker.yml | 57 +++++++++++++++++-------------- build.sh | 12 +++++++ 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 7542dcf11..1f1418de1 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -14,10 +14,13 @@ concurrency: cancel-in-progress: true env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 + FRONTEND_REPO: ${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} IMAGE_NAME: openlist REGISTRY: ghcr.io ARTIFACT_NAME: 'binaries_docker_release' + WEB_VERSION: latest # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 RELEASE_PLATFORMS: 'linux/amd64' # 👇 关键修改：强制允许推送，不用管是不是 push 事件 @@ -43,32 +46,39 @@ jobs: cache: true cache-dependency-path: go.sum - # 获取前端仓库的最新commit SHA - - name: Get Frontend Commit SHA - id: frontend-sha + - name: Get Frontend Cache Version + id: frontend-cache + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - FRONTEND_REPO="${{ env.FRONTEND_REPO }}" - # 如果未设置FRONTEND_REPO，使用默认值 - if [ -z "$FRONTEND_REPO" ]; then - FRONTEND_REPO="OpenListTeam/OpenList-Frontend" + frontend_repo="${{ env.FRONTEND_REPO }}" + web_version="${{ env.WEB_VERSION }}" + github_auth_args=() + + if [ -n "$GH_TOKEN" ]; then + github_auth_args=(-H "Authorization: Bearer $GH_TOKEN") + fi + + if [ "$web_version" = "latest" ]; then + frontend_version=$(curl -fsSL "${github_auth_args[@]}" "https://api.github.com/repos/$frontend_repo/releases/latest" | jq -r '.tag_name') + else + frontend_version="$web_version" fi - FRONTEND_SHA=$(curl -s https://api.github.com/repos/$FRONTEND_REPO/commits/main | jq -r '.sha') - echo "sha=$FRONTEND_SHA" >> $GITHUB_OUTPUT - echo "repo=$FRONTEND_REPO" >> $GITHUB_OUTPUT - echo "Frontend repo: $FRONTEND_REPO" - echo "Frontend repo latest commit: $FRONTEND_SHA" - # 缓存前端下载 - key包含前端仓库的commit SHA + echo "repo=$frontend_repo" >> "$GITHUB_OUTPUT" + echo "version=$frontend_version" >> "$GITHUB_OUTPUT" + echo "Frontend repo: $frontend_repo" + echo "Frontend cache version: $frontend_version" + - name: Cache Frontend id: cache-frontend uses: actions/cache@v4 with: path: public/dist - key: frontend-${{ steps.frontend-sha.outputs.repo }}-${{ steps.frontend-sha.outputs.sha }} + key: frontend-${{ steps.frontend-cache.outputs.repo }}-${{ steps.frontend-cache.outputs.version }} restore-keys: | - frontend-${{ steps.frontend-sha.outputs.repo }}- + frontend-${{ steps.frontend-cache.outputs.repo }}- - # 即使只构建 x64，我们也需要 musl 工具链（因为 BuildDockerMultiplatform 默认会检查它） - name: Cache Musl id: cache-musl uses: actions/cache@v4 @@ -78,27 +88,24 @@ jobs: - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' - run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash build.sh prepare docker-multiplatform - name: Build go binary - # 这里还是跑 docker-multiplatform，虽然会多编译一些架构，但这是兼容 Dockerfile 路径最稳妥的方法 run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WEB_VERSION: latest - # FRONTEND_REPO 使用 build.sh 默认值 (OpenListTeam/OpenList-Frontend) + WEB_VERSION: ${{ env.WEB_VERSION }} + FRONTEND_REPO: ${{ env.FRONTEND_REPO }} + SKIP_FRONTEND_FETCH: ${{ steps.cache-frontend.outputs.cache-hit == 'true' && 'true' || 'false' }} - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true - path: | - build/ - !build/*.tgz - !build/musl-libs/** + path: build/linux/amd64/openlist release_docker: needs: build_binary @@ -135,7 +142,7 @@ jobs: - uses: actions/download-artifact@v4 with: name: ${{ env.ARTIFACT_NAME }} - path: 'build/' + path: 'build/linux/amd64' - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/build.sh b/build.sh index afbe44ae1..04cb3ddf0 100644 --- a/build.sh +++ b/build.sh @@ -18,6 +18,8 @@ if [[ "$*" == *"lite"* ]]; then useLite=true fi +skipFrontendFetch="${SKIP_FRONTEND_FETCH:-false}" + if [ "$1" = "dev" ]; then version="dev" webVersion="rolling" @@ -66,6 +68,11 @@ GetBuildTagsForTarget() { } FetchWebRolling() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') @@ -79,6 +86,11 @@ FetchWebRolling() { } FetchWebRelease() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/latest\"") release_assets=$(echo "$release_json" | jq -r '.assets[].browser_download_url') From 86eac21cfd50fb4e3a151919ffe7201961dfb7cc Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 5 Apr 2026 00:17:03 +0800 Subject: [PATCH 67/86] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=20WebVersion?= =?UTF-8?q?=20=E4=B8=BA=20rolling=EF=BC=8C=E5=B9=B6=E6=B7=BB=E5=8A=A0=20GO?= =?UTF-8?q?FLAGS=20=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/beta_release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 98615b0e0..97312f52e 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -128,7 +128,9 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag - github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=latest + github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + env: + GOFLAGS: ${{ matrix.goflags }} - name: Compress run: | From 9b793c08f955a814b2587da7a93b8759b6aa7eaf Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 5 Apr 2026 00:31:47 +0800 Subject: [PATCH 68/86] =?UTF-8?q?feat:=20=E5=B0=86=20WEB=5FVERSION=20?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=B8=BA=20rolling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 1f1418de1..e949d8249 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -20,7 +20,7 @@ env: IMAGE_NAME: openlist REGISTRY: ghcr.io ARTIFACT_NAME: 'binaries_docker_release' - WEB_VERSION: latest + WEB_VERSION: rolling # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 RELEASE_PLATFORMS: 'linux/amd64' # 👇 关键修改：强制允许推送，不用管是不是 push 事件 From a51385fb34165caf5d22a7066794cf44ac138d8a Mon Sep 17 00:00:00 2001 From: cyk Date: Sun, 5 Apr 2026 00:39:20 +0800 Subject: [PATCH 69/86] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=20Docker=20?= =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E6=B5=81=E4=BB=A5=E6=94=AF=E6=8C=81=E5=89=8D?= =?UTF-8?q?=E7=AB=AF=E7=89=88=E6=9C=AC=E7=9F=A9=E9=98=B5=E6=9E=84=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test_docker.yml | 40 ++++++++++++++++--------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index e949d8249..96259f989 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -19,22 +19,21 @@ env: FRONTEND_REPO: ${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} IMAGE_NAME: openlist REGISTRY: ghcr.io - ARTIFACT_NAME: 'binaries_docker_release' - WEB_VERSION: rolling + ARTIFACT_NAME_PREFIX: 'binaries_docker_release' # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 RELEASE_PLATFORMS: 'linux/amd64' # 👇 关键修改：强制允许推送，不用管是不是 push 事件 IMAGE_PUSH: 'true' # 👇 使用默认的前端仓库 (OpenListTeam/OpenList-Frontend) # FRONTEND_REPO: 'Ironboxplus/OpenList-Frontend' - IMAGE_TAGS_BETA: | - type=ref,event=pr - type=raw,value=beta-retry jobs: build_binary: - name: Build Binaries (x64 Only) + name: Build Binaries (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest + strategy: + matrix: + frontend_channel: ["latest", "rolling"] steps: - name: Checkout uses: actions/checkout@v4 @@ -50,9 +49,10 @@ jobs: id: frontend-cache env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WEB_VERSION: ${{ matrix.frontend_channel }} run: | frontend_repo="${{ env.FRONTEND_REPO }}" - web_version="${{ env.WEB_VERSION }}" + web_version="$WEB_VERSION" github_auth_args=() if [ -n "$GH_TOKEN" ]; then @@ -96,52 +96,53 @@ jobs: run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WEB_VERSION: ${{ env.WEB_VERSION }} + WEB_VERSION: ${{ matrix.frontend_channel }} FRONTEND_REPO: ${{ env.FRONTEND_REPO }} SKIP_FRONTEND_FETCH: ${{ steps.cache-frontend.outputs.cache-hit == 'true' && 'true' || 'false' }} - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: ${{ env.ARTIFACT_NAME }} + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} overwrite: true path: build/linux/amd64/openlist release_docker: needs: build_binary - name: Release Docker (x64) + name: Release Docker (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: + frontend_channel: ["latest", "rolling"] # 构建所有变体 image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" base_image_tag: "base" build_arg: "" - tag_favor: "" + image_tag_suffix: "" - image: "ffmpeg" base_image_tag: "ffmpeg" build_arg: INSTALL_FFMPEG=true - tag_favor: "suffix=-ffmpeg,onlatest=true" + image_tag_suffix: "-ffmpeg" - image: "aria2" base_image_tag: "aria2" build_arg: INSTALL_ARIA2=true - tag_favor: "suffix=-aria2,onlatest=true" + image_tag_suffix: "-aria2" - image: "aio" base_image_tag: "aio" build_arg: | INSTALL_FFMPEG=true INSTALL_ARIA2=true - tag_favor: "suffix=-aio,onlatest=true" + image_tag_suffix: "-aio" steps: - name: Checkout uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: - name: ${{ env.ARTIFACT_NAME }} + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} path: 'build/linux/amd64' - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -160,8 +161,9 @@ jobs: with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} - tags: ${{ env.IMAGE_TAGS_BETA }} - flavor: ${{ matrix.tag_favor }} + tags: | + type=raw,value=front-${{ matrix.frontend_channel }}${{ matrix.image_tag_suffix }} + type=raw,value=latest,enable=${{ matrix.frontend_channel == 'latest' && matrix.image == 'latest' }} - name: Build and push uses: docker/build-push-action@v6 @@ -175,5 +177,5 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.image }} - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.image }},mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }},mode=max From da26e72beeed608c4d4bf3add1e6b801fba32bae Mon Sep 17 00:00:00 2001 From: Jealous Date: Thu, 9 Apr 2026 09:32:18 +0800 Subject: [PATCH 70/86] fix(op): invalidate new path cache on meta path update (#2322) --- internal/op/meta.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/op/meta.go b/internal/op/meta.go index ed9e422a9..b7d867307 100644 --- a/internal/op/meta.go +++ b/internal/op/meta.go @@ -78,6 +78,7 @@ func UpdateMeta(u *model.Meta) error { return err } metaCache.Del(old.Path) + metaCache.Del(u.Path) return db.UpdateMeta(u) } From 8d39d636be112532d89ff83a5de4cb9fd62c0883 Mon Sep 17 00:00:00 2001 From: Suyunjing Date: Thu, 9 Apr 2026 20:57:04 +0800 Subject: [PATCH 71/86] fix(build): lock musl outputs to fully static linking in build script and CI workflows (#2330) --- .github/workflows/beta_release.yml | 32 +++++++++++++++-- .github/workflows/build.yml | 12 +++++++ build.sh | 58 +++++++++++++++++++++++++----- 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 97312f52e..1ea9e6203 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -67,32 +67,39 @@ jobs: goflags: "" - target: "linux-(mips|mips64|mipsle|mips64le|loong64)-musl*" # musl-compat-family hash: "md5-linux-musl-mips" - flags: "" + flags: "-ldflags=-linkmode external -extldflags '-static -fpic'" goflags: "" + musl_static: "true" - target: "linux-!(arm*|mips|mips64|mipsle|mips64le|loong64)-musl*" # musl-not-arm (exclude compat-family) hash: "md5-linux-musl" - flags: "" + flags: "-ldflags=-linkmode external -extldflags '-static -fpic'" goflags: "" + musl_static: "true" - target: "linux-arm*-musl*" #musl-arm hash: "md5-linux-musl-arm" - flags: "" + flags: "-ldflags=-linkmode external -extldflags '-static -fpic'" goflags: "" + musl_static: "true" - target: "windows-arm64" #win-arm64 hash: "md5-windows-arm64" flags: "" goflags: "" + musl_static: "false" - target: "windows7-*" #win7 hash: "md5-windows7" flags: "" goflags: "-tags=sqlite_cgo_compat" + musl_static: "false" - target: "android-*" #android hash: "md5-android" flags: "" goflags: "" + musl_static: "false" - target: "freebsd-*" #freebsd hash: "md5-freebsd" flags: "" goflags: "" + musl_static: "false" name: Beta Release runs-on: ubuntu-latest @@ -118,6 +125,7 @@ jobs: with: targets: ${{ matrix.target }} flags: ${{ matrix.flags || '-ldflags=' }} + static-link-for-musl: true musl-target-format: $os-$musl-$arch github-token: ${{ secrets.GITHUB_TOKEN }} out-dir: build @@ -132,6 +140,24 @@ jobs: env: GOFLAGS: ${{ matrix.goflags }} + - name: Verify musl binaries are static + if: matrix.musl_static == 'true' + run: | + set -e + shopt -s nullglob + files=(build/openlist-*-musl-*) + if [ ${#files[@]} -eq 0 ]; then + echo "No musl binaries found" + exit 1 + fi + for f in "${files[@]}"; do + if readelf -l "$f" | grep -q "Requesting program interpreter"; then + echo "Dynamic binary detected: $f" + readelf -l "$f" | grep "Requesting program interpreter" || true + exit 1 + fi + done + - name: Compress run: | bash build.sh zip ${{ matrix.hash }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2393b84b..c54dde663 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,8 @@ jobs: uses: OpenListTeam/cgo-actions@v1.2.2 with: targets: ${{ matrix.target }} + flags: ${{ contains(matrix.target, '-musl') && '-ldflags=-linkmode external -extldflags ''-static -fpic''' || '-ldflags=' }} + static-link-for-musl: true musl-target-format: $os-$musl-$arch github-token: ${{ secrets.GITHUB_TOKEN }} out-dir: build @@ -56,6 +58,16 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling output: openlist$ext + - name: Verify musl binary is static + if: contains(matrix.target, '-musl') + run: | + set -e + if readelf -l build/openlist | grep -q "Requesting program interpreter"; then + echo "Dynamic binary detected: build/openlist" + readelf -l build/openlist | grep "Requesting program interpreter" || true + exit 1 + fi + - name: Upload artifact uses: actions/upload-artifact@v4 with: diff --git a/build.sh b/build.sh index 3198d7ce3..b6baca1fe 100644 --- a/build.sh +++ b/build.sh @@ -61,6 +61,41 @@ GetBuildTagsForTarget() { esac } +# Keep musl static link flags centralized for all musl build paths. +GetMuslStaticLdflags() { + echo "-linkmode external -extldflags '-static -fpic' $ldflags" +} + +# Fail fast if a musl build artifact is not fully static. +AssertStaticBinary() { + local binary="$1" + if [ ! -f "$binary" ]; then + echo "Error: binary not found: $binary" + return 1 + fi + + if command -v readelf >/dev/null 2>&1; then + if readelf -l "$binary" 2>/dev/null | grep -q "Requesting program interpreter"; then + echo "Error: binary is not fully static: $binary" + readelf -l "$binary" | grep "Requesting program interpreter" || true + return 1 + fi + return 0 + fi + + if command -v file >/dev/null 2>&1; then + if file "$binary" | grep -qi "dynamically linked"; then + echo "Error: binary is dynamically linked: $binary" + file "$binary" + return 1 + fi + return 0 + fi + + echo "Warning: readelf/file not found, skip static verification for $binary" + return 0 +} + FetchWebRolling() { pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') @@ -145,7 +180,7 @@ BuildWin7() { BuildDev() { rm -rf .git/ mkdir -p "dist" - muslflags="--extldflags '-static -fpic' $ldflags" + muslflags="$(GetMuslStaticLdflags)" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross) for i in "${FILES[@]}"; do @@ -163,7 +198,8 @@ BuildDev() { export GOARCH=${os_arch##*-} export CC=${cgo_cc} export CGO_ENABLED=1 - go build -o ./dist/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + CGO_LDFLAGS="-static" go build -o ./dist/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + AssertStaticBinary "./dist/$appName-$os_arch" done xgo -targets=windows/amd64,darwin/amd64,darwin/arm64 -out "$appName" -ldflags="$ldflags" -tags=jsoniter . mv "$appName"-* dist @@ -197,7 +233,7 @@ BuildDockerMultiplatform() { # run PrepareBuildDockerMusl before build export PATH=$PATH:$PWD/build/musl-libs/bin - docker_lflags="--extldflags '-static -fpic' $ldflags" + docker_lflags="$(GetMuslStaticLdflags)" export CGO_ENABLED=1 OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds @@ -212,7 +248,8 @@ BuildDockerMultiplatform() { export GOARCH=$arch export CC=${cgo_cc} echo "building for $os_arch" - go build -o build/$os/$arch/"$appName" -ldflags="$docker_lflags" -tags="$build_tags" . + CGO_LDFLAGS="-static" go build -o build/$os/$arch/"$appName" -ldflags="$docker_lflags" -tags="$build_tags" . + AssertStaticBinary "build/$os/$arch/$appName" done DOCKER_ARM_ARCHES=(linux-arm/v6 linux-arm/v7) @@ -226,7 +263,8 @@ BuildDockerMultiplatform() { export GOARM=${GO_ARM[$i]} export CC=${cgo_cc} echo "building for $docker_arch" - go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + CGO_LDFLAGS="-static" go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + AssertStaticBinary "build/${docker_arch%%-*}/${docker_arch##*-}/$appName" done } @@ -406,7 +444,7 @@ BuildLoongGLIBC() { BuildReleaseLinuxMusl() { rm -rf .git/ mkdir -p "build" - muslflags="--extldflags '-static -fpic' $ldflags" + muslflags="$(GetMuslStaticLdflags)" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" # Keep mips-family targets enabled; sqlite driver selection is handled by Go build tags. FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross mips-linux-musl-cross mips64-linux-musl-cross mips64el-linux-musl-cross mipsel-linux-musl-cross powerpc64le-linux-musl-cross s390x-linux-musl-cross loongarch64-linux-musl-cross) @@ -427,14 +465,15 @@ BuildReleaseLinuxMusl() { export GOARCH=${os_arch##*-} export CC=${cgo_cc} export CGO_ENABLED=1 - go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags="$build_tags" . + CGO_LDFLAGS="-static" go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags="$build_tags" . + AssertStaticBinary "./build/$appName-$os_arch" done } BuildReleaseLinuxMuslArm() { rm -rf .git/ mkdir -p "build" - muslflags="--extldflags '-static -fpic' $ldflags" + muslflags="$(GetMuslStaticLdflags)" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" FILES=(arm-linux-musleabi-cross arm-linux-musleabihf-cross armel-linux-musleabi-cross armel-linux-musleabihf-cross armv5l-linux-musleabi-cross armv5l-linux-musleabihf-cross armv6-linux-musleabi-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross armv7m-linux-musleabi-cross armv7r-linux-musleabihf-cross) for i in "${FILES[@]}"; do @@ -456,7 +495,8 @@ BuildReleaseLinuxMuslArm() { export CC=${cgo_cc} export CGO_ENABLED=1 export GOARM=${arm} - go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + CGO_LDFLAGS="-static" go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + AssertStaticBinary "./build/$appName-$os_arch" done } From b31ae9c9f6d831877566560b0827897f8eb23914 Mon Sep 17 00:00:00 2001 From: ShenLin <773933146@qq.com> Date: Fri, 10 Apr 2026 20:47:45 +0800 Subject: [PATCH 72/86] ci(actions): upgrade actions/checkout to v6 (#2338) Replace actions/checkout from v4 to v6 across workflows to prepare for the GitHub Actions Node.js 24 migration and avoid Node.js 20 deprecation risk. --- .github/workflows/beta_release.yml | 4 ++-- .github/workflows/build.yml | 2 +- .github/workflows/changelog.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/release_docker.yml | 8 ++++---- .github/workflows/sync_repo.yml | 2 +- .github/workflows/test_docker.yml | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 1ea9e6203..d5817e6b4 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 @@ -105,7 +105,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c54dde663..a3a501ffa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - uses: benjlevesque/short-sha@v3.0 id: short-sha diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 8d458314a..e14eaaa76 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9bbf0f1ca..2fa13af18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,7 @@ jobs: go-version: '1.25.0' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 80c065647..80bdf9e3c 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - uses: actions/setup-go@v5 with: @@ -83,7 +83,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - uses: actions/setup-go@v5 with: @@ -146,7 +146,7 @@ jobs: tag_favor: "suffix=-aio,onlatest=true" steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - uses: actions/download-artifact@v4 with: name: ${{ env.ARTIFACT_NAME }} @@ -230,7 +230,7 @@ jobs: tag_favor: "suffix=-lite-aio,onlatest=true" steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - uses: actions/download-artifact@v4 with: name: ${{ env.ARTIFACT_NAME_LITE }} diff --git a/.github/workflows/sync_repo.yml b/.github/workflows/sync_repo.yml index 621d5fb5d..fe57905dc 100644 --- a/.github/workflows/sync_repo.yml +++ b/.github/workflows/sync_repo.yml @@ -12,7 +12,7 @@ jobs: name: Sync GitHub to Gitee steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index aa6fe8966..16c299401 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - uses: actions/setup-go@v5 with: @@ -97,7 +97,7 @@ jobs: tag_favor: "suffix=-aio,onlatest=true" steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - uses: actions/download-artifact@v4 with: name: ${{ env.ARTIFACT_NAME }} From 331f575c170258baf3bcc7042df2bc86f3a24813 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sun, 19 Apr 2026 09:54:17 +0800 Subject: [PATCH 73/86] fix(drivers/139): check cdnSwitch before returning cdnUrl in personalGetLink (#2379) Signed-off-by: MadDogOwner --- drivers/139/util.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/139/util.go b/drivers/139/util.go index 2c7ac242d..85c798cc7 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -655,10 +655,12 @@ func (d *Yun139) personalGetLink(fileId string) (string, error) { } cdnUrl := jsoniter.Get(res, "data", "cdnUrl").ToString() if cdnUrl != "" { - return cdnUrl, nil - } else { - return jsoniter.Get(res, "data", "url").ToString(), nil + cdnSwitch := jsoniter.Get(res, "data", "cdnSwitch").ToBool() + if cdnSwitch { + return cdnUrl, nil + } } + return jsoniter.Get(res, "data", "url").ToString(), nil } func (d *Yun139) getAuthorization() string { From a5ba6a0e9d816d1b1ea8a5777c68b041eeb96a1f Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 20 Apr 2026 18:05:15 +0800 Subject: [PATCH 74/86] refactor(settings)!: move FilterReadMeScripts to frontend (#2346) * refactor(settings)!: move FilterReadMeScripts to frontend Signed-off-by: MadDogOwner * chore: run go mod tidy Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner --- go.mod | 1 - go.sum | 2 -- internal/bootstrap/data/setting.go | 2 +- server/handles/down.go | 33 +----------------------------- 4 files changed, 2 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index 2fc141f2e..cd86a8147 100644 --- a/go.mod +++ b/go.mod @@ -168,7 +168,6 @@ require ( github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/yuin/goldmark v1.7.13 go4.org v0.0.0-20260112195520-a5071408f32f resty.dev/v3 v3.0.0-beta.2 // indirect ) diff --git a/go.sum b/go.sum index 0f69ce117..758741249 100644 --- a/go.sum +++ b/go.sum @@ -668,8 +668,6 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= -github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 h1:PSRwrE5QBufPnOjdgIkRs5KBV1Avq3SY8oksj2Z+k3o= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 7bff851de..d7fd8ea47 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -153,7 +153,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.SharePreviewDownloadByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.SharePreviewArchivesByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, - {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, // frontend {Key: conf.NonEFSZipEncoding, Value: "IBM437", Type: conf.TypeString, Group: model.PREVIEW}, // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, diff --git a/server/handles/down.go b/server/handles/down.go index d4d634cbe..50025a0b6 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -1,11 +1,8 @@ package handles import ( - "bytes" "errors" - "fmt" stdpath "path" - "strconv" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -17,9 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" - "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" - "github.com/yuin/goldmark" ) func Down(c *gin.Context) { @@ -115,33 +110,7 @@ func proxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) { link = common.ProxyRange(c, link, file.GetSize()) } Writer := &common.WrittenResponseWriter{ResponseWriter: c.Writer} - raw, _ := strconv.ParseBool(c.DefaultQuery("raw", "false")) - if utils.Ext(file.GetName()) == "md" && setting.GetBool(conf.FilterReadMeScripts) && !raw { - buf := bytes.NewBuffer(make([]byte, 0, file.GetSize())) - w := &common.InterceptResponseWriter{ResponseWriter: Writer, Writer: buf} - err = common.Proxy(w, c.Request, link, file) - if err == nil && buf.Len() > 0 { - if c.Writer.Status() < 200 || c.Writer.Status() > 300 { - c.Writer.Write(buf.Bytes()) - return - } - - var html bytes.Buffer - if err = goldmark.Convert(buf.Bytes(), &html); err != nil { - err = fmt.Errorf("markdown conversion failed: %w", err) - } else { - buf.Reset() - err = bluemonday.UGCPolicy().SanitizeReaderToWriter(&html, buf) - if err == nil { - Writer.Header().Set("Content-Length", strconv.FormatInt(int64(buf.Len()), 10)) - Writer.Header().Set("Content-Type", "text/html; charset=utf-8") - _, err = utils.CopyWithBuffer(Writer, buf) - } - } - } - } else { - err = common.Proxy(Writer, c.Request, link, file) - } + err = common.Proxy(Writer, c.Request, link, file) if err == nil { return } From ece1518eaca04de9fb32599a180e611e5a6279bf Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sun, 26 Apr 2026 15:17:40 +0800 Subject: [PATCH 75/86] fix(drivers): add headers in Link methods (#2401) * fix(drivers): add headers in Link methods Signed-off-by: MadDogOwner --- drivers/cloudreve/driver.go | 4 ++++ drivers/cloudreve_v4/driver.go | 4 ++++ drivers/lanzou/driver.go | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 55462be7e..12c4fc028 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -89,6 +89,10 @@ func (d *Cloudreve) Link(ctx context.Context, file model.Obj, args model.LinkArg } return &model.Link{ URL: dUrl, + Header: http.Header{ + "Referer": {d.Address}, + "User-Agent": {d.getUA()}, + }, }, nil } diff --git a/drivers/cloudreve_v4/driver.go b/drivers/cloudreve_v4/driver.go index 2963bf467..afd64d3d5 100644 --- a/drivers/cloudreve_v4/driver.go +++ b/drivers/cloudreve_v4/driver.go @@ -167,6 +167,10 @@ func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkA return &model.Link{ URL: url.Urls[0].URL, Expiration: &exp, + Header: http.Header{ + "Referer": {d.Address}, + "User-Agent": {d.getUA()}, + }, }, nil } diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index 01d7c1ece..e143cde20 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -117,7 +117,7 @@ func (d *LanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return &model.Link{ URL: dfile.Url, Header: http.Header{ - "User-Agent": []string{base.UserAgent}, + "User-Agent": {d.UserAgent}, }, Expiration: &exp, }, nil From 29ec90e8e8f2d915073435d6a59ac90f7ae39731 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 27 Apr 2026 17:33:05 +0800 Subject: [PATCH 76/86] refactor(drivers/wps): remove resolveCache (#2411) * refactor(drivers/wps): remove resolveCache Signed-off-by: MadDogOwner * fix(drivers/wps): add a small helper that unwraps model.ObjUnwrap Co-authored-by: Copilot Signed-off-by: MadDogOwner * fix(drivers/wps): correct misspell Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner Co-authored-by: Copilot --- drivers/wps/driver.go | 292 +++++++++++++- drivers/wps/meta.go | 5 +- drivers/wps/put.go | 315 +++++++++++++++ drivers/wps/types.go | 128 ++++--- drivers/wps/util.go | 867 ++---------------------------------------- 5 files changed, 709 insertions(+), 898 deletions(-) create mode 100644 drivers/wps/put.go diff --git a/drivers/wps/driver.go b/drivers/wps/driver.go index 8a3ccb6ce..6f78c0b10 100644 --- a/drivers/wps/driver.go +++ b/drivers/wps/driver.go @@ -3,16 +3,23 @@ package wps import ( "context" "fmt" + "net/http" + "strconv" + "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" ) type Wps struct { model.Storage Addition - companyID string + + login *loginState + client *resty.Client } func (d *Wps) Config() driver.Config { @@ -27,10 +34,28 @@ func (d *Wps) Init(ctx context.Context) error { if d.Cookie == "" { return fmt.Errorf("cookie is empty") } - return d.ensureCompanyID(ctx) + + d.client = base.NewRestyClient() + + resp, err := d.request(ctx).SetResult(&d.login).Get("https://account.kdocs.cn/api/v3/islogin") + if err != nil { + return err + } + if !resp.IsSuccess() { + return fmt.Errorf("failed to check login status, status code: %d, body: %s", resp.StatusCode(), resp.String()) + } + + return nil } func (d *Wps) Drop(ctx context.Context) error { + + if d.client != nil { + d.client = nil + } + if d.login != nil { + d.login = nil + } return nil } @@ -41,34 +66,272 @@ func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]mode basePath = p } } - return d.list(ctx, basePath) + if basePath == "/" { + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0, len(groups)) + for _, g := range groups { + path := joinPath(basePath, g.Name) + obj := &Obj{ + Obj: &model.Object{ + ID: strconv.FormatInt(g.GroupID, 10), + Path: path, + Name: g.Name, + Modified: parseTime(0), + Ctime: parseTime(0), + IsFolder: true, + }, + Kind: "group", + GroupID: g.GroupID, + } + res = append(res, obj) + } + return res, nil + } + node, err := unwrapWpsObj(dir) + if err != nil { + return nil, err + } + if node.Kind != "group" && node.Kind != "folder" { + return nil, nil + } + parentID := int64(0) + if node.HasFile && node.Kind == "folder" { + parentID = node.FileID + } + files, err := d.getFiles(ctx, node.GroupID, parentID) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0, len(files)) + for _, f := range files { + res = append(res, f.fileToObj(basePath, d.isPersonal())) + } + return res, nil } func (d *Wps) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { if file == nil { return nil, errs.NotSupport } - return d.link(ctx, file.GetPath()) + node, err := unwrapWpsObj(file) + if err != nil { + return nil, err + } + if node.Kind != "file" || !node.HasFile { + return nil, errs.NotSupport + } + if !node.CanDownload { + return nil, fmt.Errorf("can not download") + } + url := fmt.Sprintf("%s/api/v5/groups/%d/files/%d/download?support_checksums=sha1", d.driveHost()+d.drivePrefix(), node.GroupID, node.FileID) + var resp downloadResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + if resp.URL == "" { + return nil, fmt.Errorf("empty download url") + } + return &model.Link{URL: resp.URL, Header: http.Header{}}, nil } func (d *Wps) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - return d.makeDir(ctx, parentDir, dirName) + if parentDir == nil { + return errs.NotSupport + } + node, err := unwrapWpsObj(parentDir) + if err != nil { + return err + } + if node.Kind != "group" && node.Kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.HasFile && node.Kind == "folder" { + parentID = node.FileID + } + body := map[string]interface{}{ + "groupid": node.GroupID, + "name": dirName, + "parentid": parentID, + } + if err := d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/folder"), body); err != nil { + return err + } + return nil } func (d *Wps) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.move(ctx, srcObj, dstDir) + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := unwrapWpsObj(srcObj) + if err != nil { + return fmt.Errorf("invalid source object type: %w", err) + } + nodeDst, err := unwrapWpsObj(dstDir) + if err != nil { + return fmt.Errorf("invalid destination object type: %w", err) + } + if nodeSrc.Kind != "file" && nodeSrc.Kind != "folder" { + return errs.NotSupport + } + if nodeDst.Kind != "group" && nodeDst.Kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.HasFile && nodeDst.Kind == "folder" { + targetParentID = nodeDst.FileID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.FileID}, + "target_groupid": nodeDst.GroupID, + "target_parentid": targetParentID, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/move", nodeSrc.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + return nil } func (d *Wps) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - return d.rename(ctx, srcObj, newName) + if srcObj == nil { + return errs.NotSupport + } + node, err := unwrapWpsObj(srcObj) + if err != nil { + return err + } + if node.Kind != "file" && node.Kind != "folder" { + return errs.NotSupport + } + url := fmt.Sprintf("/api/v3/groups/%d/files/%d", node.GroupID, node.FileID) + body := map[string]string{"fname": newName} + if err := d.doJSON(ctx, http.MethodPut, d.driveURL(url), body); err != nil { + return err + } + return nil } func (d *Wps) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.copy(ctx, srcObj, dstDir) + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := unwrapWpsObj(srcObj) + if err != nil { + return fmt.Errorf("invalid source object type: %w", err) + } + nodeDst, err := unwrapWpsObj(dstDir) + if err != nil { + return fmt.Errorf("invalid destination object type: %w", err) + } + if nodeSrc.Kind != "file" && nodeSrc.Kind != "folder" { + return errs.NotSupport + } + if nodeDst.Kind != "group" && nodeDst.Kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.HasFile && nodeDst.Kind == "folder" { + targetParentID = nodeDst.FileID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.FileID}, + "groupid": nodeSrc.GroupID, + "target_groupid": nodeDst.GroupID, + "target_parentid": targetParentID, + "duplicated_name_model": 1, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/copy", nodeSrc.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + return nil } func (d *Wps) Remove(ctx context.Context, obj model.Obj) error { - return d.remove(ctx, obj) + if obj == nil { + return errs.NotSupport + } + node, err := unwrapWpsObj(obj) + if err != nil { + return err + } + if node.Kind != "file" && node.Kind != "folder" { + return errs.NotSupport + } + + body := map[string]interface{}{ + "fileids": []int64{node.FileID}, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/delete", node.GroupID) + + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + // 无法连续创建文件夹删除。如果一定要删除，每0.5s 尝试一次创建下一个删除请求，应当避免递归删除文件夹 + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + return nil } func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { @@ -76,14 +339,19 @@ func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer } func (d *Wps) GetDetails(ctx context.Context) (*model.StorageDetails, error) { - quota, err := d.spaces(ctx) + url := fmt.Sprintf("%s/api/v3/spaces", d.driveHost()+d.drivePrefix()) + var resp spacesResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) if err != nil { return nil, err } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: quota.Total, - UsedSpace: quota.Used, + TotalSpace: resp.Total, + UsedSpace: resp.Used, }, }, nil } diff --git a/drivers/wps/meta.go b/drivers/wps/meta.go index 7a3362f3a..a1fb79485 100644 --- a/drivers/wps/meta.go +++ b/drivers/wps/meta.go @@ -7,8 +7,9 @@ import ( type Addition struct { driver.RootPath - Cookie string `json:"cookie" required:"true" type:"text"` - Mode string `json:"mode" type:"select" options:"Personal,Business" default:"Business"` + Cookie string `json:"cookie" required:"true"` + Mode string `json:"mode" type:"select" options:"Personal,Business" default:"Personal"` + CustomUA string `json:"custom_ua"` } var config = driver.Config{ diff --git a/drivers/wps/put.go b/drivers/wps/put.go new file mode 100644 index 000000000..94ba72e72 --- /dev/null +++ b/drivers/wps/put.go @@ -0,0 +1,315 @@ +package wps + +import ( + "bytes" + "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type countingWriter struct { + n *int64 +} + +func (w countingWriter) Write(p []byte) (int, error) { + *w.n += int64(len(p)) + return len(p), nil +} + +func cacheAndHash(file model.FileStreamer, up driver.UpdateProgress) (model.File, int64, string, string, error) { + h1 := sha1.New() + h256 := sha256.New() + size := file.GetSize() + var counted int64 + ws := []io.Writer{h1, h256} + if size <= 0 { + ws = append(ws, countingWriter{n: &counted}) + } + p := up + f, err := file.CacheFullAndWriter(&p, io.MultiWriter(ws...)) + if err != nil { + return nil, 0, "", "", err + } + if size <= 0 { + size = counted + } + return f, size, hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h256.Sum(nil)), nil +} + +func (d *Wps) createUpload(ctx context.Context, groupID, parentID int64, name string, size int64, sha1Hex, sha256Hex string) (*uploadCreateUpdateResp, error) { + body := map[string]string{ + "group_id": strconv.FormatInt(groupID, 10), + "name": name, + "parent_id": strconv.FormatInt(parentID, 10), + "sha1": sha1Hex, + "sha256": sha256Hex, + "size": strconv.FormatInt(size, 10), + } + var resp uploadCreateUpdateResp + r, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&resp). + SetError(&resp). + Put(d.driveURL("/api/v5/files/upload/create_update")) + if err != nil { + return nil, err + } + if err := checkAPI(r, resp.apiResult); err != nil { + return nil, err + } + if resp.URL == "" { + return nil, fmt.Errorf("empty upload url") + } + return &resp, nil +} + +func normalizeETag(v string) string { + v = strings.TrimSpace(v) + if strings.HasPrefix(v, "W/") { + v = strings.TrimSpace(strings.TrimPrefix(v, "W/")) + } + return strings.Trim(v, `"`) +} + +func (d *Wps) commitUpload(ctx context.Context, etag, key string, groupID, parentID int64, name, sha1Hex string, size int64, store string) error { + store = strings.TrimSpace(store) + if store == "" { + store = "ks3" + } + storeKey := "" + if key != "" { + storeKey = key + } + body := map[string]interface{}{ + "etag": etag, + "groupid": groupID, + "key": key, + "name": name, + "parentid": parentID, + "sha1": sha1Hex, + "size": size, + "store": store, + "storekey": storeKey, + } + return d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/file"), body) +} + +func (d *Wps) put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if dstDir == nil || file == nil { + return errs.NotSupport + } + if up == nil { + up = func(float64) {} + } + node, err := unwrapWpsObj(dstDir) + if err != nil { + return err + } + if node.Kind != "group" && node.Kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.HasFile && node.Kind == "folder" { + parentID = node.FileID + } + f, size, sha1Hex, sha256Hex, err := cacheAndHash(file, func(float64) {}) + if err != nil { + return err + } + if c, ok := f.(io.Closer); ok { + defer c.Close() + } + + // 在隐藏文件名前加_上传，这是WPS的限制，无法上传隐藏文件，也无法将任何文件重命名为隐藏文件，所有隐藏文件会被自动加上_ 上传 + // 甚至可以上传前缀是..的文件，但是单个点就是不行 + realName := file.GetName() + uploadName := realName + if strings.HasPrefix(realName, ".") { + uploadName = "_" + realName + } + + info, err := d.createUpload(ctx, node.GroupID, parentID, uploadName, size, sha1Hex, sha256Hex) + if err != nil { + return err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + rf := driver.NewLimitedUploadFile(ctx, f) + prog := driver.NewProgress(size, model.UpdateProgressWithRange(up, 0, 1)) + + method := strings.ToUpper(strings.TrimSpace(info.Method)) + if method == "" { + method = http.MethodPut + } + + var req *http.Request + if method == http.MethodPost && len(info.Request.FormData) > 0 { + if size == 0 { + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + return err + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + return err + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + return err + } + if err := mw.Close(); err != nil { + return err + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, bytes.NewReader(buf.Bytes())) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(buf.Len()) + req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10)) + } else { + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + req, err = http.NewRequestWithContext(ctx, method, info.URL, pr) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + go func() { + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + pw.CloseWithError(err) + return + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + pw.CloseWithError(err) + return + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + pw.CloseWithError(err) + return + } + if err := mw.Close(); err != nil { + pw.CloseWithError(err) + return + } + pw.Close() + }() + } + } else { + var body = io.TeeReader(rf, prog) + if size == 0 { + body = bytes.NewReader(nil) + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, body) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.ContentLength = size + req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) + } + + c := *d.client.GetClient() + c.Timeout = 0 + resp, err := (&c).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !statusOK(resp.StatusCode, info.Response.ExpectCode) { + io.Copy(io.Discard, resp.Body) + return fmt.Errorf("http error: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + etag := normalizeETag(respArg(info.Response.ArgsETag, resp, body)) + if etag == "" { + etag = normalizeETag(resp.Header.Get("ETag")) + } + + key := strings.TrimSpace(respArg(info.Response.ArgsKey, resp, body)) + if key == "" { + key = strings.TrimSpace(resp.Header.Get("x-obs-save-key")) + } + + var pr uploadPutResp + sha1FromServer := "" + if err := json.Unmarshal(body, &pr); err == nil { + sha1FromServer = strings.TrimSpace(pr.NewFilename) + if sha1FromServer == "" { + sha1FromServer = strings.TrimSpace(pr.Sha1) + } + if etag == "" && pr.MD5 != "" { + etag = strings.TrimSpace(pr.MD5) + } + } + + if sha1FromServer == "" { + if v := extractXMLTag(string(body), "ETag"); v != "" { + sha1FromServer = v + if etag == "" { + etag = v + } + } + } + if sha1FromServer == "" && key != "" && len(key) == 40 { + sha1FromServer = key + } + if sha1FromServer == "" { + sha1FromServer = sha1Hex + } + + if etag == "" { + return fmt.Errorf("empty etag") + } + if sha1FromServer == "" { + return fmt.Errorf("empty sha1") + } + + store := strings.TrimSpace(info.Store) + commitKey := "" + if strings.TrimSpace(info.Response.ArgsKey) != "" { + commitKey = key + if commitKey == "" { + commitKey = sha1FromServer + } + } + + if err := d.commitUpload(ctx, etag, commitKey, node.GroupID, parentID, uploadName, sha1FromServer, size, store); err != nil { + return err + } + + up(1) + return nil +} diff --git a/drivers/wps/types.go b/drivers/wps/types.go index a04df3d11..4a1b6d9e1 100644 --- a/drivers/wps/types.go +++ b/drivers/wps/types.go @@ -1,15 +1,24 @@ package wps import ( - "time" + "strconv" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/internal/model" ) -type workspaceResp struct { - Companies []struct { - ID int64 `json:"id"` - } `json:"companies"` +type apiResult struct { + Result string `json:"result"` + Msg string `json:"msg"` +} + +type loginState struct { + AccountNum int `json:"account_num"` + CompanyID int64 `json:"companyid"` + CurrentCompanyID int64 `json:"current_companyid"` + IsCompanyAccount bool `json:"is_company_account"` + IsPlus bool `json:"is_plus"` + LoginMode string `json:"loginmode"` + UserID int64 `json:"userid"` } type Group struct { @@ -23,6 +32,14 @@ type groupsResp struct { Groups []Group `json:"groups"` } +type personalGroupsResp struct { + apiResult + Groups []struct { + ID int64 `json:"id"` + Name string `json:"name"` + } `json:"groups"` +} + type filePerms struct { Download int `json:"download"` } @@ -40,6 +57,42 @@ type FileInfo struct { FilePerms filePerms `json:"file_perms_acl"` } +func (f *FileInfo) canDownload(isPersonal bool) bool { + if f == nil || f.Type == "folder" { + return false + } + if f.FilePerms.Download != 0 { + return true + } + return isPersonal +} + +func (f FileInfo) fileToObj(basePath string, isPersonal bool) *Obj { + name := f.Name + path := joinPath(basePath, name) + kind := "file" + if f.Type == "folder" { + kind = "folder" + } + obj := &Obj{ + Obj: &model.Object{ + ID: strconv.FormatInt(f.ID, 10), + Path: path, + Name: name, + Size: f.Size, + Modified: parseTime(f.Mtime), + Ctime: parseTime(f.Ctime), + IsFolder: f.Type == "folder", + }, + Kind: kind, + FileID: f.ID, + GroupID: f.GroupID, + HasFile: true, + CanDownload: f.canDownload(isPersonal), + } + return obj +} + type filesResp struct { Files []FileInfo `json:"files"` NextOffset int `json:"next_offset"` @@ -62,46 +115,33 @@ type spacesResp struct { } `json:"used_parts"` } -type Obj struct { - id string - name string - size int64 - ctime time.Time - mtime time.Time - isDir bool - hash utils.HashInfo - path string - canDownload bool +type uploadCreateUpdateResp struct { + apiResult + Method string `json:"method"` + URL string `json:"url"` + Store string `json:"store"` + Request struct { + Headers map[string]string `json:"headers"` + FormData map[string]string `json:"formData"` + } `json:"request"` + Response struct { + ExpectCode []int `json:"expect_code"` + ArgsETag string `json:"args_etag"` + ArgsKey string `json:"args_key"` + } `json:"response"` } -func (o *Obj) GetSize() int64 { - return o.size +type uploadPutResp struct { + NewFilename string `json:"newfilename"` + Sha1 string `json:"sha1"` + MD5 string `json:"md5"` } -func (o *Obj) GetName() string { - return o.name -} - -func (o *Obj) ModTime() time.Time { - return o.mtime -} - -func (o *Obj) CreateTime() time.Time { - return o.ctime -} - -func (o *Obj) IsDir() bool { - return o.isDir -} - -func (o *Obj) GetHash() utils.HashInfo { - return o.hash -} - -func (o *Obj) GetID() string { - return o.id -} - -func (o *Obj) GetPath() string { - return o.path +type Obj struct { + model.Obj + Kind string // root / group / file / folder + FileID int64 + GroupID int64 + HasFile bool // only FileInfo has file, otherwise the FileID is 0 + CanDownload bool } diff --git a/drivers/wps/util.go b/drivers/wps/util.go index 6f8f342da..5541406a2 100644 --- a/drivers/wps/util.go +++ b/drivers/wps/util.go @@ -1,102 +1,39 @@ package wps import ( - "bytes" "context" - "crypto/sha1" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" - "io" - "mime/multipart" "net/http" "strconv" "strings" - "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/go-resty/resty/v2" ) -const endpoint = "https://365.kdocs.cn" -const personalEndpoint = "https://drive.wps.cn" - -type resolvedNode struct { - kind string - group Group - file *FileInfo -} - -type resolveCacheEntry struct { - node *resolvedNode - expire time.Time -} - -type resolveCacheStore struct { - mu sync.RWMutex - m map[string]resolveCacheEntry -} - -var resolveCaches sync.Map - -type apiResult struct { - Result string `json:"result"` - Msg string `json:"msg"` -} - -type uploadCreateUpdateResp struct { - apiResult - Method string `json:"method"` - URL string `json:"url"` - Store string `json:"store"` - Request struct { - Headers map[string]string `json:"headers"` - FormData map[string]string `json:"formData"` - } `json:"request"` - Response struct { - ExpectCode []int `json:"expect_code"` - ArgsETag string `json:"args_etag"` - ArgsKey string `json:"args_key"` - } `json:"response"` -} - -type uploadPutResp struct { - NewFilename string `json:"newfilename"` - Sha1 string `json:"sha1"` - MD5 string `json:"md5"` -} - -type personalGroupsResp struct { - apiResult - Groups []struct { - ID int64 `json:"id"` - Name string `json:"name"` - } `json:"groups"` -} - -type countingWriter struct { - n *int64 -} - -func (w countingWriter) Write(p []byte) (int, error) { - *w.n += int64(len(p)) - return len(p), nil -} +const ENDPOINT_BUSINESS = "https://365.kdocs.cn" +const ENDPOINT_PERSONAL = "https://drive.wps.cn" func (d *Wps) isPersonal() bool { + // prefer d.login if available, as it may be set by islogin API + // which can determine account type more reliably + // one login session only support one type + // can not use personal and company account at the same time + if d.login != nil { + return !d.login.IsCompanyAccount + } return strings.TrimSpace(d.Mode) == "Personal" } func (d *Wps) driveHost() string { if d.isPersonal() { - return personalEndpoint + return ENDPOINT_PERSONAL } - return endpoint + return ENDPOINT_BUSINESS } func (d *Wps) drivePrefix() string { @@ -114,20 +51,18 @@ func (d *Wps) origin() string { return d.driveHost() } -func (d *Wps) canDownload(f *FileInfo) bool { - if f == nil || f.Type == "folder" { - return false +func (d *Wps) getUA() string { + if d.CustomUA != "" { + return d.CustomUA } - if f.FilePerms.Download != 0 { - return true - } - return d.isPersonal() + return base.UserAgent } func (d *Wps) request(ctx context.Context) *resty.Request { - return base.RestyClient.R(). + return d.client.R(). SetHeader("Cookie", d.Cookie). SetHeader("Accept", "application/json"). + SetHeader("User-Agent", d.getUA()). SetContext(ctx) } @@ -219,28 +154,6 @@ func checkAPI(resp *resty.Response, result apiResult) error { return nil } -func (d *Wps) ensureCompanyID(ctx context.Context) error { - if d.isPersonal() { - return nil - } - if d.companyID != "" { - return nil - } - var resp workspaceResp - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(endpoint + "/3rd/plussvr/compose/v1/users/self/workspaces?fields=name&comp_status=active") - if err != nil { - return err - } - if r != nil && r.IsError() { - return fmt.Errorf("http error: %d", r.StatusCode()) - } - if len(resp.Companies) == 0 { - return fmt.Errorf("no company id") - } - d.companyID = strconv.FormatInt(resp.Companies[0].ID, 10) - return nil -} - func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { if d.isPersonal() { var resp personalGroupsResp @@ -257,11 +170,8 @@ func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { } return res, nil } - if err := d.ensureCompanyID(ctx); err != nil { - return nil, err - } var resp groupsResp - url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%s/users/self/groups/private", endpoint, d.companyID) + url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%d/users/self/groups/private", ENDPOINT_BUSINESS, d.login.CompanyID) r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) if err != nil { return nil, err @@ -313,165 +223,6 @@ func joinPath(basePath, name string) string { return strings.TrimRight(basePath, "/") + "/" + name } -func normalizePath(path string) string { - clean := strings.TrimSpace(path) - if clean == "" || clean == "/" { - return "/" - } - return "/" + strings.Trim(clean, "/") -} - -func (d *Wps) resolveCacheStore() *resolveCacheStore { - if d == nil { - return nil - } - if v, ok := resolveCaches.Load(d); ok { - if s, ok := v.(*resolveCacheStore); ok { - return s - } - } - s := &resolveCacheStore{m: make(map[string]resolveCacheEntry)} - if v, loaded := resolveCaches.LoadOrStore(d, s); loaded { - if s2, ok := v.(*resolveCacheStore); ok { - return s2 - } - } - return s -} - -func (d *Wps) getResolveCache(path string) (*resolvedNode, bool) { - s := d.resolveCacheStore() - if s == nil { - return nil, false - } - s.mu.RLock() - e, ok := s.m[path] - s.mu.RUnlock() - if !ok || e.node == nil { - return nil, false - } - if !e.expire.IsZero() && time.Now().After(e.expire) { - s.mu.Lock() - delete(s.m, path) - s.mu.Unlock() - return nil, false - } - return e.node, true -} - -func (d *Wps) setResolveCache(path string, node *resolvedNode) { - s := d.resolveCacheStore() - if s == nil || node == nil { - return - } - s.mu.Lock() - s.m[path] = resolveCacheEntry{node: node, expire: time.Now().Add(10 * time.Minute)} - s.mu.Unlock() -} - -func (d *Wps) clearResolveCache() { - s := d.resolveCacheStore() - if s == nil { - return - } - s.mu.Lock() - if len(s.m) != 0 { - s.m = make(map[string]resolveCacheEntry) - } - s.mu.Unlock() -} - -func (d *Wps) resolvePath(ctx context.Context, path string) (*resolvedNode, error) { - cacheKey := normalizePath(path) - if n, ok := d.getResolveCache(cacheKey); ok { - return n, nil - } - clean := strings.TrimSpace(path) - if clean == "" { - clean = "/" - } - clean = strings.Trim(clean, "/") - if clean == "" { - n := &resolvedNode{kind: "root"} - d.setResolveCache("/", n) - return n, nil - } - seg := strings.Split(clean, "/") - groups, err := d.getGroups(ctx) - if err != nil { - return nil, err - } - var grp *Group - for i := range groups { - if groups[i].Name == seg[0] { - grp = &groups[i] - break - } - } - if grp == nil { - return nil, fmt.Errorf("group not found") - } - cur := "/" + seg[0] - gn := &resolvedNode{kind: "group", group: *grp} - d.setResolveCache(cur, gn) - if len(seg) == 1 { - return gn, nil - } - parentID := int64(0) - var lastNode *resolvedNode - for i := 1; i < len(seg); i++ { - files, err := d.getFiles(ctx, grp.GroupID, parentID) - if err != nil { - return nil, err - } - var found *FileInfo - for j := range files { - if files[j].Name == seg[i] { - found = &files[j] - break - } - } - if found == nil { - return nil, fmt.Errorf("path not found") - } - if i < len(seg)-1 && found.Type != "folder" { - return nil, fmt.Errorf("path not found") - } - fi := *found - parentID = fi.ID - cur = cur + "/" + seg[i] - kind := "file" - if fi.Type == "folder" { - kind = "folder" - } - n := &resolvedNode{kind: kind, group: *grp, file: &fi} - d.setResolveCache(cur, n) - lastNode = n - } - if lastNode == nil { - return nil, fmt.Errorf("path not found") - } - return lastNode, nil -} - -func (d *Wps) fileToObj(basePath string, f FileInfo) *Obj { - name := f.Name - path := joinPath(basePath, name) - obj := &Obj{ - id: path, - name: name, - size: f.Size, - ctime: parseTime(f.Ctime), - mtime: parseTime(f.Mtime), - isDir: f.Type == "folder", - path: path, - } - if !obj.isDir { - obj.canDownload = d.canDownload(&f) - } - return obj -} - func (d *Wps) doJSON(ctx context.Context, method, url string, body interface{}) error { var result apiResult req := d.jsonRequest(ctx).SetBody(body).SetResult(&result).SetError(&result) @@ -493,580 +244,16 @@ func (d *Wps) doJSON(ctx context.Context, method, url string, body interface{}) return checkAPI(resp, result) } -func (d *Wps) list(ctx context.Context, basePath string) ([]model.Obj, error) { - if strings.TrimSpace(basePath) == "" { - basePath = "/" - } - node, err := d.resolvePath(ctx, basePath) - if err != nil { - return nil, err - } - if node.kind == "root" { - groups, err := d.getGroups(ctx) - if err != nil { - return nil, err - } - res := make([]model.Obj, 0, len(groups)) - for _, g := range groups { - path := joinPath(basePath, g.Name) - obj := &Obj{ - id: path, - name: g.Name, - ctime: parseTime(0), - mtime: parseTime(0), - isDir: true, - path: path, - } - res = append(res, obj) - d.setResolveCache(normalizePath(path), &resolvedNode{kind: "group", group: g}) - } - d.setResolveCache("/", &resolvedNode{kind: "root"}) - return res, nil - } - if node.kind != "group" && node.kind != "folder" { - return nil, nil - } - parentID := int64(0) - if node.file != nil && node.kind == "folder" { - parentID = node.file.ID - } - files, err := d.getFiles(ctx, node.group.GroupID, parentID) - if err != nil { - return nil, err - } - res := make([]model.Obj, 0, len(files)) - for _, f := range files { - res = append(res, d.fileToObj(basePath, f)) - path := normalizePath(joinPath(basePath, f.Name)) - fi := f - kind := "file" - if fi.Type == "folder" { - kind = "folder" - } - d.setResolveCache(path, &resolvedNode{kind: kind, group: node.group, file: &fi}) - } - return res, nil -} - -func (d *Wps) link(ctx context.Context, path string) (*model.Link, error) { - node, err := d.resolvePath(ctx, path) - if err != nil { - return nil, err - } - if node.kind != "file" || node.file == nil { - return nil, errs.NotSupport - } - if !d.canDownload(node.file) { - return nil, fmt.Errorf("no download permission") - } - url := fmt.Sprintf("%s/api/v5/groups/%d/files/%d/download?support_checksums=sha1", d.driveHost()+d.drivePrefix(), node.group.GroupID, node.file.ID) - var resp downloadResp - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) - } - if resp.URL == "" { - return nil, fmt.Errorf("empty download url") - } - return &model.Link{URL: resp.URL, Header: http.Header{}}, nil -} - -func (d *Wps) makeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - if parentDir == nil { - return errs.NotSupport - } - node, err := d.resolvePath(ctx, parentDir.GetPath()) - if err != nil { - return err - } - if node.kind != "group" && node.kind != "folder" { - return errs.NotSupport - } - parentID := int64(0) - if node.file != nil && node.kind == "folder" { - parentID = node.file.ID - } - body := map[string]interface{}{ - "groupid": node.group.GroupID, - "name": dirName, - "parentid": parentID, - } - if err := d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/folder"), body); err != nil { - return err - } - d.clearResolveCache() - return nil -} - -func (d *Wps) move(ctx context.Context, srcObj, dstDir model.Obj) error { - if srcObj == nil || dstDir == nil { - return errs.NotSupport - } - nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) - if err != nil { - return err - } - nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) - if err != nil { - return err - } - if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { - return errs.NotSupport - } - if nodeDst.kind != "group" && nodeDst.kind != "folder" { - return errs.NotSupport - } - targetParentID := int64(0) - if nodeDst.file != nil && nodeDst.kind == "folder" { - targetParentID = nodeDst.file.ID - } - body := map[string]interface{}{ - "fileids": []int64{nodeSrc.file.ID}, - "target_groupid": nodeDst.group.GroupID, - "target_parentid": targetParentID, - } - url := fmt.Sprintf("/api/v3/groups/%d/files/batch/move", nodeSrc.group.GroupID) - for { - var res apiResult - resp, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&res). - SetError(&res). - Post(d.driveURL(url)) - if err != nil { - return err - } - - if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { - time.Sleep(500 * time.Millisecond) - continue - } - - if err := checkAPI(resp, res); err != nil { - return err - } - break - } - d.clearResolveCache() - return nil -} - -func (d *Wps) rename(ctx context.Context, srcObj model.Obj, newName string) error { - if srcObj == nil { - return errs.NotSupport - } - node, err := d.resolvePath(ctx, srcObj.GetPath()) - if err != nil { - return err - } - if node.kind != "file" && node.kind != "folder" { - return errs.NotSupport - } - url := fmt.Sprintf("/api/v3/groups/%d/files/%d", node.group.GroupID, node.file.ID) - body := map[string]string{"fname": newName} - if err := d.doJSON(ctx, http.MethodPut, d.driveURL(url), body); err != nil { - return err - } - d.clearResolveCache() - return nil -} - -func (d *Wps) copy(ctx context.Context, srcObj, dstDir model.Obj) error { - if srcObj == nil || dstDir == nil { - return errs.NotSupport - } - nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) - if err != nil { - return err - } - nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) - if err != nil { - return err - } - if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { - return errs.NotSupport - } - if nodeDst.kind != "group" && nodeDst.kind != "folder" { - return errs.NotSupport - } - targetParentID := int64(0) - if nodeDst.file != nil && nodeDst.kind == "folder" { - targetParentID = nodeDst.file.ID - } - body := map[string]interface{}{ - "fileids": []int64{nodeSrc.file.ID}, - "groupid": nodeSrc.group.GroupID, - "target_groupid": nodeDst.group.GroupID, - "target_parentid": targetParentID, - "duplicated_name_model": 1, - } - url := fmt.Sprintf("/api/v3/groups/%d/files/batch/copy", nodeSrc.group.GroupID) - for { - var res apiResult - resp, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&res). - SetError(&res). - Post(d.driveURL(url)) - if err != nil { - return err - } - - if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { - time.Sleep(500 * time.Millisecond) - continue - } - - if err := checkAPI(resp, res); err != nil { - return err - } - break - } - d.clearResolveCache() - return nil -} - -func (d *Wps) remove(ctx context.Context, obj model.Obj) error { - if obj == nil { - return errs.NotSupport - } - node, err := d.resolvePath(ctx, obj.GetPath()) - if err != nil { - return err - } - if node.kind != "file" && node.kind != "folder" { - return errs.NotSupport - } - - body := map[string]interface{}{ - "fileids": []int64{node.file.ID}, - } - url := fmt.Sprintf("/api/v3/groups/%d/files/batch/delete", node.group.GroupID) - - for { - var res apiResult - resp, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&res). - SetError(&res). - Post(d.driveURL(url)) - if err != nil { - return err - } - - // 无法连续创建文件夹删除。如果一定要删除，每0.5s 尝试一次创建下一个删除请求，应当避免递归删除文件夹 - if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { - time.Sleep(500 * time.Millisecond) - continue - } - - if err := checkAPI(resp, res); err != nil { - return err - } - break - } - d.clearResolveCache() - return nil -} - -func cacheAndHash(file model.FileStreamer, up driver.UpdateProgress) (model.File, int64, string, string, error) { - h1 := sha1.New() - h256 := sha256.New() - size := file.GetSize() - var counted int64 - ws := []io.Writer{h1, h256} - if size <= 0 { - ws = append(ws, countingWriter{n: &counted}) - } - p := up - f, err := file.CacheFullAndWriter(&p, io.MultiWriter(ws...)) - if err != nil { - return nil, 0, "", "", err - } - if size <= 0 { - size = counted - } - return f, size, hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h256.Sum(nil)), nil -} - -func (d *Wps) createUpload(ctx context.Context, groupID, parentID int64, name string, size int64, sha1Hex, sha256Hex string) (*uploadCreateUpdateResp, error) { - body := map[string]string{ - "group_id": strconv.FormatInt(groupID, 10), - "name": name, - "parent_id": strconv.FormatInt(parentID, 10), - "sha1": sha1Hex, - "sha256": sha256Hex, - "size": strconv.FormatInt(size, 10), - } - var resp uploadCreateUpdateResp - r, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&resp). - SetError(&resp). - Put(d.driveURL("/api/v5/files/upload/create_update")) - if err != nil { - return nil, err - } - if err := checkAPI(r, resp.apiResult); err != nil { - return nil, err - } - if resp.URL == "" { - return nil, fmt.Errorf("empty upload url") - } - return &resp, nil -} - -func normalizeETag(v string) string { - v = strings.TrimSpace(v) - if strings.HasPrefix(v, "W/") { - v = strings.TrimSpace(strings.TrimPrefix(v, "W/")) - } - return strings.Trim(v, `"`) -} - -func (d *Wps) commitUpload(ctx context.Context, etag, key string, groupID, parentID int64, name, sha1Hex string, size int64, store string) error { - store = strings.TrimSpace(store) - if store == "" { - store = "ks3" - } - storeKey := "" - if key != "" { - storeKey = key - } - body := map[string]interface{}{ - "etag": etag, - "groupid": groupID, - "key": key, - "name": name, - "parentid": parentID, - "sha1": sha1Hex, - "size": size, - "store": store, - "storekey": storeKey, - } - return d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/file"), body) -} - -func (d *Wps) put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { - if dstDir == nil || file == nil { - return errs.NotSupport - } - if up == nil { - up = func(float64) {} - } - node, err := d.resolvePath(ctx, dstDir.GetPath()) - if err != nil { - return err - } - if node.kind != "group" && node.kind != "folder" { - return errs.NotSupport - } - parentID := int64(0) - if node.file != nil && node.kind == "folder" { - parentID = node.file.ID - } - f, size, sha1Hex, sha256Hex, err := cacheAndHash(file, func(float64) {}) - if err != nil { - return err - } - if c, ok := f.(io.Closer); ok { - defer c.Close() - } - - // 在隐藏文件名前加_上传，这是WPS的限制，无法上传隐藏文件，也无法将任何文件重命名为隐藏文件，所有隐藏文件会被自动加上_ 上传 - // 甚至可以上传前缀是..的文件，但是单个点就是不行 - realName := file.GetName() - uploadName := realName - if strings.HasPrefix(realName, ".") { - uploadName = "_" + realName - } - - info, err := d.createUpload(ctx, node.group.GroupID, parentID, uploadName, size, sha1Hex, sha256Hex) - if err != nil { - return err - } - if _, err := f.Seek(0, io.SeekStart); err != nil { - return err - } - rf := driver.NewLimitedUploadFile(ctx, f) - prog := driver.NewProgress(size, model.UpdateProgressWithRange(up, 0, 1)) - - method := strings.ToUpper(strings.TrimSpace(info.Method)) - if method == "" { - method = http.MethodPut - } - - var req *http.Request - if method == http.MethodPost && len(info.Request.FormData) > 0 { - if size == 0 { - var buf bytes.Buffer - mw := multipart.NewWriter(&buf) - for k, v := range info.Request.FormData { - if err := mw.WriteField(k, v); err != nil { - return err - } - } - part, err := mw.CreateFormFile("file", uploadName) - if err != nil { - return err - } - if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { - return err - } - if err := mw.Close(); err != nil { - return err - } - req, err = http.NewRequestWithContext(ctx, method, info.URL, bytes.NewReader(buf.Bytes())) - if err != nil { - return err - } - for k, v := range info.Request.Headers { - req.Header.Set(k, v) - } - req.Header.Set("Content-Type", mw.FormDataContentType()) - req.ContentLength = int64(buf.Len()) - req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10)) - } else { - pr, pw := io.Pipe() - mw := multipart.NewWriter(pw) - req, err = http.NewRequestWithContext(ctx, method, info.URL, pr) - if err != nil { - return err - } - for k, v := range info.Request.Headers { - req.Header.Set(k, v) - } - req.Header.Set("Content-Type", mw.FormDataContentType()) - go func() { - for k, v := range info.Request.FormData { - if err := mw.WriteField(k, v); err != nil { - pw.CloseWithError(err) - return - } - } - part, err := mw.CreateFormFile("file", uploadName) - if err != nil { - pw.CloseWithError(err) - return - } - if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { - pw.CloseWithError(err) - return - } - if err := mw.Close(); err != nil { - pw.CloseWithError(err) - return - } - pw.Close() - }() - } - } else { - var body = io.TeeReader(rf, prog) - if size == 0 { - body = bytes.NewReader(nil) - } - req, err = http.NewRequestWithContext(ctx, method, info.URL, body) - if err != nil { - return err - } - for k, v := range info.Request.Headers { - req.Header.Set(k, v) +func unwrapWpsObj(obj model.Obj) (*Obj, error) { + for obj != nil { + if node, ok := obj.(*Obj); ok { + return node, nil } - req.ContentLength = size - req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) - } - - c := *base.RestyClient.GetClient() - c.Timeout = 0 - resp, err := (&c).Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if !statusOK(resp.StatusCode, info.Response.ExpectCode) { - io.Copy(io.Discard, resp.Body) - return fmt.Errorf("http error: %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - - etag := normalizeETag(respArg(info.Response.ArgsETag, resp, body)) - if etag == "" { - etag = normalizeETag(resp.Header.Get("ETag")) - } - - key := strings.TrimSpace(respArg(info.Response.ArgsKey, resp, body)) - if key == "" { - key = strings.TrimSpace(resp.Header.Get("x-obs-save-key")) - } - - var pr uploadPutResp - sha1FromServer := "" - if err := json.Unmarshal(body, &pr); err == nil { - sha1FromServer = strings.TrimSpace(pr.NewFilename) - if sha1FromServer == "" { - sha1FromServer = strings.TrimSpace(pr.Sha1) - } - if etag == "" && pr.MD5 != "" { - etag = strings.TrimSpace(pr.MD5) - } - } - - if sha1FromServer == "" { - if v := extractXMLTag(string(body), "ETag"); v != "" { - sha1FromServer = v - if etag == "" { - etag = v - } - } - } - if sha1FromServer == "" && key != "" && len(key) == 40 { - sha1FromServer = key - } - if sha1FromServer == "" { - sha1FromServer = sha1Hex - } - - if etag == "" { - return fmt.Errorf("empty etag") - } - if sha1FromServer == "" { - return fmt.Errorf("empty sha1") - } - - store := strings.TrimSpace(info.Store) - commitKey := "" - if strings.TrimSpace(info.Response.ArgsKey) != "" { - commitKey = key - if commitKey == "" { - commitKey = sha1FromServer + unwrap, ok := obj.(model.ObjUnwrap) + if !ok { + break } + obj = unwrap.Unwrap() } - - if err := d.commitUpload(ctx, etag, commitKey, node.group.GroupID, parentID, uploadName, sha1FromServer, size, store); err != nil { - return err - } - - up(1) - return nil -} - -func (d *Wps) spaces(ctx context.Context) (*spacesResp, error) { - url := fmt.Sprintf("%s/api/v3/spaces", d.driveHost()+d.drivePrefix()) - var resp spacesResp - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) - } - return &resp, nil + return nil, fmt.Errorf("invalid object type") } From 376638978a345103813d79ca132caa703d9c4d0a Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 27 Apr 2026 21:13:44 +0800 Subject: [PATCH 77/86] fix(drivers/wps): implement driver.GetRooter interface (#2414) * fix(drivers/wps): implement driver.GetRooter interface * fix(drivers/wps): add User-Agent and Referer headers in Link method * fix(drivers/wps): removing NoOverwriteUpload field --------- Signed-off-by: MadDogOwner --- drivers/wps/driver.go | 102 ++++++++++++++++++++++++++++++------------ drivers/wps/meta.go | 9 ++-- drivers/wps/types.go | 13 ++++++ 3 files changed, 90 insertions(+), 34 deletions(-) diff --git a/drivers/wps/driver.go b/drivers/wps/driver.go index 6f78c0b10..12bf0c537 100644 --- a/drivers/wps/driver.go +++ b/drivers/wps/driver.go @@ -4,13 +4,14 @@ import ( "context" "fmt" "net/http" - "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -59,41 +60,79 @@ func (d *Wps) Drop(ctx context.Context) error { return nil } -func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]model.Obj, error) { - basePath := "/" - if dir != nil { - if p := dir.GetPath(); p != "" { - basePath = p - } +func (d *Wps) GetRoot(ctx context.Context) (model.Obj, error) { + root := &Obj{ + Obj: &model.Object{ + Path: "/", + Name: "root", + Modified: d.Modified, + Ctime: d.Modified, + IsFolder: true, + }, + Kind: "root", } - if basePath == "/" { + rootPath := d.RootFolderPath + if rootPath != "" && rootPath != "/" { + parts := strings.Split(strings.Trim(rootPath, "/"), "/") groups, err := d.getGroups(ctx) if err != nil { return nil, err } - res := make([]model.Obj, 0, len(groups)) + var current *Obj for _, g := range groups { - path := joinPath(basePath, g.Name) - obj := &Obj{ - Obj: &model.Object{ - ID: strconv.FormatInt(g.GroupID, 10), - Path: path, - Name: g.Name, - Modified: parseTime(0), - Ctime: parseTime(0), - IsFolder: true, - }, - Kind: "group", - GroupID: g.GroupID, + if g.Name == parts[0] { + current = g.groupToObj("/") + break } - res = append(res, obj) } - return res, nil + if current == nil { + return nil, fmt.Errorf("root path %q not found", rootPath) + } + parentID := int64(0) + for _, name := range parts[1:] { + files, err := d.getFiles(ctx, current.GroupID, parentID) + if err != nil { + return nil, err + } + var next *Obj + for _, f := range files { + if f.Type == "folder" && f.Name == name { + next = f.fileToObj(current.GetPath(), d.isPersonal()) + break + } + } + if next == nil { + return nil, fmt.Errorf("root path %q not found", rootPath) + } + current = next + parentID = current.FileID + } + current.Obj = &model.Object{ID: current.GetID(), Path: "/", Name: current.GetName(), IsFolder: true} + root = current + } + return root, nil +} + +func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]model.Obj, error) { + basePath := "/" + if dir != nil { + if p := dir.GetPath(); p != "" { + basePath = p + } } node, err := unwrapWpsObj(dir) if err != nil { return nil, err } + if node.Kind == "root" { + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + return utils.SliceConvert(groups, func(g Group) (model.Obj, error) { + return g.groupToObj(basePath), nil + }) + } if node.Kind != "group" && node.Kind != "folder" { return nil, nil } @@ -105,11 +144,9 @@ func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]mode if err != nil { return nil, err } - res := make([]model.Obj, 0, len(files)) - for _, f := range files { - res = append(res, f.fileToObj(basePath, d.isPersonal())) - } - return res, nil + return utils.SliceConvert(files, func(f FileInfo) (model.Obj, error) { + return f.fileToObj(basePath, d.isPersonal()), nil + }) } func (d *Wps) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { @@ -138,7 +175,13 @@ func (d *Wps) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*mode if resp.URL == "" { return nil, fmt.Errorf("empty download url") } - return &model.Link{URL: resp.URL, Header: http.Header{}}, nil + return &model.Link{ + URL: resp.URL, + Header: http.Header{ + "User-Agent": []string{d.getUA()}, + "Referer": []string{d.driveHost()}, + }, + }, nil } func (d *Wps) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { @@ -357,3 +400,4 @@ func (d *Wps) GetDetails(ctx context.Context) (*model.StorageDetails, error) { } var _ driver.Driver = (*Wps)(nil) +var _ driver.GetRooter = (*Wps)(nil) diff --git a/drivers/wps/meta.go b/drivers/wps/meta.go index a1fb79485..a520bb433 100644 --- a/drivers/wps/meta.go +++ b/drivers/wps/meta.go @@ -13,11 +13,10 @@ type Addition struct { } var config = driver.Config{ - Name: "WPS", - LocalSort: true, - DefaultRoot: "/", - Alert: "", - NoOverwriteUpload: true, + Name: "WPS", + LocalSort: true, + DefaultRoot: "/", + Alert: "", } func init() { diff --git a/drivers/wps/types.go b/drivers/wps/types.go index 4a1b6d9e1..1890976f1 100644 --- a/drivers/wps/types.go +++ b/drivers/wps/types.go @@ -93,6 +93,19 @@ func (f FileInfo) fileToObj(basePath string, isPersonal bool) *Obj { return obj } +func (g Group) groupToObj(basePath string) *Obj { + return &Obj{ + Obj: &model.Object{ + ID: strconv.FormatInt(g.GroupID, 10), + Path: joinPath(basePath, g.Name), + Name: g.Name, + IsFolder: true, + }, + Kind: "group", + GroupID: g.GroupID, + } +} + type filesResp struct { Files []FileInfo `json:"files"` NextOffset int `json:"next_offset"` From 2d2d9ae2132178e82951429c3dd9aacd91e8b0ff Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 27 Apr 2026 22:19:03 +0800 Subject: [PATCH 78/86] fix(drivers/wps): correct account relevant handling (#2415) * fix(drivers/wps): correct account modes handling Signed-off-by: MadDogOwner * feat(drivers/wps): enhance GetDetails for business account Co-authored-by: Copilot Signed-off-by: MadDogOwner --------- Signed-off-by: MadDogOwner Co-authored-by: Copilot --- drivers/wps/driver.go | 42 ++++++++++++++++++++++++++++++++++-------- drivers/wps/types.go | 8 ++++++++ drivers/wps/util.go | 26 +++++++++++++++----------- 3 files changed, 57 insertions(+), 19 deletions(-) diff --git a/drivers/wps/driver.go b/drivers/wps/driver.go index 12bf0c537..847425b11 100644 --- a/drivers/wps/driver.go +++ b/drivers/wps/driver.go @@ -382,8 +382,25 @@ func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer } func (d *Wps) GetDetails(ctx context.Context) (*model.StorageDetails, error) { - url := fmt.Sprintf("%s/api/v3/spaces", d.driveHost()+d.drivePrefix()) - var resp spacesResp + if d.isPersonal() { + url := ENDPOINT_PERSONAL + "/api/v3/spaces" + var resp spacesResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: resp.Total, + UsedSpace: resp.Used, + }, + }, nil + } + url := ENDPOINT_BUSINESS + "/3rd/plussvr/compose/v1/u/companies/batch/service-space?comp_ids=" + fmt.Sprint(d.login.CompanyID) + var resp serviceSpaceResp r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) if err != nil { return nil, err @@ -391,12 +408,21 @@ func (d *Wps) GetDetails(ctx context.Context) (*model.StorageDetails, error) { if r != nil && r.IsError() { return nil, fmt.Errorf("http error: %d", r.StatusCode()) } - return &model.StorageDetails{ - DiskUsage: model.DiskUsage{ - TotalSpace: resp.Total, - UsedSpace: resp.Used, - }, - }, nil + if len(resp.Info) == 0 { + return nil, fmt.Errorf("empty service space info") + } + // info := resp.Info[0] + for _, info := range resp.Info { + if info.ID == d.login.CompanyID { + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: info.SpaceTotal, + UsedSpace: info.SpaceUsed, + }, + }, nil + } + } + return nil, fmt.Errorf("service space info not found for company ID: %d", d.login.CompanyID) } var _ driver.Driver = (*Wps)(nil) diff --git a/drivers/wps/types.go b/drivers/wps/types.go index 1890976f1..d5329808f 100644 --- a/drivers/wps/types.go +++ b/drivers/wps/types.go @@ -128,6 +128,14 @@ type spacesResp struct { } `json:"used_parts"` } +type serviceSpaceResp struct { + Info []struct { + ID int64 `json:"id"` + SpaceTotal int64 `json:"space_total"` + SpaceUsed int64 `json:"space_used"` + } `json:"info"` +} + type uploadCreateUpdateResp struct { apiResult Method string `json:"method"` diff --git a/drivers/wps/util.go b/drivers/wps/util.go index 5541406a2..cb6e84d66 100644 --- a/drivers/wps/util.go +++ b/drivers/wps/util.go @@ -155,7 +155,9 @@ func checkAPI(resp *resty.Response, result apiResult) error { } func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { - if d.isPersonal() { + // different APIs + switch d.Mode { + case "Personal": var resp personalGroupsResp r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(d.driveURL("/api/v3/groups")) if err != nil { @@ -169,17 +171,19 @@ func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { res = append(res, Group{GroupID: g.ID, Name: g.Name}) } return res, nil + case "Business": + var resp groupsResp + url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%d/users/self/groups/private", ENDPOINT_BUSINESS, d.login.CompanyID) + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return resp.Groups, nil } - var resp groupsResp - url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%d/users/self/groups/private", ENDPOINT_BUSINESS, d.login.CompanyID) - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) - } - return resp.Groups, nil + return nil, fmt.Errorf("unsupported mode: %s", d.Mode) } func (d *Wps) getFiles(ctx context.Context, groupID, parentID int64) ([]FileInfo, error) { From c356bb4128e1df63c44156ec569595b052660543 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:16 +0800 Subject: [PATCH 79/86] feat(stream): link refresh, self-healing reader, seekable prefetch, and upload hash rework - Add link refresh capability with RefreshableRangeReader for expired download links - Implement self-healing reader for interrupted stream reconnection - Add seekable prefetch window with strict tests - Enhance hash calculation and upload logic for various stream types - Fix link expiry detection to avoid treating context cancellation as expired - Optimize link cache logic, remove SyncClosers dependency - Support all 4xx client errors in expired link check - Fix local file 'file already closed' error - Unify hash calculation progress weight to 100 - Add slow network support with adjusted timeout and retry --- internal/model/args.go | 8 + internal/net/serve.go | 11 +- internal/op/fs.go | 46 +- internal/op/storage.go | 5 +- .../stream/section_reader_prefetch_test.go | 320 ++++++++++ internal/stream/stream.go | 44 +- internal/stream/util.go | 604 ++++++++++++++++-- internal/stream/util_test.go | 160 +++++ pkg/utils/hash.go | 6 + server/handles/fsup.go | 12 + 10 files changed, 1140 insertions(+), 76 deletions(-) create mode 100644 internal/stream/section_reader_prefetch_test.go create mode 100644 internal/stream/util_test.go diff --git a/internal/model/args.go b/internal/model/args.go index 073c94a63..d165908fb 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -25,6 +25,10 @@ type LinkArgs struct { Redirect bool } +// LinkRefresher is a callback function type for refreshing download links +// It returns a new Link and the associated object, or an error +type LinkRefresher func(ctx context.Context) (*Link, Obj, error) + type Link struct { URL string `json:"url"` // most common way Header http.Header `json:"header"` // needed header (for url) @@ -37,6 +41,10 @@ type Link struct { PartSize int `json:"part_size"` ContentLength int64 `json:"content_length"` // 转码视频、缩略图 + // Refresher is a callback to refresh the link when it expires during long downloads + // This field is not serialized and is optional - if nil, no refresh will be attempted + Refresher LinkRefresher `json:"-"` + utils.SyncClosers `json:"-"` // 如果SyncClosers中的资源被关闭后Link将不可用，则此值应为 true RequireReference bool `json:"-"` diff --git a/internal/net/serve.go b/internal/net/serve.go index 6a20460b1..ee288b86a 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "mime/multipart" + stdnet "net" // 标准库net包，用于Dialer "net/http" "strconv" "strings" @@ -286,12 +287,20 @@ func NewHttpClient() *http.Client { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + // 快速连接超时：10秒建立连接，失败快速重试 + DialContext: (&stdnet.Dialer{ + Timeout: 10 * time.Second, // TCP握手超时 + KeepAlive: 30 * time.Second, // TCP keep-alive + }).DialContext, + // 响应头超时：15秒等待服务器响应头（平衡API调用与下载检测） + ResponseHeaderTimeout: 15 * time.Second, + // 允许长时间读取数据（无 IdleConnTimeout 限制） } SetProxyIfConfigured(transport) return &http.Client{ - Timeout: time.Hour * 48, + Timeout: time.Hour * 48, // 总超时保持48小时（允许大文件慢速下载） Transport: transport, } } diff --git a/internal/op/fs.go b/internal/op/fs.go index 5116bbef5..534e73697 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -150,6 +150,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, IsFolder: true, Mask: model.Locked, + HashInfo: utils.NewHashInfo(nil, ""), }, nil case driver.IRootPath: return &model.Object{ @@ -158,6 +159,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, Mask: model.Locked, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), }, nil } return nil, errors.New("please implement GetRooter or IRootPath or IRootId interface") @@ -247,6 +249,8 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { return ol.link, ol.obj, nil } + // SyncClosers 已关闭（文件句柄已关闭），删除缓存条目，重新获取 + Cache.linkCache.DeleteKey(key) } fn := func() (*objWithLink, error) { @@ -262,11 +266,37 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil { return nil, errors.Wrapf(err, "failed get link") } + + // Set up link refresher for automatic refresh on expiry during long downloads + // This enables all download scenarios to handle link expiration gracefully + if link.Refresher == nil { + storageCopy := storage + pathCopy := path + argsCopy := args + link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + log.Infof("Refreshing download link for: %s", pathCopy) + // Get fresh link directly from storage, bypassing cache + file, err := GetUnwrap(refreshCtx, storageCopy, pathCopy) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file for refresh") + } + newLink, err := storageCopy.Link(refreshCtx, file, argsCopy) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to refresh link") + } + return newLink, file, nil + } + } + ol := &objWithLink{link: link, obj: file} if link.Expiration != nil { Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) - } else { + } else if link.RequireReference { + // 本地文件等需要引用计数的链接，缓存与文件句柄生命周期绑定 Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) + } else { + // 不需要引用计数（如云盘链接无过期时间），使用默认 TTL，多客户端复用 + Cache.linkCache.SetType(key, typeKey, ol) } return ol, nil } @@ -326,9 +356,16 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) } parentDir, err := GetUnwrap(ctx, storage, parentPath) - // this should not happen if err != nil { - return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + if errs.IsObjectNotFound(err) { + // Retry once after a short delay (handles cloud storage API sync delay) + log.Debugf("[op] parent dir [%s] not found immediately after creation, retrying...", parentPath) + time.Sleep(100 * time.Millisecond) + parentDir, err = GetUnwrap(ctx, storage, parentPath) + } + if err != nil { + return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } } if model.ObjHasMask(parentDir, model.NoWrite) { return nil, errors.WithStack(errs.PermissionDenied) @@ -358,6 +395,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } dirCache.UpdateObject("", wrapObjName(storage, newObj)) @@ -682,6 +720,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod Modified: file.ModTime(), Ctime: file.CreateTime(), Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) @@ -750,6 +789,7 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) diff --git a/internal/op/storage.go b/internal/op/storage.go index da4c84e31..2e93bf569 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -368,7 +368,9 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, }(d) select { case r := <-resultChan: - ret.StorageDetails = r + if r != nil { + ret.StorageDetails = r + } case <-time.After(time.Second): } return ret @@ -419,6 +421,7 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver Name: name, Modified: v.GetStorage().Modified, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), } if !found { idx := len(files) diff --git a/internal/stream/section_reader_prefetch_test.go b/internal/stream/section_reader_prefetch_test.go new file mode 100644 index 000000000..c4a4fb66f --- /dev/null +++ b/internal/stream/section_reader_prefetch_test.go @@ -0,0 +1,320 @@ +package stream + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestDirectSectionReader_SeekablePrefetchPipeline(t *testing.T) { + data := []byte("abcdefgh") + + var ( + mu sync.Mutex + calls []http_range.Range + ) + secondStarted := make(chan struct{}) + releaseSecond := make(chan struct{}) + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + mu.Lock() + calls = append(calls, r) + mu.Unlock() + + if r.Start == 4 { + select { + case <-secondStarted: + default: + close(secondStarted) + } + <-releaseSecond + } + + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, context.Background(), data, rr) + ss, err := NewStreamSectionReader(seekable, 4, nil) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + b1, err := io.ReadAll(rd1) + if err != nil { + t.Fatalf("ReadAll first chunk error = %v", err) + } + if !bytes.Equal(b1, data[0:4]) { + t.Fatalf("first chunk = %q, want %q", b1, data[0:4]) + } + ss.FreeSectionReader(rd1) + + select { + case <-secondStarted: + case <-time.After(2 * time.Second): + t.Fatalf("prefetch for second chunk did not start asynchronously") + } + + type secondResult struct { + data []byte + err error + } + resCh := make(chan secondResult, 1) + go func() { + rd2, err := ss.GetSectionReader(4, 4) + if err != nil { + resCh <- secondResult{err: err} + return + } + b2, err := io.ReadAll(rd2) + ss.FreeSectionReader(rd2) + resCh <- secondResult{data: b2, err: err} + }() + + select { + case res := <-resCh: + t.Fatalf("second GetSectionReader returned too early: err=%v data=%q", res.err, res.data) + case <-time.After(100 * time.Millisecond): + // expected: wait prefetch completion + } + + close(releaseSecond) + res := <-resCh + if res.err != nil { + t.Fatalf("GetSectionReader(4,4) error = %v", res.err) + } + if !bytes.Equal(res.data, data[4:8]) { + t.Fatalf("second chunk = %q, want %q", res.data, data[4:8]) + } + + mu.Lock() + defer mu.Unlock() + if len(calls) != 2 { + t.Fatalf("RangeRead call count = %d, want 2", len(calls)) + } + if calls[0].Start != 0 || calls[1].Start != 4 { + t.Fatalf("RangeRead starts = [%d, %d], want [0, 4]", calls[0].Start, calls[1].Start) + } +} + +func TestDirectSectionReader_SeekablePrefetchMissFallsBackToSyncRead(t *testing.T) { + data := []byte("abcdefghijkl") + + var ( + mu sync.Mutex + calls []http_range.Range + ) + prefetchStarted := make(chan struct{}) + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + mu.Lock() + calls = append(calls, r) + mu.Unlock() + + if r.Start == 4 { + select { + case <-prefetchStarted: + default: + close(prefetchStarted) + } + } + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, context.Background(), data, rr) + ss, err := NewStreamSectionReader(seekable, 4, nil) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + _, _ = io.ReadAll(rd1) + ss.FreeSectionReader(rd1) + + select { + case <-prefetchStarted: + case <-time.After(2 * time.Second): + t.Fatalf("prefetch for second chunk did not start") + } + + rd3, err := ss.GetSectionReader(8, 4) + if err != nil { + t.Fatalf("GetSectionReader(8,4) error = %v", err) + } + b3, err := io.ReadAll(rd3) + if err != nil { + t.Fatalf("ReadAll third chunk error = %v", err) + } + ss.FreeSectionReader(rd3) + if !bytes.Equal(b3, data[8:12]) { + t.Fatalf("third chunk = %q, want %q", b3, data[8:12]) + } + + mu.Lock() + defer mu.Unlock() + if len(calls) != 3 { + t.Fatalf("RangeRead call count = %d, want 3", len(calls)) + } + starts := []int64{calls[0].Start, calls[1].Start, calls[2].Start} + if !((starts[0] == 0 && starts[1] == 4 && starts[2] == 8) || (starts[0] == 0 && starts[1] == 8 && starts[2] == 4)) { + t.Fatalf("unexpected RangeRead starts sequence: %v", starts) + } +} + +func TestDirectSectionReader_SeekablePrefetchDoesNotReportProgress(t *testing.T) { + data := []byte("abcdefgh") + + var progressCalls int + up := model.UpdateProgress(func(_ float64) { + progressCalls++ + }) + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, context.Background(), data, rr) + ss, err := NewStreamSectionReader(seekable, 4, &up) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + _, _ = io.ReadAll(rd1) + ss.FreeSectionReader(rd1) + + rd2, err := ss.GetSectionReader(4, 4) + if err != nil { + t.Fatalf("GetSectionReader(4,4) error = %v", err) + } + _, _ = io.ReadAll(rd2) + ss.FreeSectionReader(rd2) + + if progressCalls != 0 { + t.Fatalf("progress callback called %d times, want 0", progressCalls) + } +} + +func TestDirectSectionReader_SeekablePrefetchFailureIsObservable(t *testing.T) { + data := []byte("abcdefgh") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + started := make(chan struct{}) + var onceStarted sync.Once + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + if r.Start == 4 { + onceStarted.Do(func() { close(started) }) + <-ctx.Done() + return nil, ctx.Err() + } + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, ctx, data, rr) + ss, err := NewStreamSectionReader(seekable, 4, nil) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + _, _ = io.ReadAll(rd1) + ss.FreeSectionReader(rd1) + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatalf("prefetch did not start") + } + + cancel() + _, err = ss.GetSectionReader(4, 4) + if err == nil { + t.Fatalf("GetSectionReader(4,4) expected error, got nil") + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context canceled error, got: %v", err) + } +} + +func TestDirectSectionReader_SeekablePrefetchStopsOnContextCancel(t *testing.T) { + data := []byte("abcdefgh") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + started := make(chan struct{}) + exited := make(chan struct{}) + var onceStarted, onceExited sync.Once + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + if r.Start == 4 { + onceStarted.Do(func() { close(started) }) + <-ctx.Done() + onceExited.Do(func() { close(exited) }) + } + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, ctx, data, rr) + ss, err := NewStreamSectionReader(seekable, 4, nil) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + _, _ = io.ReadAll(rd1) + ss.FreeSectionReader(rd1) + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatalf("prefetch did not start") + } + + cancel() + + select { + case <-exited: + // pass: 预读协程在 ctx cancel 后退出 + case <-time.After(2 * time.Second): + t.Fatalf("prefetch goroutine did not exit after context cancel") + } +} + +func newSeekableStreamForSectionTest(t *testing.T, ctx context.Context, data []byte, rr model.RangeReaderIF) *SeekableStream { + t.Helper() + obj := &model.Object{Name: "section-test.bin", Size: int64(len(data))} + fs := &FileStream{Ctx: ctx, Obj: obj} + link := &model.Link{RangeReader: rr, ContentLength: int64(len(data))} + ss, err := NewSeekableStream(fs, link) + if err != nil { + t.Fatalf("NewSeekableStream() error = %v", err) + } + t.Cleanup(func() { + _ = ss.Close() + }) + return ss +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 4c8238100..7eec75dd9 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -211,7 +211,9 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { return io.NewSectionReader(f.GetFile(), httpRange.Start, httpRange.Length), nil } - cache, err := f.cache(httpRange.Start + httpRange.Length) + // 限制缓存大小，避免累积缓存整个文件 + maxCache := min(httpRange.Start+httpRange.Length, int64(conf.MaxBufferLimit)) + cache, err := f.cache(maxCache) if err != nil { return nil, err } @@ -224,31 +226,13 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { // 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大 // 确保指定大小的数据被缓存 +// 注意：此方法只缓存到 maxCacheSize，不会缓存整个文件 func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { + // 限制缓存大小，避免超大文件占用过多资源 + // 如果需要缓存整个文件，应该显式调用 CacheFullAndWriter if maxCacheSize > int64(conf.MaxBufferLimit) { - size := f.GetSize() - reader := f.Reader - if f.peekBuff != nil { - size -= f.peekBuff.Size() - reader = f.oriReader - } - tmpF, err := utils.CreateTempFile(reader, size) - if err != nil { - return nil, err - } - f.Add(utils.CloseFunc(func() error { - return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) - })) - if f.peekBuff != nil { - peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) - if err != nil { - return nil, err - } - f.Reader = peekF - return peekF, nil - } - f.Reader = tmpF - return tmpF, nil + // 不再创建整个文件的临时文件，只缓存到 MaxBufferLimit + maxCacheSize = int64(conf.MaxBufferLimit) } if f.peekBuff == nil { @@ -315,15 +299,9 @@ func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error if err != nil { return nil, err } - if _, ok := rr.(*model.FileRangeReader); ok { - var rc io.ReadCloser - rc, err = rr.RangeRead(fs.Ctx, http_range.Range{Length: -1}) - if err != nil { - return nil, err - } - fs.Reader = rc - fs.Add(rc) - } + // IMPORTANT: Do NOT create Reader early for FileRangeReader! + // Let generateReader() create it on-demand when actually needed for reading + // This prevents the Reader from being consumed by intermediate operations like hash calculation fs.size = size fs.Add(link) return &SeekableStream{FileStream: fs, rangeReader: rr}, nil diff --git a/internal/stream/util.go b/internal/stream/util.go index 6aa3dda5d..9c6c6357f 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -9,6 +9,9 @@ import ( "io" "net/http" "os" + "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -21,13 +24,298 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + // 链接刷新相关常量 + MAX_LINK_REFRESH_COUNT = 50 // 下载链接最大刷新次数（支持长时间传输） + + // RangeRead 重试相关常量 + MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead 最大重试次数（从3增加到5） +) + type RangeReaderFunc func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return f(ctx, httpRange) } +// IsLinkExpiredError checks if the error indicates an expired download link +func IsLinkExpiredError(err error) bool { + if err == nil { + return false + } + + // Don't treat context cancellation as link expiration + // This happens when user pauses/seeks video or cancels download + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + + errStr := strings.ToLower(err.Error()) + + // Common expired link error keywords + expiredKeywords := []string{ + "expired", "invalid signature", "token expired", + "access denied", "forbidden", "unauthorized", + "link has expired", "url expired", "request has expired", + "signature expired", "accessdenied", "invalidtoken", + } + for _, keyword := range expiredKeywords { + if strings.Contains(errStr, keyword) { + return true + } + } + + // Check for HTTP status codes that typically indicate expired links + if statusErr, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { + code := int(statusErr) + // All 4xx client errors may indicate expired/invalid links + // 400 Bad Request, 401 Unauthorized, 403 Forbidden, 404 Not Found, 410 Gone, etc. + if code >= 400 && code < 500 { + return true + } + } + + return false +} + +// RefreshableRangeReader wraps a RangeReader with link refresh capability +type RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // track refresh count to avoid infinite loops +} + +// NewRefreshableRangeReader creates a new RefreshableRangeReader +func NewRefreshableRangeReader(link *model.Link, size int64) *RefreshableRangeReader { + return &RefreshableRangeReader{ + link: link, + size: size, + } +} + +func (r *RefreshableRangeReader) getInnerReader() (model.RangeReaderIF, error) { + if r.innerReader != nil { + return r.innerReader, nil + } + + // Create inner reader without Refresher to avoid recursion + linkCopy := *r.link + linkCopy.Refresher = nil + + reader, err := GetRangeReaderFromLink(r.size, &linkCopy) + if err != nil { + return nil, err + } + r.innerReader = reader + return reader, nil +} + +func (r *RefreshableRangeReader) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + reader, err := r.getInnerReader() + r.mu.Unlock() + if err != nil { + return nil, err + } + + rc, err := reader.RangeRead(ctx, httpRange) + if err != nil { + // Check if we should try to refresh on initial connection error + if IsLinkExpiredError(err) && r.link.Refresher != nil { + rc, err = r.refreshAndRetry(ctx, httpRange) + } + if err != nil { + return nil, err + } + } + + // Wrap the ReadCloser with self-healing capability to detect 0-byte reads + // This handles cases where cloud providers return 200 OK but empty body for expired links + return &selfHealingReadCloser{ + ReadCloser: rc, + refresher: r, + ctx: ctx, + httpRange: httpRange, + firstRead: false, + closed: false, + }, nil +} + +func (r *RefreshableRangeReader) refreshAndRetry(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if err := r.doRefreshLocked(ctx); err != nil { + return nil, err + } + + reader, err := r.getInnerReader() + if err != nil { + return nil, err + } + return reader.RangeRead(ctx, httpRange) +} + +// doRefreshLocked 执行实际的刷新逻辑（需要持有锁） +func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { + if r.refreshCount >= MAX_LINK_REFRESH_COUNT { + return fmt.Errorf("max refresh attempts (%d) reached", MAX_LINK_REFRESH_COUNT) + } + + log.Infof("Link expired, attempting to refresh...") + // Use independent context for refresh to prevent cancellation from affecting link refresh + refreshCtx := context.WithoutCancel(ctx) + newLink, _, refreshErr := r.link.Refresher(refreshCtx) + if refreshErr != nil { + return fmt.Errorf("failed to refresh link: %w", refreshErr) + } + + newLink.Refresher = r.link.Refresher + r.link = newLink + r.innerReader = nil + r.refreshCount++ + + log.Infof("Link refreshed successfully") + return nil +} + +// selfHealingReadCloser wraps an io.ReadCloser and automatically refreshes the link +// if the upstream reader dies before the requested range is fully delivered. +type selfHealingReadCloser struct { + io.ReadCloser + refresher *RefreshableRangeReader + ctx context.Context + httpRange http_range.Range + firstRead bool + bytesRead int64 + closed bool + mu sync.Mutex +} + +func (s *selfHealingReadCloser) Read(p []byte) (n int, err error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return 0, errors.New("read from closed reader") + } + + n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + wasFirstRead := !s.firstRead + s.firstRead = true + + // Detect 0-byte read on first attempt (indicates link may be expired but returned 200 OK) + if s.shouldReconnectAfterRead(wasFirstRead, n, err) { + if reconnectErr := s.reconnectFromCurrentOffsetLocked(); reconnectErr != nil { + log.Errorf("Failed to refresh link after interrupted read: %v", reconnectErr) + return n, err + } + + if n > 0 { + return n, nil + } + + n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + return n, err + } + + return n, err +} + +func (s *selfHealingReadCloser) shouldReconnectAfterRead(wasFirstRead bool, n int, err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + if s.remainingBytes() <= 0 { + return false + } + + if wasFirstRead && n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { + log.Warnf("Detected 0-byte read on first attempt, attempting to refresh link...") + return true + } + + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + log.Warnf("Detected interrupted read after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } + + errStr := strings.ToLower(err.Error()) + if strings.Contains(errStr, "connection reset by peer") { + log.Warnf("Detected upstream connection reset after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } + + return false +} + +func (s *selfHealingReadCloser) reconnectFromCurrentOffsetLocked() error { + nextRange := s.httpRange + nextRange.Start += s.bytesRead + if nextRange.Length >= 0 { + nextRange.Length -= s.bytesRead + } + + s.refresher.mu.Lock() + refreshErr := s.refresher.doRefreshLocked(s.ctx) + if refreshErr != nil { + s.refresher.mu.Unlock() + return refreshErr + } + + reader, getErr := s.refresher.getInnerReader() + s.refresher.mu.Unlock() + if getErr != nil { + return getErr + } + + newRc, rangeErr := reader.RangeRead(s.ctx, nextRange) + if rangeErr != nil { + return rangeErr + } + + _ = s.ReadCloser.Close() + s.ReadCloser = newRc + log.Infof("Successfully refreshed link and reconnected from offset %d", nextRange.Start) + return nil +} + +func (s *selfHealingReadCloser) remainingBytes() int64 { + length := s.httpRange.Length + if length < 0 || s.httpRange.Start+length > s.refresher.size { + length = s.refresher.size - s.httpRange.Start + } + remaining := length - s.bytesRead + if remaining < 0 { + return 0 + } + return remaining +} + +func (s *selfHealingReadCloser) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return nil + } + s.closed = true + return s.ReadCloser.Close() +} + func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { + // If link has a Refresher, wrap with RefreshableRangeReader for automatic refresh on expiry + if link.Refresher != nil { + return NewRefreshableRangeReader(link, size), nil + } + if link.RangeReader != nil { if link.Concurrency < 1 && link.PartSize < 1 { return link.RangeReader, nil @@ -174,6 +462,120 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } +// ReadFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf +// file: 文件流 +// buf: 目标缓冲区 +// off: 读取的起始偏移量 +// 返回值: 实际读取的字节数和错误 +// 支持自动重试（最多5次），快速重试策略（1秒、2秒、3秒、4秒、5秒） +// 注意：链接刷新现在由 RefreshableRangeReader 内部的 selfHealingReadCloser 自动处理 +func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { + length := int64(len(buf)) + var lastErr error + + // 重试最多 MAX_RANGE_READ_RETRY_COUNT 次 + for retry := 0; retry < MAX_RANGE_READ_RETRY_COUNT; retry++ { + reader, err := file.RangeRead(http_range.Range{Start: off, Length: length}) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return 0, err + } + lastErr = fmt.Errorf("RangeRead failed at offset %d: %w", off, err) + log.Debugf("RangeRead retry %d failed: %v", retry+1, lastErr) + // 快速重试：1秒、2秒、3秒、4秒、5秒（连接失败快速重试） + time.Sleep(time.Duration(retry+1) * time.Second) + continue + } + + n, err := io.ReadFull(reader, buf) + if closer, ok := reader.(io.Closer); ok { + closer.Close() + } + + if err == nil { + return n, nil + } + + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return n, err + } + + lastErr = fmt.Errorf("failed to read all data via RangeRead at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) + log.Debugf("RangeRead retry %d read failed: %v", retry+1, lastErr) + + // 快速重试：1秒、2秒、3秒、4秒、5秒（读取失败快速重试） + // 注意：0字节读取导致的链接过期现在由 selfHealingReadCloser 自动处理 + time.Sleep(time.Duration(retry+1) * time.Second) + } + + return 0, lastErr +} + +// StreamHashFile 流式计算文件哈希值，避免将整个文件加载到内存 +// file: 文件流 +// hashType: 哈希算法类型 +// progressWeight: 进度权重（0-100），用于计算整体进度 +// up: 进度回调函数 +func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressWeight float64, up *model.UpdateProgress) (string, error) { + // 如果已经有完整缓存文件，直接使用 + if cache := file.GetFile(); cache != nil { + hashFunc := hashType.NewFunc() + cache.Seek(0, io.SeekStart) + _, err := io.Copy(hashFunc, cache) + if err != nil { + return "", err + } + if up != nil && progressWeight > 0 { + (*up)(progressWeight) + } + return hex.EncodeToString(hashFunc.Sum(nil)), nil + } + + hashFunc := hashType.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk + buf := make([]byte, chunkSize) + var offset int64 = 0 + + for offset < size { + readSize := chunkSize + if size-offset < chunkSize { + readSize = size - offset + } + + var n int + var err error + + // 对于 SeekableStream，优先使用 RangeRead 避免消耗 Reader + // 这样后续发送时 Reader 还能正常工作 + if _, ok := file.(*SeekableStream); ok { + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) + } else { + // 对于 FileStream，首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） + n, err = io.ReadFull(file, buf[:readSize]) + if err != nil { + // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） + log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) + } + } + + if err != nil { + return "", fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + + hashFunc.Write(buf[:n]) + offset += int64(n) + + if up != nil && progressWeight > 0 { + progress := progressWeight * float64(offset) / float64(size) + (*up)(progress) + } + } + + return hex.EncodeToString(hashFunc.Sum(nil)), nil +} + type StreamSectionReaderIF interface { // 线程不安全 GetSectionReader(off, length int64) (io.ReadSeeker, error) @@ -188,37 +590,9 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode } maxBufferSize = min(maxBufferSize, int(file.GetSize())) - if maxBufferSize > conf.MaxBufferLimit { - f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - - if f.Truncate(file.GetSize()) != nil { - // fallback to full cache - _, _ = f.Close(), os.Remove(f.Name()) - cache, err := file.CacheFullAndWriter(up, nil) - if err != nil { - return nil, err - } - return &cachedSectionReader{cache}, nil - } - - ss := &fileSectionReader{file: file, temp: f} - ss.bufPool = &pool.Pool[*offsetWriterWithBase]{ - New: func() *offsetWriterWithBase { - base := ss.tempOffset - ss.tempOffset += int64(maxBufferSize) - return &offsetWriterWithBase{io.NewOffsetWriter(ss.temp, base), base} - }, - } - file.Add(utils.CloseFunc(func() error { - ss.bufPool.Reset() - return errors.Join(ss.temp.Close(), os.Remove(ss.temp.Name())) - })) - return ss, nil - } + // 始终使用 directSectionReader，只在内存中缓存当前分片 + // 避免创建临时文件导致中间文件增长到整个文件大小 ss := &directSectionReader{file: file} if conf.MmapThreshold > 0 && maxBufferSize >= conf.MmapThreshold { ss.bufPool = &pool.Pool[[]byte]{ @@ -243,6 +617,7 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode } file.Add(utils.CloseFunc(func() error { + ss.clearPrefetch() ss.bufPool.Reset() return nil })) @@ -319,10 +694,129 @@ type directSectionReader struct { file model.FileStreamer fileOffset int64 bufPool *pool.Pool[[]byte] + prefetchMu sync.Mutex + prefetch *sectionPrefetchTask } -// 线程不安全 +type sectionPrefetchTask struct { + off int64 + length int64 + buf []byte + n int + err error + ready chan struct{} +} + +func (ss *directSectionReader) recyclePrefetchTask(task *sectionPrefetchTask) { + if task == nil || task.buf == nil { + return + } + ss.bufPool.Put(task.buf[0:cap(task.buf)]) + task.buf = nil +} + +func (ss *directSectionReader) clearPrefetch() { + ss.prefetchMu.Lock() + task := ss.prefetch + ss.prefetch = nil + ss.prefetchMu.Unlock() + if task == nil { + return + } + go func(t *sectionPrefetchTask) { + <-t.ready + ss.recyclePrefetchTask(t) + }(task) +} + +func (ss *directSectionReader) takeMatchingPrefetch(off, length int64) *sectionPrefetchTask { + ss.prefetchMu.Lock() + task := ss.prefetch + if task != nil && task.off == off && task.length == length { + ss.prefetch = nil + ss.prefetchMu.Unlock() + return task + } + ss.prefetch = nil + ss.prefetchMu.Unlock() + if task != nil { + go func(t *sectionPrefetchTask) { + <-t.ready + ss.recyclePrefetchTask(t) + }(task) + } + return nil +} + +func (ss *directSectionReader) launchPrefetch(off, length int64) { + if length <= 0 { + return + } + tempBuf := ss.bufPool.Get() + if int64(cap(tempBuf)) < length { + tempBuf = make([]byte, length) + } + task := §ionPrefetchTask{ + off: off, + length: length, + buf: tempBuf, + ready: make(chan struct{}), + } + + ss.prefetchMu.Lock() + old := ss.prefetch + ss.prefetch = task + ss.prefetchMu.Unlock() + + if old != nil { + go func(t *sectionPrefetchTask) { + <-t.ready + ss.recyclePrefetchTask(t) + }(old) + } + + go func(t *sectionPrefetchTask) { + buf := t.buf[:int(t.length)] + n, err := ReadFullWithRangeRead(ss.file, buf, t.off) + if err != nil { + t.err = fmt.Errorf("RangeRead failed at offset %d: (expect=%d, actual=%d) %w", t.off, t.length, n, err) + } else if int64(n) != t.length { + t.err = fmt.Errorf("RangeRead failed at offset %d: (expect=%d, actual=%d)", t.off, t.length, n) + } else { + t.n = n + } + close(t.ready) + }(task) +} + +func (ss *directSectionReader) scheduleNextPrefetch(curOff, curLen int64) { + if _, ok := ss.file.(*SeekableStream); !ok { + return + } + nextOff := curOff + curLen + if nextOff >= ss.file.GetSize() { + ss.clearPrefetch() + return + } + nextLen := min(curLen, ss.file.GetSize()-nextOff) + if nextLen <= 0 { + ss.clearPrefetch() + return + } + ss.launchPrefetch(nextOff, nextLen) +} + +// 线程不安全（依赖调用方保证串行调用） +// 对于 SeekableStream：直接跳过（无需实际读取） +// 对于 FileStream：必须顺序读取并丢弃 func (ss *directSectionReader) DiscardSection(off int64, length int64) error { + // 对于 SeekableStream，直接跳过（RangeRead 支持随机访问，不需要实际读取） + if _, ok := ss.file.(*SeekableStream); ok { + ss.clearPrefetch() + return nil + } + + // 对于 FileStream，必须顺序读取并丢弃 if off != ss.fileOffset { return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } @@ -339,19 +833,53 @@ type bufferSectionReader struct { buf []byte } -// 线程不安全 +// 线程不安全（依赖调用方保证串行调用） +// 对于 SeekableStream：使用 RangeRead，支持随机访问（续传场景可跳过已上传分片） +// 对于 FileStream：必须顺序读取 func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { + tempBuf := ss.bufPool.Get() + if int64(cap(tempBuf)) < length { + tempBuf = make([]byte, length) + } + buf := tempBuf[:int(length)] + + // 对于 SeekableStream，直接使用 RangeRead（支持随机访问，适用于续传场景） + if _, ok := ss.file.(*SeekableStream); ok { + if task := ss.takeMatchingPrefetch(off, length); task != nil { + <-task.ready + if task.err != nil { + ss.recyclePrefetchTask(task) + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) + return nil, task.err + } + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) + ss.scheduleNextPrefetch(off, length) + return &bufferSectionReader{bytes.NewReader(task.buf[:int(length)]), task.buf}, nil + } + + n, err := ReadFullWithRangeRead(ss.file, buf, off) + if err != nil { + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) + return nil, fmt.Errorf("RangeRead failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) + } + ss.scheduleNextPrefetch(off, length) + return &bufferSectionReader{bytes.NewReader(buf), tempBuf}, nil + } + + // 对于 FileStream，必须顺序读取 if off != ss.fileOffset { + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } - tempBuf := ss.bufPool.Get() - buf := tempBuf[:length] + n, err := io.ReadFull(ss.file, buf) - ss.fileOffset += int64(n) - if int64(n) != length { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) + if err != nil { + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) + return nil, fmt.Errorf("sequential read failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) } - return &bufferSectionReader{bytes.NewReader(buf), buf}, nil + + ss.fileOffset = off + int64(n) + return &bufferSectionReader{bytes.NewReader(buf), tempBuf}, nil } func (ss *directSectionReader) FreeSectionReader(rs io.ReadSeeker) { if sr, ok := rs.(*bufferSectionReader); ok { diff --git a/internal/stream/util_test.go b/internal/stream/util_test.go new file mode 100644 index 000000000..6dc4c8a09 --- /dev/null +++ b/internal/stream/util_test.go @@ -0,0 +1,160 @@ +package stream + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: int64(len(data))}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != int64(len(data)-5) { + t.Fatalf("resumed range length = %d, want %d", resumedRanges[0].Length, len(data)-5) + } +} + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset_UnboundedRange(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: -1}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != -1 { + t.Fatalf("resumed range length = %d, want -1", resumedRanges[0].Length) + } +} + +type flakyReadCloser struct { + data []byte + failAfter int + failErr error + failed bool +} + +func newFlakyReadCloser(data []byte, failAfter int, failErr error) *flakyReadCloser { + return &flakyReadCloser{ + data: data, + failAfter: failAfter, + failErr: failErr, + } +} + +func (f *flakyReadCloser) Read(p []byte) (int, error) { + if f.failed { + return 0, io.EOF + } + if f.failAfter >= len(f.data) { + f.failed = true + n := copy(p, f.data) + return n, io.EOF + } + + n := copy(p, f.data[:f.failAfter]) + f.failed = true + return n, f.failErr +} + +func (f *flakyReadCloser) Close() error { + return nil +} + +func sliceForRange(data []byte, httpRange http_range.Range) []byte { + start := int(httpRange.Start) + length := int(httpRange.Length) + if httpRange.Length < 0 || httpRange.Start+httpRange.Length > int64(len(data)) { + length = len(data) - start + } + return data[start : start+length] +} diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index 596e61e54..c4b4e735f 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -90,6 +90,12 @@ var ( // SHA256 indicates SHA-256 support SHA256 = RegisterHash("sha256", "SHA-256", 64, sha256.New) + + // SHA1_128K is SHA1 of first 128KB, used by 115 driver for rapid upload + SHA1_128K = RegisterHash("sha1_128k", "SHA1-128K", 40, sha1.New) + + // PRE_HASH is SHA1 of first 1024 bytes, used by Aliyundrive for rapid upload + PRE_HASH = RegisterHash("pre_hash", "PRE-HASH", 40, sha1.New) ) // HashData get hash of one hashType diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 0f46398cd..54cdb4fee 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -93,6 +93,12 @@ func FsStream(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := c.GetHeader("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) @@ -190,6 +196,12 @@ func FsForm(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := file.Header.Get("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) From 8b4a17072304de232bbf1e19c4014364734a1d17 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:24 +0800 Subject: [PATCH 80/86] feat(google_drive): duplicate filename handling, folder lock, retry, and MD5 checksum - Add duplicate filename handling to ensure file name uniqueness - Add folder creation lock mechanism and retry logic - Increase MakeDir wait time for Google Drive API sync delay - Add retry mechanism for small file upload read errors - Update Put method to support MD5 checksum on seekable and non-seekable streams --- drivers/google_drive/driver.go | 163 +++++++++++++++++++++++++++++---- drivers/google_drive/util.go | 51 +++++++++++ 2 files changed, 197 insertions(+), 17 deletions(-) diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 94ef854f2..1e2e476ce 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -3,17 +3,29 @@ package google_drive import ( "context" "fmt" + "io" "net/http" "strconv" + "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) +// mkdirLocks prevents race conditions when creating folders with the same name +// Google Drive allows duplicate folder names, so we need application-level locking +var mkdirLocks sync.Map // map[string]*sync.Mutex - key is parentID + "/" + dirName + type GoogleDrive struct { model.Storage Addition @@ -67,15 +79,76 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkA } func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + // Use per-folder lock to prevent concurrent creation of same folder + // This is critical because Google Drive allows duplicate folder names + lockKey := parentDir.GetID() + "/" + dirName + lockVal, _ := mkdirLocks.LoadOrStore(lockKey, &sync.Mutex{}) + lock := lockVal.(*sync.Mutex) + lock.Lock() + defer lock.Unlock() + + // Check if folder already exists with retry to handle API eventual consistency + escapedDirName := strings.ReplaceAll(dirName, "'", "\\'") + query := map[string]string{ + "q": fmt.Sprintf("name='%s' and '%s' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false", escapedDirName, parentDir.GetID()), + "fields": "files(id)", + } + + var existingFiles Files + err := retry.Do(func() error { + var checkErr error + _, checkErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &existingFiles) + return checkErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(200*time.Millisecond), + ) + + // If query succeeded and folder exists, return success (idempotent) + if err == nil && len(existingFiles.Files) > 0 { + log.Debugf("[google_drive] Folder '%s' already exists in parent %s, skipping creation", dirName, parentDir.GetID()) + return nil + } + // If query failed, return error to prevent duplicate creation + if err != nil { + return fmt.Errorf("failed to check existing folder '%s': %w", dirName, err) + } + + // Create new folder (only when confirmed folder doesn't exist) data := base.Json{ "name": dirName, "parents": []string{parentDir.GetID()}, "mimeType": "application/vnd.google-apps.folder", } - _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { - req.SetBody(data) - }, nil) - return err + + var createErr error + err = retry.Do(func() error { + _, createErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + return createErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) + + if err != nil { + return err + } + + // Wait for API eventual consistency before releasing lock + // This helps prevent race conditions where a concurrent request + // checks for folder existence before the newly created folder is visible + // 500ms is needed because Google Drive API has significant sync delay + time.Sleep(500 * time.Millisecond) + + return nil } func (d *GoogleDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -111,8 +184,44 @@ func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - obj := stream.GetExist() +func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + // 1. 准备MD5（用于完整性校验） + md5Hash := file.GetHash().GetHash(utils.MD5) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(file, utils.MD5, 100, &up) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验（Google Drive会自动校验） + } + } else { + // 不可重复读取的流（如 HTTP body） + if len(md5Hash) != utils.MD5.Width { + // 缓存整个文件并计算 MD5 + var err error + _, md5Hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验 + } else if file.GetFile() == nil { + // 有 MD5 但没有缓存，需要缓存以支持后续 RangeRead + _, err := file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + } + + // 2. 初始化可恢复上传会话 + obj := file.GetExist() var ( e Error url string @@ -125,7 +234,7 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi data = base.Json{} } else { data = base.Json{ - "name": stream.GetName(), + "name": file.GetName(), "parents": []string{dstDir.GetID()}, } url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" @@ -133,8 +242,8 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi req := base.NoRedirectClient.R(). SetHeaders(map[string]string{ "Authorization": "Bearer " + d.AccessToken, - "X-Upload-Content-Type": stream.GetMimetype(), - "X-Upload-Content-Length": strconv.FormatInt(stream.GetSize(), 10), + "X-Upload-Content-Type": file.GetMimetype(), + "X-Upload-Content-Length": strconv.FormatInt(file.GetSize(), 10), }). SetError(&e).SetBody(data).SetContext(ctx) if obj != nil { @@ -151,20 +260,40 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi if err != nil { return err } - return d.Put(ctx, dstDir, stream, up) + return d.Put(ctx, dstDir, file, up) } return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } + + // 3. 上传文件内容 putUrl := res.Header().Get("location") - if stream.GetSize() < d.ChunkSize*1024*1024 { - _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { - req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). - SetBody(driver.NewLimitedUploadStream(ctx, stream)) - }, nil) + if file.GetSize() < d.ChunkSize*1024*1024 { + // 小文件上传：使用 RangeRead 读取整个文件（避免消费已计算hash的stream） + err = retry.Do(func() error { + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } + + _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { + req.SetHeader("Content-Length", strconv.FormatInt(file.GetSize(), 10)). + SetBody(driver.NewLimitedUploadStream(ctx, reader)) + }, nil) + return err + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) + return err } else { - err = d.chunkUpload(ctx, stream, putUrl, up) + // 大文件分片上传 + return d.chunkUpload(ctx, file, putUrl, up) } - return err } func (d *GoogleDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 042abafa4..1fc68c335 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -296,9 +296,60 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { res = append(res, resp.Files...) } + + // Handle duplicate filenames by adding suffixes like (1), (2), etc. + // Google Drive allows multiple files with the same name in one folder, + // but OpenList uses path-based file system which requires unique names + res = handleDuplicateNames(res) + return res, nil } +// handleDuplicateNames adds suffixes to duplicate filenames to make them unique +// For example: file.txt, file (1).txt, file (2).txt +func handleDuplicateNames(files []File) []File { + if len(files) <= 1 { + return files + } + + // Track how many files with each name we've seen + nameCount := make(map[string]int) + + // First pass: count occurrences of each name + for _, file := range files { + nameCount[file.Name]++ + } + + // Second pass: add suffixes to duplicates + nameIndex := make(map[string]int) + for i := range files { + name := files[i].Name + if nameCount[name] > 1 { + index := nameIndex[name] + nameIndex[name]++ + + if index > 0 { + // Add suffix for all except the first occurrence + // Split name into base and extension + ext := "" + base := name + for j := len(name) - 1; j >= 0; j-- { + if name[j] == '.' { + ext = name[j:] + base = name[:j] + break + } + } + + // Add (1), (2), etc. suffix + files[i].Name = fmt.Sprintf("%s (%d)%s", base, index, ext) + } + } + } + + return files +} + // getTargetFileInfo gets target file details for shortcuts func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) { var targetFile File From bf6efdb1ae94a5c982c694eb1c86155273362a08 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:32 +0800 Subject: [PATCH 81/86] feat(115_open): permanent delete, proxy_range, offline task fixes, and error handling - Add permanent delete feature with recycle-bin retry logic - Expose proxy_range option in driver meta - Fix duplicate link error handling via error code 10008 detection - Fix folder delete cid JSON number deserialization error - Move offline task Remove before Transfer - Cleanup completed offline download tasks --- drivers/115_open/driver.go | 330 ++++++++- drivers/115_open/driver_test.go | 639 ++++++++++++++++++ drivers/115_open/meta.go | 8 +- drivers/115_open/upload.go | 44 +- internal/offline_download/115_open/client.go | 503 +++++++++++++- .../offline_download/115_open/client_test.go | 616 +++++++++++++++++ 6 files changed, 2107 insertions(+), 33 deletions(-) create mode 100644 drivers/115_open/driver_test.go create mode 100644 internal/offline_download/115_open/client_test.go diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index ec76a6bc8..f22958429 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -2,6 +2,7 @@ package _115_open import ( "context" + "encoding/json" "fmt" "net/http" "strconv" @@ -17,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -27,6 +29,12 @@ type Open115 struct { limiter *rate.Limiter } +var ( + // 回收站列表存在短暂最终一致性延迟，永久删除 fallback 查找增加短重试。 + recycleBinLookupMaxAttempts = 4 + recycleBinLookupRetryDelay = 300 * time.Millisecond +) + func (d *Open115) Config() driver.Config { return config } @@ -74,13 +82,20 @@ func (d *Open115) Drop(ctx context.Context) error { } func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + start := time.Now() + log.Infof("[115] List request started for dir: %s (ID: %s)", dir.GetName(), dir.GetID()) + var res []model.Obj pageSize := int64(d.PageSize) offset := int64(0) + pageCount := 0 + for { if err := d.WaitLimit(ctx); err != nil { return nil, err } + + pageStart := time.Now() resp, err := d.client.GetFiles(ctx, &sdk.GetFilesReq{ CID: dir.GetID(), Limit: pageSize, @@ -90,7 +105,12 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) // Cur: 1, ShowDir: true, }) + pageDuration := time.Since(pageStart) + pageCount++ + log.Infof("[115] GetFiles page %d took: %v (offset=%d, limit=%d)", pageCount, pageDuration, offset, pageSize) + if err != nil { + log.Errorf("[115] GetFiles page %d failed after %v: %v", pageCount, pageDuration, err) return nil, err } res = append(res, utils.MustSliceConvert(resp.Data, func(src sdk.GetFilesResp_File) model.Obj { @@ -102,10 +122,17 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) } offset += pageSize } + + totalDuration := time.Since(start) + log.Infof("[115] List request completed in %v (%d pages, %d files)", totalDuration, pageCount, len(res)) + return res, nil } func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + start := time.Now() + log.Infof("[115] Link request started for file: %s", file.GetName()) + if err := d.WaitLimit(ctx); err != nil { return nil, err } @@ -121,14 +148,25 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, fmt.Errorf("can't convert obj") } pc := obj.Pc + + apiStart := time.Now() + log.Infof("[115] Calling DownURL API...") resp, err := d.client.DownURL(ctx, pc, ua) + apiDuration := time.Since(apiStart) + log.Infof("[115] DownURL API took: %v", apiDuration) + if err != nil { + log.Errorf("[115] DownURL API failed after %v: %v", apiDuration, err) return nil, err } u, ok := resp[obj.GetID()] if !ok { return nil, fmt.Errorf("can't get link") } + + totalDuration := time.Since(start) + log.Infof("[115] Link request completed in %v (API: %v)", totalDuration, apiDuration) + return &model.Link{ URL: u.URL.URL, Header: http.Header{ @@ -175,7 +213,7 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) return nil, err } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ - FileID: srcObj.GetID(), + FileID: srcObj.GetID(), FileName: newName, }) if err != nil { @@ -211,13 +249,164 @@ func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { if !ok { return fmt.Errorf("can't convert obj") } - _, err := d.client.DelFile(ctx, &sdk.DelFileReq{ + resp, err := d.client.DelFile(ctx, &sdk.DelFileReq{ FileIDs: _obj.GetID(), ParentID: _obj.Pid, }) if err != nil { return err } + if d.RemoveWay != "delete" { + return nil + } + return d.removePermanently(ctx, _obj, resp) +} + +func (d *Open115) removePermanently(ctx context.Context, obj *Obj, deleteResp []string) error { + var directDeleteErr error + for _, tid := range deleteResp { + tid = strings.TrimSpace(tid) + if tid == "" { + continue + } + if err := d.deleteRecycleBinEntry(ctx, tid); err == nil { + return nil + } else if directDeleteErr == nil { + directDeleteErr = err + } + } + + recycleEntry, err := d.findRecycleBinEntryWithRetry(ctx, obj) + if err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin candidate: %w; fallback lookup failed: %v", directDeleteErr, err) + } + return err + } + if err := d.deleteRecycleBinEntry(ctx, recycleEntry.ID); err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin entry %s after candidate delete error %v: %w", recycleEntry.ID, directDeleteErr, err) + } + return err + } + return nil +} + +func (d *Open115) deleteRecycleBinEntry(ctx context.Context, tid string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + _, err := d.client.RbDelete(ctx, tid) + return err +} + +func (d *Open115) findRecycleBinEntry(ctx context.Context, obj *Obj) (*sdk.RbListResp_FileInfo, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 200 + } else if pageSize > 1150 { + pageSize = 1150 + } + + offset := int64(0) + for { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + resp, err := d.client.RbList(ctx, pageSize, offset) + if err != nil { + return nil, err + } + if entry := matchRecycleBinEntry(obj, resp.Files); entry != nil { + return entry, nil + } + + count, err := strconv.ParseInt(resp.Count, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse recycle bin count %q: %w", resp.Count, err) + } + offset += pageSize + if offset >= count || len(resp.Files) == 0 { + break + } + } + + return nil, fmt.Errorf("recycle bin entry not found for object id=%s name=%s parent=%s", obj.GetID(), obj.GetName(), obj.Pid) +} + +func isRecycleBinEntryNotFoundErr(err error) bool { + return err != nil && strings.Contains(err.Error(), "recycle bin entry not found") +} + +func (d *Open115) findRecycleBinEntryWithRetry(ctx context.Context, obj *Obj) (*sdk.RbListResp_FileInfo, error) { + attempts := recycleBinLookupMaxAttempts + if attempts < 1 { + attempts = 1 + } + + var lastErr error + for i := 0; i < attempts; i++ { + entry, err := d.findRecycleBinEntry(ctx, obj) + if err == nil { + return entry, nil + } + + lastErr = err + if !isRecycleBinEntryNotFoundErr(err) || i == attempts-1 { + break + } + + wait := recycleBinLookupRetryDelay * time.Duration(i+1) + if wait <= 0 { + continue + } + + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + + return nil, lastErr +} + +func matchRecycleBinEntry(obj *Obj, files map[string]sdk.RbListResp_FileInfo) *sdk.RbListResp_FileInfo { + if len(files) == 0 { + return nil + } + if entry, ok := files[obj.GetID()]; ok { + matched := entry + return &matched + } + + size := strconv.FormatInt(obj.GetSize(), 10) + for _, entry := range files { + if entry.ID == obj.GetID() { + matched := entry + return &matched + } + cid := string(entry.CID) + if obj.IsDir() { + if entry.FileName == obj.GetName() && cid == obj.Pid { + matched := entry + return &matched + } + continue + } + if obj.Sha1 != "" && entry.SHA1 != "" && strings.EqualFold(entry.SHA1, obj.Sha1) { + if entry.FileName == obj.GetName() || cid == obj.Pid { + matched := entry + return &matched + } + } + if entry.FileName == obj.GetName() && cid == obj.Pid && entry.FileSize == size { + matched := entry + return &matched + } + } return nil } @@ -226,27 +415,97 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } + sha1 := file.GetHash().GetHash(utils.SHA1) - if len(sha1) != utils.SHA1.Width { - _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + sha1128k := file.GetHash().GetHash(utils.SHA1_128K) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ + FileName: file.GetName(), + FileSize: file.GetSize(), + Target: dstDir.GetID(), + FileID: strings.ToUpper(sha1), + PreID: strings.ToUpper(sha1128k), + }) if err != nil { return err } + if resp.Status == 2 { + up(100) + return nil + } + // 秒传失败，继续后续流程 } - const PreHashSize int64 = 128 * utils.KB - hashSize := PreHashSize - if file.GetSize() < PreHashSize { - hashSize = file.GetSize() - } - reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) - if err != nil { - return err - } - sha1128k, err := utils.HashReader(utils.SHA1, reader) - if err != nil { - return err + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(sha1) != utils.SHA1.Width { + sha1, err = stream.StreamHashFile(file, utils.SHA1, 100, &up) + if err != nil { + return err + } + } + // 计算 sha1_128k（如果没有预计算） + if len(sha1128k) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 如果有预计算的 hash，上面已经尝试过秒传了 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + // 秒传失败，需要缓存文件进行实际上传 + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } else { + // 没有预计算的 hash，缓存整个文件并计算 + if len(sha1) != utils.SHA1.Width { + _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + if err != nil { + return err + } + } else if file.GetFile() == nil { + // 有 SHA1 但没有缓存，需要缓存以支持后续 RangeRead + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + // 计算 sha1_128k + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } } - // 1. Init + + // 1. Init（SeekableStream 或已缓存的 FileStream） resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), @@ -272,11 +531,11 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } - reader, err = file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) + signReader, err := file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) if err != nil { return err } - signVal, err := utils.HashReader(utils.SHA1, reader) + signVal, err := utils.HashReader(utils.SHA1, signReader) if err != nil { return err } @@ -314,15 +573,48 @@ func (d *Open115) OfflineDownload(ctx context.Context, uris []string, dstDir mod return d.client.AddOfflineTaskURIs(ctx, uris, dstDir.GetID()) } +func (d *Open115) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + var envelope sdk.Resp[[]sdk.AddOfflineTaskURIsResp] + response, err := d.client.AuthRequestRaw(ctx, sdk.ApiAddOffline, http.MethodPost, nil, sdk.ReqWithForm(sdk.Form{ + "urls": strings.Join(uris, "\n"), + "wp_path_id": dstDir.GetID(), + })) + if response != nil { + _ = json.Unmarshal(response.Bytes(), &envelope) + } + hashes := make([]string, 0, len(envelope.Data)) + for _, item := range envelope.Data { + if item.State && item.InfoHash != "" { + hashes = append(hashes, item.InfoHash) + } + } + rawResponse := "" + if response != nil { + rawResponse = response.String() + } + return hashes, envelope.Data, rawResponse, err +} + func (d *Open115) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { return d.client.DeleteOfflineTask(ctx, infoHash, deleteFiles) } func (d *Open115) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + // 获取第一页 resp, err := d.client.OfflineTaskList(ctx, 1) if err != nil { return nil, err } + // 如果有多页，获取所有页面的任务 + if resp.PageCount > 1 { + for page := 2; page <= resp.PageCount; page++ { + pageResp, err := d.client.OfflineTaskList(ctx, int64(page)) + if err != nil { + return nil, err + } + resp.Tasks = append(resp.Tasks, pageResp.Tasks...) + } + } return resp, nil } diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go new file mode 100644 index 000000000..1d9664695 --- /dev/null +++ b/drivers/115_open/driver_test.go @@ -0,0 +1,639 @@ +package _115_open + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "testing" + "time" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type recordedRequest struct { + Path string + Form url.Values +} + +type rewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (t *rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + cloned := req.Clone(req.Context()) + cloned.URL.Scheme = t.target.Scheme + cloned.URL.Host = t.target.Host + return t.base.RoundTrip(cloned) +} + +func TestOpen115RemoveTrashUsesDelFileOnly(t *testing.T) { + driver, requests := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { + writeSDKSuccess(t, w, []string{"rb-123"}) + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete") + assertFormValue(t, requests()[0].Form, "file_ids", "file-1") + assertFormValue(t, requests()[0].Form, "parent_id", "dir-1") +} + +func TestOpen115RemoveDeleteUsesDelFileResponseIDWhenAvailable(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/del": + writeSDKSuccess(t, w, []string{"rb-123"}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del") + assertFormValue(t, requests()[1].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteFallsBackToRecycleBinLookup(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "file_size": "123", + "cid": "dir-1", + "sha1": "sha-demo", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteReturnsErrorWhenRecycleEntryMissing(t *testing.T) { + driver, _ := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + err := driver.Remove(context.Background(), obj) + if err == nil { + t.Fatalf("expected Remove to fail when recycle-bin entry is missing") + } + if !strings.Contains(err.Error(), "recycle bin entry not found") { + t.Fatalf("expected recycle-bin lookup error, got: %v", err) + } +} + +func TestOpen115DriverInfoIncludesRemoveWay(t *testing.T) { + info, ok := op.GetDriverInfoMap()["115 Open"] + if !ok { + t.Fatalf("115 Open driver info was not registered") + } + + for _, item := range info.Additional { + if item.Name != "remove_way" { + continue + } + if item.Type != "select" { + t.Fatalf("unexpected remove_way type: %q", item.Type) + } + if item.Options != "trash,delete" { + t.Fatalf("unexpected remove_way options: %q", item.Options) + } + if item.Default != "trash" { + t.Fatalf("unexpected remove_way default: %q", item.Default) + } + if !item.Required { + t.Fatalf("expected remove_way to be required") + } + return + } + + t.Fatalf("remove_way item not found in 115 Open driver info") +} + +func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { + t.Helper() + + var ( + mu sync.Mutex + requests []recordedRequest + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Fatalf("ParseForm failed: %v", err) + } + mu.Lock() + requests = append(requests, recordedRequest{ + Path: r.URL.Path, + Form: cloneValues(r.Form), + }) + mu.Unlock() + responder(w, r) + })) + t.Cleanup(server.Close) + + target, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("Parse server URL failed: %v", err) + } + + client := sdk.New(sdk.WithAccessToken("test-token")) + client.SetHttpClient(&http.Client{ + Transport: &rewriteTransport{ + target: target, + base: http.DefaultTransport, + }, + }) + + return &Open115{ + Addition: Addition{ + RemoveWay: removeWay, + PageSize: 1, + }, + client: client, + }, func() []recordedRequest { + mu.Lock() + defer mu.Unlock() + return append([]recordedRequest(nil), requests...) + } +} + +func writeSDKSuccess(t *testing.T, w http.ResponseWriter, data any) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": true, + "data": data, + }) +} + +func writeSDKError(t *testing.T, w http.ResponseWriter, code int64, message string) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": false, + "code": code, + "message": message, + }) +} + +func writeSDKResponse(t *testing.T, w http.ResponseWriter, payload map[string]any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(payload); err != nil { + t.Fatalf("Encode response failed: %v", err) + } +} + +func assertRequestPaths(t *testing.T, requests []recordedRequest, want ...string) { + t.Helper() + got := make([]string, 0, len(requests)) + for _, req := range requests { + got = append(got, req.Path) + } + if !slices.Equal(got, want) { + t.Fatalf("unexpected request paths: got %v want %v", got, want) + } +} + +func assertFormValue(t *testing.T, form url.Values, key, want string) { + t.Helper() + if got := form.Get(key); got != want { + t.Fatalf("unexpected form value for %s: got %q want %q", key, got, want) + } +} + +func cloneValues(src url.Values) url.Values { + dst := make(url.Values, len(src)) + for key, values := range src { + dst[key] = append([]string(nil), values...) + } + return dst +} + +// --- FlexString / numeric CID tests --- + +func TestFlexStringUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {"string value", `{"cid":"dir-1"}`, "dir-1"}, + {"integer value", `{"cid":3383942108160578280}`, "3383942108160578280"}, + {"large integer", `{"cid":9999999999999999999}`, "9999999999999999999"}, + {"zero", `{"cid":0}`, "0"}, + {"negative", `{"cid":-123}`, "-123"}, + {"float", `{"cid":1.5}`, "1.5"}, + {"empty string", `{"cid":""}`, ""}, + {"null", `{"cid":null}`, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v struct { + CID sdk.FlexString `json:"cid"` + } + if err := json.Unmarshal([]byte(tt.input), &v); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if got := string(v.CID); got != tt.want { + t.Fatalf("got %q, want %q", got, tt.want) + } + }) + } +} + +func TestFlexStringUnmarshalInvalid(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"boolean", `{"cid":true}`}, + {"array", `{"cid":[1]}`}, + {"object", `{"cid":{}}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v struct { + CID sdk.FlexString `json:"cid"` + } + if err := json.Unmarshal([]byte(tt.input), &v); err == nil { + t.Fatalf("expected error for input %s", tt.input) + } + }) + } +} + +func TestRbListRespUnmarshalCIDAsString(t *testing.T) { + raw := `{"id":"rb-1","file_name":"demo.txt","cid":"dir-1","file_size":"123"}` + var info sdk.RbListResp_FileInfo + if err := json.Unmarshal([]byte(raw), &info); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if string(info.CID) != "dir-1" { + t.Fatalf("got CID %q, want %q", string(info.CID), "dir-1") + } +} + +func TestRbListRespUnmarshalCIDAsNumber(t *testing.T) { + raw := `{"id":"rb-1","file_name":"MyFolder","cid":3383942108160578280,"file_size":"0"}` + var info sdk.RbListResp_FileInfo + if err := json.Unmarshal([]byte(raw), &info); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if string(info.CID) != "3383942108160578280" { + t.Fatalf("got CID %q, want %q", string(info.CID), "3383942108160578280") + } +} + +// --- matchRecycleBinEntry with numeric CID --- + +func TestMatchRecycleBinEntryDirMatchWithNumericCID(t *testing.T) { + // obj represents a directory with Pid (parent ID) as a large number + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-folder-1", + FileName: "MyFolder", + CID: sdk.FlexString("3383942108160578280"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match for directory with numeric CID, got nil") + } + if result.ID != "rb-folder-1" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-folder-1") + } +} + +func TestMatchRecycleBinEntryDirNoMatchWhenCIDWrong(t *testing.T) { + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-folder-1", + FileName: "MyFolder", + CID: sdk.FlexString("9999999999"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result != nil { + t.Fatalf("expected no match, got %+v", result) + } +} + +func TestMatchRecycleBinEntryFileSHA1MatchWithNumericCID(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "3383942108160578280", Fn: "video.mp4", Fc: "1", FS: 1024, Sha1: "abc123"} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "video.mp4", + CID: sdk.FlexString("3383942108160578280"), + SHA1: "ABC123", + FileSize: "1024", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via SHA1+CID, got nil") + } + if result.ID != "rb-file-1" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-file-1") + } +} + +func TestMatchRecycleBinEntryFileSHA1MatchByNameOnly(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "wrong-pid", Fn: "video.mp4", Fc: "1", FS: 1024, Sha1: "abc123"} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "video.mp4", + CID: sdk.FlexString("3383942108160578280"), + SHA1: "ABC123", + FileSize: "1024", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via SHA1+name, got nil") + } +} + +func TestMatchRecycleBinEntryFileNameSizeCIDMatch(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "3383942108160578280", Fn: "doc.pdf", Fc: "1", FS: 500} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "doc.pdf", + CID: sdk.FlexString("3383942108160578280"), + FileSize: "500", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via name+size+CID, got nil") + } +} + +func TestMatchRecycleBinEntryDirectIDMatch(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123} + files := map[string]sdk.RbListResp_FileInfo{ + "file-1": { + ID: "rb-123", + FileName: "demo.txt", + CID: sdk.FlexString("dir-1"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected direct ID match, got nil") + } + if result.ID != "rb-123" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-123") + } +} + +func TestMatchRecycleBinEntryEmptyFiles(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123} + result := matchRecycleBinEntry(obj, nil) + if result != nil { + t.Fatalf("expected nil for nil files, got %+v", result) + } + result = matchRecycleBinEntry(obj, map[string]sdk.RbListResp_FileInfo{}) + if result != nil { + t.Fatalf("expected nil for empty files, got %+v", result) + } +} + +// --- Full Remove flow with numeric CID in recycle bin --- + +func TestOpen115RemoveDeleteWithNumericCID(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"folder-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "folder-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-folder-1"}) + case "/open/rb/list": + // CID returned as number (the real bug scenario) + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-folder-1": map[string]any{ + "id": "rb-folder-1", + "file_name": "MyFolder", + "cid": 3383942108160578280, + "file_size": "0", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-folder-1") +} + +func TestOpen115RemoveDeleteWithStringCIDStillWorks(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + // CID returned as string (normal case) + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "cid": "dir-1", + "sha1": "sha-demo", + "file_size": "123", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteRetriesRecycleBinLookupUntilVisible(t *testing.T) { + oldAttempts, oldDelay := recycleBinLookupMaxAttempts, recycleBinLookupRetryDelay + recycleBinLookupMaxAttempts = 3 + recycleBinLookupRetryDelay = time.Millisecond + t.Cleanup(func() { + recycleBinLookupMaxAttempts = oldAttempts + recycleBinLookupRetryDelay = oldDelay + }) + + rbListCalls := 0 + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + rbListCalls++ + if rbListCalls < 3 { + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + return + } + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "cid": "dir-1", + "sha1": "sha-demo", + "file_size": "123", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + if rbListCalls != 3 { + t.Fatalf("rbListCalls = %d, want 3", rbListCalls) + } + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/list", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[5].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteStopsRetryWhenContextCancelled(t *testing.T) { + oldAttempts, oldDelay := recycleBinLookupMaxAttempts, recycleBinLookupRetryDelay + recycleBinLookupMaxAttempts = 5 + recycleBinLookupRetryDelay = 50 * time.Millisecond + t.Cleanup(func() { + recycleBinLookupMaxAttempts = oldAttempts + recycleBinLookupRetryDelay = oldDelay + }) + + driver, _ := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + writeSDKError(t, w, 404, "not found") + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + err := driver.Remove(ctx, obj) + if err == nil { + t.Fatalf("expected Remove to fail due to context cancellation") + } + if !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + t.Fatalf("expected context deadline exceeded, got: %v", err) + } +} diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go index ed908e2e6..0479ac1fd 100644 --- a/drivers/115_open/meta.go +++ b/drivers/115_open/meta.go @@ -11,6 +11,7 @@ type Addition struct { // define other OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete" default:"trash"` LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` PageSize int64 `json:"page_size" type:"number" default:"200" help:"list api per page size of 115open driver"` AccessToken string `json:"access_token" required:"true"` @@ -18,9 +19,10 @@ type Addition struct { } var config = driver.Config{ - Name: "115 Open", - DefaultRoot: "0", - LinkCacheMode: driver.LinkCacheUA, + Name: "115 Open", + DefaultRoot: "0", + ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index d02640e2c..6af4403cf 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "io" + "strings" "time" sdk "github.com/OpenListTeam/115-sdk-go" @@ -14,8 +15,19 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" ) +// isTokenExpiredError 检测是否为OSS凭证过期错误 +func isTokenExpiredError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "SecurityTokenExpired") || + strings.Contains(errStr, "InvalidAccessKeyId") +} + func calPartSize(fileSize int64) int64 { var partSize int64 = 20 * utils.MB if fileSize > partSize { @@ -71,11 +83,16 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp // } func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { - ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) - if err != nil { - return err + // 创建OSS客户端的辅助函数 + createBucket := func(token *sdk.UploadGetTokenResp) (*oss.Bucket, error) { + ossClient, err := netutil.NewOSSClient(token.Endpoint, token.AccessKeyId, token.AccessKeySecret, oss.SecurityToken(token.SecurityToken)) + if err != nil { + return nil, err + } + return ossClient.Bucket(initResp.Bucket) } - bucket, err := ossClient.Bucket(initResp.Bucket) + + bucket, err := createBucket(tokenResp) if err != nil { return err } @@ -120,7 +137,24 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), - retry.Delay(time.Second)) + retry.Delay(time.Second), + retry.OnRetry(func(n uint, err error) { + // 如果是凭证过期错误，在重试前刷新凭证并重建bucket + if isTokenExpiredError(err) { + log.Warnf("115 OSS token expired, refreshing token...") + if newToken, refreshErr := d.client.UploadGetToken(ctx); refreshErr == nil { + tokenResp = newToken + if newBucket, bucketErr := createBucket(tokenResp); bucketErr == nil { + bucket = newBucket + log.Infof("115 OSS token refreshed successfully") + } else { + log.Errorf("Failed to create new bucket with refreshed token: %v", bucketErr) + } + } else { + log.Errorf("Failed to refresh 115 OSS token: %v", refreshErr) + } + } + })) ss.FreeSectionReader(rd) if err != nil { return err diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index d12e02ec5..56669a674 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -2,8 +2,16 @@ package _115_open import ( "context" + "encoding/base32" + "encoding/hex" + "errors" "fmt" + "net/url" + "strconv" + "strings" + "time" + sdk "github.com/OpenListTeam/115-sdk-go" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -12,11 +20,34 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" + log "github.com/sirupsen/logrus" ) type Open115 struct { } +type offlineTaskClient interface { + OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) + DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error +} + +type offlineTaskDetailClient interface { + OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) +} + +type offlineTaskLimiter interface { + WaitLimit(ctx context.Context) error +} + +func waitOfflineTaskLimit(ctx context.Context, client offlineTaskClient) error { + limiter, ok := client.(offlineTaskLimiter) + if !ok { + return nil + } + return limiter.WaitLimit(ctx) +} + func (o *Open115) Name() string { return "115 Open" } @@ -68,15 +99,473 @@ func (o *Open115) AddURL(args *tool.AddUrlArgs) (string, error) { if err != nil { return "", err } + log.Infof("[115_open] AddURL start: temp_dir=%q actual_path=%q parent_id=%q parent_name=%q url=%q", args.TempDir, actualPath, parentDir.GetID(), parentDir.GetName(), args.Url) + logOfflineURLDetails("[115_open] AddURL input", args.Url) - hashs, err := driver115Open.OfflineDownload(ctx, []string{args.Url}, parentDir) - if err != nil || len(hashs) < 1 { - return "", fmt.Errorf("failed to add offline download task: %w", err) + hashs, err := addOfflineDownloadTask(ctx, driver115Open, args.Url, parentDir) + if err != nil { + return "", err + } + + if len(hashs) < 1 { + return "", fmt.Errorf("failed to add offline download task: no task hash returned") } return hashs[0], nil } +func addOfflineDownloadTask(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, error) { + parentID, parentName := "", "" + if parentDir != nil { + parentID = parentDir.GetID() + parentName = parentDir.GetName() + } + log.Infof("[115_open] addOfflineDownloadTask: parent_id=%q parent_name=%q url=%q", parentID, parentName, url) + logOfflineURLDetails("[115_open] addOfflineDownloadTask target", url) + if err := preCleanDuplicateOfflineTasks(ctx, client, url); err != nil { + return nil, err + } + hashs, addItems, rawResp, err := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] addOfflineDownloadTask first attempt result: hashes=%v err=%v add_items=%d", hashs, err, len(addItems)) + if err == nil { + return hashs, nil + } + if !isDuplicateOfflineTaskError(err) { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] duplicate offline task detected, trying cleanup before retry") + if rawResp != "" { + log.Infof("[115_open] duplicate add response: %s", rawResp) + } + for _, item := range addItems { + log.Infof("[115_open] duplicate add item: state=%v code=%d info_hash=%q url=%q", item.State, item.Code, item.InfoHash, item.URL) + logOfflineURLDetails("[115_open] duplicate add item url", item.URL) + if item.InfoHash == "" { + log.Infof("[115_open] skipping add-response duplicate item: empty info_hash") + continue + } + if item.URL != "" && !offlineTaskURLMatches(item.URL, url) { + log.Infof("[115_open] skipping add-response duplicate item: url mismatch") + continue + } + log.Infof("[115_open] deleting duplicate task directly from add response: info_hash=%s url=%s", item.InfoHash, item.URL) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + if deleteErr := client.DeleteOfflineTask(ctx, item.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete duplicate task from add response failed: info_hash=%s err=%v", item.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task from add response: %w", deleteErr) + } + log.Infof("[115_open] delete duplicate task from add response success: info_hash=%s", item.InfoHash) + waitForOfflineTaskRemoval(ctx, client, item.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after add-response delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after add-response delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] matched duplicate offline task: info_hash=%s, name=%s", task.InfoHash, task.Name) + log.Infof("[115_open] deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + log.Infof("[115_open] delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after matched delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after matched delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + log.Warnf("[115_open] duplicate offline task detected but no matching task found in offline list") + return nil, fmt.Errorf("failed to add offline download task: %w", err) +} + +func preCleanDuplicateOfflineTasks(ctx context.Context, client offlineTaskClient, url string) error { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + log.Warnf("[115_open] pre-add offline list failed: err=%v", listErr) + return nil + } + log.Infof("[115_open] pre-add offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + deleted := 0 + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] pre-add duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] pre-add duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] pre-add deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] pre-add delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + deleted++ + log.Infof("[115_open] pre-add delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + } + if deleted == 0 { + log.Infof("[115_open] pre-add duplicate scan found no matches") + } + return nil +} + +func offlineDownloadWithDetails(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, nil, "", err + } + if detailClient, ok := client.(offlineTaskDetailClient); ok { + return detailClient.OfflineDownloadWithDetails(ctx, []string{url}, parentDir) + } + hashs, err := client.OfflineDownload(ctx, []string{url}, parentDir) + return hashs, nil, "", err +} + +func isDuplicateOfflineTaskError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + return strings.Contains(errStr, "10008") || + strings.Contains(errStr, "重复") || + strings.Contains(errStr, "已存在") || + strings.Contains(errStr, "duplicate") +} + +func offlineTaskURLMatches(taskURL string, rawURL string) bool { + taskVariants := normalizedOfflineTaskURLVariants(taskURL) + rawVariants := normalizedOfflineTaskURLVariants(rawURL) + for candidate := range taskVariants { + if _, ok := rawVariants[candidate]; ok { + return true + } + } + return false +} + +func offlineTaskMatches(task sdk.OfflineTask, rawURL string) bool { + matched, _ := offlineTaskMatchReason(task, rawURL) + return matched +} + +func offlineTaskMatchReason(task sdk.OfflineTask, rawURL string) (bool, string) { + if offlineTaskURLMatches(task.URL, rawURL) { + return true, "url variants matched" + } + if httpURLMatches(task.URL, rawURL) { + return true, "http url host+path matched" + } + rawMagnet := parseMagnetBTIH(rawURL) + if rawMagnet != "" { + taskHash := normalizeInfoHash(task.InfoHash) + if taskHash != "" && taskHash == rawMagnet { + return true, "task info_hash matched raw magnet" + } + taskURLHash := parseMagnetBTIH(task.URL) + if taskURLHash != "" && taskURLHash == rawMagnet { + return true, "task url magnet hash matched" + } + return false, fmt.Sprintf("task magnet hash mismatch: task_info_hash=%q task_url_hash=%q raw_hash=%q", taskHash, taskURLHash, rawMagnet) + } + taskED2K, rawED2K := parseED2KLink(task.URL), parseED2KLink(rawURL) + if taskED2K != nil && rawED2K != nil { + if taskED2K.Hash == rawED2K.Hash { + if taskED2K.Size == rawED2K.Size { + return true, "task url ed2k hash matched" + } + return true, "task url ed2k hash matched despite size mismatch" + } + return false, fmt.Sprintf("task url ed2k mismatch: task=%s raw=%s", taskED2K.String(), rawED2K.String()) + } + if rawED2K == nil { + return false, "raw url is not ed2k and url variants did not match" + } + if normalizeOfflineTaskURL(task.InfoHash) == rawED2K.Hash { + if task.Size == rawED2K.Size { + return true, "task info_hash matched raw ed2k" + } + return true, "task info_hash matched raw ed2k despite size mismatch" + } + taskName := normalizeOfflineTaskURL(task.Name) + if taskName == normalizeOfflineTaskURL(rawED2K.Name) && task.Size == rawED2K.Size { + return true, "task name and size matched raw ed2k" + } + return false, fmt.Sprintf("task name/hash/size mismatch: task_name=%q raw_name=%q task_info_hash=%q raw_hash=%q task_size=%d raw_size=%d", taskName, normalizeOfflineTaskURL(rawED2K.Name), normalizeOfflineTaskURL(task.InfoHash), rawED2K.Hash, task.Size, rawED2K.Size) +} + +func normalizedOfflineTaskURLVariants(raw string) map[string]struct{} { + variants := map[string]struct{}{} + queue := []string{raw} + for len(queue) > 0 { + current := normalizeOfflineTaskURL(queue[0]) + queue = queue[1:] + if current == "" { + continue + } + if _, ok := variants[current]; ok { + continue + } + variants[current] = struct{}{} + if decoded, err := url.QueryUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + if decoded, err := url.PathUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + } + return variants +} + +func httpURLMatches(taskURL, rawURL string) bool { + taskNormalized := normalizeHTTPURL(taskURL) + rawNormalized := normalizeHTTPURL(rawURL) + if taskNormalized == "" || rawNormalized == "" { + return false + } + return taskNormalized == rawNormalized +} + +func normalizeHTTPURL(raw string) string { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed == nil { + return "" + } + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return "" + } + host := strings.ToLower(parsed.Host) + if host == "" { + return "" + } + path := strings.ToLower(parsed.Path) + path = strings.TrimSuffix(path, "/") + if path == "" { + path = "/" + } + return fmt.Sprintf("%s://%s%s", scheme, host, path) +} + +func normalizeOfflineTaskURL(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimSuffix(normalized, "/") + return strings.ToLower(normalized) +} + +type ed2kLink struct { + Name string + Size int64 + Hash string +} + +func parseED2KLink(raw string) *ed2kLink { + normalized := strings.TrimSpace(raw) + if !strings.HasPrefix(strings.ToLower(normalized), "ed2k://|file|") { + return nil + } + parts := strings.Split(normalized, "|") + if len(parts) < 6 { + return nil + } + name, err := url.PathUnescape(parts[2]) + if err != nil { + name = parts[2] + } + size, err := strconv.ParseInt(parts[3], 10, 64) + if err != nil { + return nil + } + return &ed2kLink{ + Name: normalizeOfflineTaskURL(name), + Size: size, + Hash: normalizeOfflineTaskURL(parts[4]), + } +} + +func parseMagnetBTIH(raw string) string { + if raw == "" { + return "" + } + lower := strings.ToLower(raw) + idx := strings.Index(lower, "btih:") + if idx == -1 { + return "" + } + candidate := raw[idx+len("btih:"):] + if candidate == "" { + return "" + } + for i, ch := range candidate { + if ch == '&' || ch == '#' || ch == '/' { + candidate = candidate[:i] + break + } + } + candidate = strings.TrimSpace(candidate) + if candidate == "" { + return "" + } + if decoded, err := url.QueryUnescape(candidate); err == nil { + candidate = decoded + } + return normalizeInfoHash(candidate) +} + +func normalizeInfoHash(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimPrefix(strings.ToLower(normalized), "urn:btih:") + if normalized == "" { + return "" + } + if len(normalized) == 40 && isHexString(normalized) { + return normalized + } + if len(normalized) == 32 && isBase32String(normalized) { + decoded, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(normalized)) + if err == nil && len(decoded) == 20 { + return hex.EncodeToString(decoded) + } + } + return normalized +} + +func isHexString(value string) bool { + for _, ch := range value { + if (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') { + continue + } + return false + } + return true +} + +func isBase32String(value string) bool { + for _, ch := range value { + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '2' && ch <= '7') { + continue + } + return false + } + return true +} + +func (e *ed2kLink) Equal(other *ed2kLink) bool { + if e == nil || other == nil { + return false + } + return e.Name == other.Name && e.Size == other.Size && e.Hash == other.Hash +} + +func (e *ed2kLink) String() string { + if e == nil { + return "" + } + return fmt.Sprintf("name=%q size=%d hash=%q", e.Name, e.Size, e.Hash) +} + +func logOfflineURLDetails(prefix string, raw string) { + if raw == "" { + log.Infof("%s details: raw is empty", prefix) + return + } + variants := normalizedOfflineTaskURLVariants(raw) + log.Infof("%s details: raw=%q normalized_variants=%v", prefix, raw, mapKeys(variants)) + if parsedMagnet := parseMagnetBTIH(raw); parsedMagnet != "" { + log.Infof("%s details: parsed_magnet_hash=%s", prefix, parsedMagnet) + } + if parsed := parseED2KLink(raw); parsed != nil { + log.Infof("%s details: parsed_ed2k=%s", prefix, parsed.String()) + } +} + +func mapKeys(values map[string]struct{}) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + return keys +} + +func waitForOfflineTaskRemoval(ctx context.Context, client offlineTaskClient, infoHash string) { + const maxChecks = 3 + for attempt := 1; attempt <= maxChecks; attempt++ { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + log.Warnf("[115_open] post-delete wait limit failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } + taskList, err := client.OfflineList(ctx) + if err != nil { + log.Warnf("[115_open] post-delete check failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } + stillExists := false + taskStatus := -999 + taskName := "" + for _, task := range taskList.Tasks { + if normalizeOfflineTaskURL(task.InfoHash) != normalizeOfflineTaskURL(infoHash) { + continue + } + stillExists = true + taskStatus = task.Status + taskName = task.Name + break + } + log.Infof("[115_open] post-delete check: info_hash=%s attempt=%d exists=%v status=%d name=%q task_count=%d", infoHash, attempt, stillExists, taskStatus, taskName, len(taskList.Tasks)) + if !stillExists { + return + } + if attempt < maxChecks { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } + } +} + func (o *Open115) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { @@ -124,13 +613,15 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = errors.New(t.GetStatus()) } return s, nil } } - s.Err = fmt.Errorf("the task has been deleted") - return nil, nil + // 任务不在列表中，可能已完成或被删除 + s.Progress = 100 + s.Completed = true + return s, nil } var _ tool.Tool = (*Open115)(nil) diff --git a/internal/offline_download/115_open/client_test.go b/internal/offline_download/115_open/client_test.go new file mode 100644 index 000000000..2e8b94778 --- /dev/null +++ b/internal/offline_download/115_open/client_test.go @@ -0,0 +1,616 @@ +package _115_open + +import ( + "context" + "fmt" + "strings" + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type mockOfflineTaskClient struct { + offlineDownloadFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + offlineDownloadWithDetailsFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) + offlineListFunc func(ctx context.Context) (*sdk.OfflineTaskListResp, error) + deleteOfflineFunc func(ctx context.Context, infoHash string, deleteFiles bool) error + waitLimitFunc func(ctx context.Context) error + waitLimitCalls int +} + +func (m *mockOfflineTaskClient) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return m.offlineDownloadFunc(ctx, uris, dstDir) +} + +func (m *mockOfflineTaskClient) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if m.offlineDownloadWithDetailsFunc == nil { + hashes, err := m.OfflineDownload(ctx, uris, dstDir) + return hashes, nil, "", err + } + return m.offlineDownloadWithDetailsFunc(ctx, uris, dstDir) +} + +func (m *mockOfflineTaskClient) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return m.offlineListFunc(ctx) +} + +func (m *mockOfflineTaskClient) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { + return m.deleteOfflineFunc(ctx, infoHash, deleteFiles) +} + +func (m *mockOfflineTaskClient) WaitLimit(ctx context.Context) error { + m.waitLimitCalls++ + if m.waitLimitFunc != nil { + return m.waitLimitFunc(ctx) + } + return nil +} + +func TestIsDuplicateOfflineTaskError(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "code 10008", err: fmt.Errorf("code: 10008"), want: true}, + {name: "chinese duplicate", err: fmt.Errorf("任务重复"), want: true}, + {name: "already exists", err: fmt.Errorf("任务已存在"), want: true}, + {name: "english duplicate", err: fmt.Errorf("duplicate task"), want: true}, + {name: "other", err: fmt.Errorf("network timeout"), want: false}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := isDuplicateOfflineTaskError(tc.err); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } +} + +func TestOfflineTaskURLMatches(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + taskURL string + rawURL string + want bool + }{ + { + name: "exact match", + taskURL: "ed2k://|file|test.avi|123|ABC|/", + rawURL: "ed2k://|file|test.avi|123|ABC|/", + want: true, + }, + { + name: "percent encoded file name", + taskURL: "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + rawURL: "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + want: true, + }, + { + name: "case and trailing slash normalized", + taskURL: "ED2K://|FILE|TEST.AVI|123|ABC|", + rawURL: "ed2k://|file|test.avi|123|abc|/", + want: true, + }, + { + name: "different link", + taskURL: "ed2k://|file|a.avi|123|ABC|/", + rawURL: "ed2k://|file|b.avi|123|ABC|/", + want: false, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := offlineTaskURLMatches(tc.taskURL, tc.rawURL); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } +} + +func TestOfflineTaskMatches(t *testing.T) { + t.Parallel() + + t.Run("match ed2k by parsed fields when task url differs", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + InfoHash: "server-task-hash", + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected task to match by ed2k parsed fields") + } + }) + + t.Run("do not match different ed2k size", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1, + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if offlineTaskMatches(task, rawURL) { + t.Fatal("expected task not to match") + } + }) + + t.Run("magnet still matches by url", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: rawURL, + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by url") + } + }) + + t.Run("match magnet by btih despite noisy tracker", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test&tr=%3C!DOCTYPE%20html%3E", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by btih") + } + }) + + t.Run("match http by host and path", func(t *testing.T) { + t.Parallel() + + rawURL := "https://example.com/files/test.mp4" + task := sdk.OfflineTask{ + URL: "https://EXAMPLE.com/files/test.mp4?token=abc", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected http task to match by host and path") + } + }) +} + +func TestAddOfflineDownloadTask(t *testing.T) { + t.Parallel() + + const ( + testURL = "https://example.com/test.torrent" + firstHash = "hash-1" + staleHash = "hash-stale" + deleteError = "delete failed" + ) + + t.Run("success on first try", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("wait limit applied for add flow", func(t *testing.T) { + t.Parallel() + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return nil + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if client.waitLimitCalls < 2 { + t.Fatalf("want wait limit calls >= 2, got %d", client.waitLimitCalls) + } + }) + + t.Run("delete duplicate and retry", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: testURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate magnet and retry", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + magnetURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: magnetURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, magnetURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry with decoded ed2k url", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + decodedURL := "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: decodedURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry with empty task url but matching name and size", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + { + InfoHash: staleHash, + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + }, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate directly from add response info hash", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadWithDetailsFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + callCount++ + if callCount == 1 { + return nil, []sdk.AddOfflineTaskURIsResp{ + {InfoHash: staleHash, URL: testURL}, + }, `{"state":false,"code":10008,"message":"任务已存在","data":[{"info_hash":"hash-stale","url":"` + testURL + `"}]}`, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil, "", nil + }, + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("unexpected fallback OfflineDownload call") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 1 { + t.Fatalf("want 1 delete attempt, got %d", deleteCount) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("duplicate delete failure", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("duplicate task") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: testURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return fmt.Errorf(deleteError) + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), deleteError) { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("non duplicate error is returned", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("network timeout") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "network timeout") { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) +} + +func TestOpen115BasicMethods(t *testing.T) { + t.Parallel() + + o := &Open115{} + + if o.Name() != "115 Open" { + t.Fatalf("unexpected name: %s", o.Name()) + } + if o.Items() != nil { + t.Fatal("Items should return nil") + } + msg, err := o.Init() + if err != nil { + t.Fatalf("unexpected init error: %v", err) + } + if msg != "ok" { + t.Fatalf("unexpected init message: %s", msg) + } +} From 9a33933824ea2b48c270912be0b545e925e68403 Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:37 +0800 Subject: [PATCH 82/86] feat(offline_download): multi-page task retrieval and task limit wait mechanism - Optimize task list retrieval with multi-page support and status hints - Add task limit wait mechanism to optimize offline download task processing --- internal/offline_download/tool/download.go | 22 ++--- .../offline_download/tool/download_test.go | 90 +++++++++++++++++++ 2 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 internal/offline_download/tool/download_test.go diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 5ee6ef4ff..477c26a51 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -32,6 +32,8 @@ type DownloadTask struct { callStatusRetried int } +var completedOfflineTaskCleanupDelay = time.Second + func (t *DownloadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) @@ -97,16 +99,7 @@ outer: if t.tool.Name() == "ThunderX" { return nil } - if t.tool.Name() == "115 Cloud" { - // hack for 115 - <-time.After(time.Second * 1) - err := t.tool.Remove(t) - if err != nil { - log.Errorln(err.Error()) - } - return nil - } - if t.tool.Name() == "115 Open" { + if t.tool.Name() == "115 Cloud" || t.tool.Name() == "115 Open" { return nil } if t.tool.Name() == "123 Open" { @@ -147,7 +140,7 @@ func (t *DownloadTask) Update() (bool, error) { if err != nil { t.callStatusRetried++ log.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) - if t.callStatusRetried > 5 { + if t.callStatusRetried > 10 { return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) } return false, nil @@ -163,6 +156,13 @@ func (t *DownloadTask) Update() (bool, error) { } // if download completed if info.Completed { + // For 115, remove offline task record before transfer so it gets cleaned up even if transfer fails + if t.tool.Name() == "115 Cloud" || t.tool.Name() == "115 Open" { + <-time.After(completedOfflineTaskCleanupDelay) + if removeErr := t.tool.Remove(t); removeErr != nil { + log.Errorln(removeErr.Error()) + } + } err := t.Transfer() return true, errors.WithMessage(err, "failed to transfer file") } diff --git a/internal/offline_download/tool/download_test.go b/internal/offline_download/tool/download_test.go new file mode 100644 index 000000000..5303d35aa --- /dev/null +++ b/internal/offline_download/tool/download_test.go @@ -0,0 +1,90 @@ +package tool + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + task2 "github.com/OpenListTeam/OpenList/v4/internal/task" +) + +type mockTool struct { + name string + addURLFunc func(args *AddUrlArgs) (string, error) + removeFunc func(task *DownloadTask) error + statusFunc func(task *DownloadTask) (*Status, error) + runFunc func(task *DownloadTask) error +} + +func (m *mockTool) Name() string { return m.name } + +func (m *mockTool) Items() []model.SettingItem { return nil } + +func (m *mockTool) Init() (string, error) { return "ok", nil } + +func (m *mockTool) IsReady() bool { return true } + +func (m *mockTool) AddURL(args *AddUrlArgs) (string, error) { + return m.addURLFunc(args) +} + +func (m *mockTool) Remove(task *DownloadTask) error { + return m.removeFunc(task) +} + +func (m *mockTool) Status(task *DownloadTask) (*Status, error) { + return m.statusFunc(task) +} + +func (m *mockTool) Run(task *DownloadTask) error { + return m.runFunc(task) +} + +func TestDownloadTaskRun_RemovesCompleted115OpenRecord(t *testing.T) { + previousDelay := completedOfflineTaskCleanupDelay + completedOfflineTaskCleanupDelay = 0 + defer func() { + completedOfflineTaskCleanupDelay = previousDelay + }() + + removeCount := 0 + tool := &mockTool{ + name: "115 Open", + addURLFunc: func(args *AddUrlArgs) (string, error) { + return "gid-1", nil + }, + removeFunc: func(task *DownloadTask) error { + removeCount++ + if task.GID != "gid-1" { + t.Fatalf("unexpected gid: %s", task.GID) + } + return nil + }, + statusFunc: func(task *DownloadTask) (*Status, error) { + return &Status{ + Completed: true, + Status: "completed", + }, nil + }, + runFunc: func(task *DownloadTask) error { + return errs.NotSupport + }, + } + + task := &DownloadTask{ + TaskExtension: task2.TaskExtension{}, + Url: "https://example.com/test.torrent", + DstDirPath: "/115", + TempDir: "/115", + tool: tool, + } + task.SetCtx(context.Background()) + + if err := task.Run(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if removeCount != 1 { + t.Fatalf("want 1 cleanup remove, got %d", removeCount) + } +} From 2315749a448d517de1d21ec00b1da82991578e7a Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:44 +0800 Subject: [PATCH 83/86] feat(drivers): baidu streaming upload, quark rate-limit/retry, 123pan etag fix - Implement streaming upload for Baidu Netdisk - Add rate limiting and retry logic for quark_open, optimize upload handling - Fix quark shard size adjustment for oversized errors - Fix file copy failure to 123pan due to incorrect etag - Fix aliyundrive upload hash calculation --- drivers/123_open/driver.go | 48 ++++- drivers/aliyundrive_open/upload.go | 41 ++-- drivers/baidu_netdisk/driver.go | 214 +++----------------- drivers/baidu_netdisk/meta.go | 4 +- drivers/baidu_netdisk/upload.go | 311 +++++++++++++++++++++++++++++ drivers/baidu_netdisk/util.go | 19 +- drivers/quark_open/driver.go | 262 ++++++++++++++++++++---- drivers/quark_open/meta.go | 4 +- drivers/quark_open/util.go | 94 ++++++++- 9 files changed, 738 insertions(+), 259 deletions(-) create mode 100644 drivers/baidu_netdisk/upload.go diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 78ff272b9..9adb1aed0 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -175,7 +175,7 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { } func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // 1. 创建文件 + // 1. 准备参数 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { @@ -197,14 +197,49 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } + // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) - if len(etag) < utils.MD5.Width { + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 + if len(etag) >= utils.MD5.Width { + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return nil, err + } + if createResp.Data.Reuse && createResp.Data.FileID != 0 { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: createResp.Data.FileID, + Type: 2, + Etag: etag, + }, nil + } + // 秒传失败，继续后续流程 + } + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(etag) < utils.MD5.Width { + etag, err = stream.StreamHashFile(file, utils.MD5, 100, &up) + if err != nil { + return nil, err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 秒传失败或没有 hash，缓存整个文件并计算 MD5 _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) if err != nil { return nil, err } } + + // 2. 创建上传任务（或再次尝试秒传） createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err @@ -223,13 +258,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } - // 2. 上传分片 - err = d.Upload(ctx, file, createResp, up) + // 3. 上传分片 + uploadProgress := func(p float64) { + up(40 + p*0.6) + } + err = d.Upload(ctx, file, createResp, uploadProgress) if err != nil { return nil, err } - // 3. 上传完毕 + // 4. 合并分片/完成上传 for range 60 { uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) // 返回错误代码未知，如：20103，文档也没有具体说 diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index a4a6c1de1..5f02c75f5 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -163,21 +163,29 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m } count := int(math.Ceil(float64(stream.GetSize()) / float64(partSize))) createData["part_info_list"] = makePartInfos(count) + + // 检查是否是可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + // rapid upload rapidUpload := !stream.IsForceStreamUpload() && stream.GetSize() > 100*utils.KB && d.RapidUpload if rapidUpload { log.Debugf("[aliyundrive_open] start cal pre_hash") - // read 1024 bytes to calculate pre hash - reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) - if err != nil { - return nil, err - } - hash, err := utils.HashReader(utils.SHA1, reader) - if err != nil { - return nil, err + // 优先使用预计算的 pre_hash + preHash := stream.GetHash().GetHash(utils.PRE_HASH) + if len(preHash) != utils.PRE_HASH.Width { + // 没有预计算的 pre_hash，使用 RangeRead 计算 + reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) + if err != nil { + return nil, err + } + preHash, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return nil, err + } } createData["size"] = stream.GetSize() - createData["pre_hash"] = hash + createData["pre_hash"] = preHash } var createResp CreateResp _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { @@ -191,9 +199,18 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m hash := stream.GetHash().GetHash(utils.SHA1) if len(hash) != utils.SHA1.Width { - _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) - if err != nil { - return nil, err + if isSeekable { + // 可重复读取的流，使用 StreamHashFile（RangeRead），不缓存 + hash, err = streamPkg.StreamHashFile(stream, utils.SHA1, 100, &up) + if err != nil { + return nil, err + } + } else { + // 不可重复读取的流，缓存并计算 + _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) + if err != nil { + return nil, err + } } } diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index fe77aca38..474dd2b98 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -1,30 +1,18 @@ package baidu_netdisk import ( - "bytes" "context" - "crypto/md5" - "encoding/hex" "errors" - "io" - "mime/multipart" - "net/http" "net/url" - "os" stdpath "path" "strconv" - "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/internal/net" - "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/avast/retry-go" log "github.com/sirupsen/logrus" ) @@ -37,6 +25,7 @@ type BaiduNetdisk struct { } var ErrUploadIDExpired = errors.New("uploadid expired") +var ErrUploadURLExpired = errors.New("upload url expired or unavailable") func (d *BaiduNetdisk) Config() driver.Config { return config @@ -199,80 +188,26 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return newObj, nil } - var ( - cache = stream.GetFile() - tmpF *os.File - err error - ) - if cache == nil { - tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - defer func() { - _ = tmpF.Close() - _ = os.Remove(tmpF.Name()) - }() - cache = tmpF - } - streamSize := stream.GetSize() sliceSize := d.getSliceSize(streamSize) count := 1 if streamSize > sliceSize { count = int((streamSize + sliceSize - 1) / sliceSize) } - lastBlockSize := streamSize % sliceSize - if lastBlockSize == 0 { - lastBlockSize = sliceSize - } - - // cal md5 for first 256k data - const SliceSize int64 = 256 * utils.KB - blockList := make([]string, 0, count) - byteSize := sliceSize - fileMd5H := md5.New() - sliceMd5H := md5.New() - sliceMd5H2 := md5.New() - slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) - writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} - if tmpF != nil { - writers = append(writers, tmpF) - } - written := int64(0) - for i := 1; i <= count; i++ { - if utils.IsCanceled(ctx) { - return nil, ctx.Err() - } - if i == count { - byteSize = lastBlockSize - } - n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) - written += n - if err != nil && err != io.EOF { - return nil, err - } - blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) - sliceMd5H.Reset() - } - if tmpF != nil { - if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) - } - _, err = tmpF.Seek(0, io.SeekStart) - if err != nil { - return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") - } - } - contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) - sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) - blockListStr, _ := utils.Json.MarshalToString(blockList) path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 尝试读取已保存进度 + // step.1 流式计算MD5哈希值（使用 RangeRead，不会消耗流） + contentMd5, sliceMd5, blockList, err := d.calculateHashesStream(ctx, stream, sliceSize, &up) + if err != nil { + return nil, err + } + + blockListStr, _ := utils.Json.MarshalToString(blockList) + + // step.2 尝试读取已保存进度或执行预上传 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { // 没有进度，走预上传 @@ -288,6 +223,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return fileToObj(precreateResp.File), nil } } + ensureUploadURL := func() { if precreateResp.UploadURL != "" { return @@ -295,58 +231,20 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) } - // step.2 上传分片 + // step.3 流式上传分片 + // 创建 StreamSectionReader 用于上传 + ss, err := streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } + uploadLoop: for range 2 { // 获取上传域名 ensureUploadURL() - // 并发上传 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(UPLOAD_RETRY_COUNT), - retry.Delay(UPLOAD_RETRY_WAIT_TIME), - retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), - retry.DelayType(retry.BackOffDelay), - retry.RetryIf(func(err error) bool { - return !errors.Is(err, ErrUploadIDExpired) - }), - retry.LastErrorOnly(true)) - - totalParts := len(precreateResp.BlockList) - - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break - } - if partseq < 0 { - continue - } - i, partseq := i, partseq - offset, size := int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - size = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), - } - section := io.NewSectionReader(cache, offset, size) - err := d.uploadSlice(ctx, precreateResp.UploadURL, params, stream.GetName(), section) - if err != nil { - return err - } - precreateResp.BlockList[i] = -1 - progress := float64(threadG.Success()+1) * 100 / float64(totalParts+1) - up(progress) - return nil - }) - } - err = threadG.Wait() + // 流式并发上传 + err = d.uploadChunksStream(ctx, ss, stream, precreateResp, path, sliceSize, count, up) if err == nil { break uploadLoop } @@ -372,13 +270,19 @@ uploadLoop: precreateResp.UploadURL = "" // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + + // 尝试重新创建 StreamSectionReader（如果流支持重新读取） + ss, err = streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } continue uploadLoop } return nil, err } defer up(100) - // step.3 创建文件 + // step.4 创建文件 var newFile File _, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) if err != nil { @@ -427,68 +331,6 @@ func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize in return &precreateResp, nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file *io.SectionReader) error { - b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) - mw := multipart.NewWriter(b) - _, err := mw.CreateFormFile("file", fileName) - if err != nil { - return err - } - headSize := b.Len() - err = mw.Close() - if err != nil { - return err - } - head := bytes.NewReader(b.Bytes()[:headSize]) - tail := bytes.NewReader(b.Bytes()[headSize:]) - rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) - if err != nil { - return err - } - query := req.URL.Query() - for k, v := range params { - query.Set(k, v) - } - req.URL.RawQuery = query.Encode() - req.Header.Set("Content-Type", mw.FormDataContentType()) - req.ContentLength = int64(b.Len()) + file.Size() - - client := net.NewHttpClient() - if d.UploadSliceTimeout > 0 { - client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) - } else { - client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT - } - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - b.Reset() - _, err = b.ReadFrom(resp.Body) - if err != nil { - return err - } - body := b.Bytes() - respStr := string(body) - log.Debugln(respStr) - lower := strings.ToLower(respStr) - // 合并 uploadid 过期检测逻辑 - if strings.Contains(lower, "uploadid") && - (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { - return ErrUploadIDExpired - } - - errCode := utils.Json.Get(body, "error_code").ToInt() - errNo := utils.Json.Get(body, "errno").ToInt() - if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) - } - return nil -} - func (d *BaiduNetdisk) GetDetails(ctx context.Context) (*model.StorageDetails, error) { du, err := d.quota(ctx) if err != nil { diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 3f3bed022..499fcd8a8 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -31,8 +31,8 @@ type Addition struct { const ( UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) - DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 60 // 上传分片请求默认超时时间 - UPLOAD_RETRY_COUNT = 3 + DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 180 // 上传分片请求默认超时时间（增加到3分钟以应对慢速网络） + UPLOAD_RETRY_COUNT = 5 // 增加重试次数以提高成功率 UPLOAD_RETRY_WAIT_TIME = time.Second * 1 UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 ) diff --git a/drivers/baidu_netdisk/upload.go b/drivers/baidu_netdisk/upload.go new file mode 100644 index 000000000..c160c3a9e --- /dev/null +++ b/drivers/baidu_netdisk/upload.go @@ -0,0 +1,311 @@ +package baidu_netdisk + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "errors" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" +) + +// calculateHashesStream 流式计算文件的MD5哈希值 +// 返回：文件MD5、前256KB的MD5、每个分片的MD5列表 +// 注意：此函数使用 RangeRead 读取数据，不会消耗流 +func (d *BaiduNetdisk) calculateHashesStream( + ctx context.Context, + stream model.FileStreamer, + sliceSize int64, + up *driver.UpdateProgress, +) (contentMd5 string, sliceMd5 string, blockList []string, err error) { + streamSize := stream.GetSize() + count := 1 + if streamSize > sliceSize { + count = int((streamSize + sliceSize - 1) / sliceSize) + } + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 前256KB的MD5 + const SliceSize int64 = 256 * utils.KB + blockList = make([]string, 0, count) + fileMd5H := md5.New() + sliceMd5H2 := md5.New() + sliceWritten := int64(0) + + // 使用固定大小的缓冲区进行流式哈希计算 + // 这样可以利用 readFullWithRangeRead 的链接刷新逻辑 + const chunkSize = 10 * 1024 * 1024 // 10MB per chunk + buf := make([]byte, chunkSize) + + for i := 0; i < count; i++ { + if utils.IsCanceled(ctx) { + return "", "", nil, ctx.Err() + } + + offset := int64(i) * sliceSize + length := sliceSize + if i == count-1 { + length = lastBlockSize + } + + // 计算分片MD5 + sliceMd5Calc := md5.New() + + // 分块读取并计算哈希 + var sliceOffset int64 = 0 + for sliceOffset < length { + readSize := chunkSize + if length-sliceOffset < int64(chunkSize) { + readSize = int(length - sliceOffset) + } + + // 使用 readFullWithRangeRead 读取数据，自动处理链接刷新 + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset+sliceOffset) + if err != nil { + return "", "", nil, err + } + + // 同时写入多个哈希计算器 + fileMd5H.Write(buf[:n]) + sliceMd5Calc.Write(buf[:n]) + if sliceWritten < SliceSize { + remaining := SliceSize - sliceWritten + if int64(n) > remaining { + sliceMd5H2.Write(buf[:remaining]) + sliceWritten += remaining + } else { + sliceMd5H2.Write(buf[:n]) + sliceWritten += int64(n) + } + } + + sliceOffset += int64(n) + } + + blockList = append(blockList, hex.EncodeToString(sliceMd5Calc.Sum(nil))) + + // 更新进度（哈希计算占总进度的一小部分） + if up != nil { + progress := float64(i+1) * 10 / float64(count) + (*up)(progress) + } + } + + return hex.EncodeToString(fileMd5H.Sum(nil)), + hex.EncodeToString(sliceMd5H2.Sum(nil)), + blockList, nil +} + +// uploadChunksStream 流式上传所有分片 +func (d *BaiduNetdisk) uploadChunksStream( + ctx context.Context, + ss streamPkg.StreamSectionReaderIF, + stream model.FileStreamer, + precreateResp *PrecreateResp, + path string, + sliceSize int64, + count int, + up driver.UpdateProgress, +) error { + streamSize := stream.GetSize() + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 使用 OrderedGroup 保证 Before 阶段有序 + thread := min(d.uploadThread, len(precreateResp.BlockList)) + threadG, upCtx := errgroup.NewOrderedGroupWithContext(ctx, thread, + retry.Attempts(UPLOAD_RETRY_COUNT), + retry.Delay(UPLOAD_RETRY_WAIT_TIME), + retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), + retry.DelayType(retry.BackOffDelay), + retry.RetryIf(func(err error) bool { + return !errors.Is(err, ErrUploadIDExpired) + }), + retry.OnRetry(func(n uint, err error) { + // 重试前检测是否需要刷新上传 URL + if errors.Is(err, ErrUploadURLExpired) { + log.Infof("[baidu_netdisk] refreshing upload URL due to error: %v", err) + precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) + } + }), + retry.LastErrorOnly(true)) + + totalParts := len(precreateResp.BlockList) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) { + break + } + if partseq < 0 { + continue + } + + i, partseq := i, partseq + offset := int64(partseq) * sliceSize + size := sliceSize + if partseq+1 == count { + size = lastBlockSize + } + + var reader io.ReadSeeker + + threadG.GoWithLifecycle(errgroup.Lifecycle{ + Before: func(ctx context.Context) error { + var err error + reader, err = ss.GetSectionReader(offset, size) + return err + }, + Do: func(ctx context.Context) error { + reader.Seek(0, io.SeekStart) + err := d.uploadSliceStream(ctx, precreateResp.UploadURL, path, + precreateResp.Uploadid, partseq, stream.GetName(), reader, size) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 进度从10%开始（前10%是哈希计算） + progress := 10 + float64(threadG.Success()+1)*90/float64(totalParts+1) + up(progress) + return nil + }, + After: func(err error) { + ss.FreeSectionReader(reader) + }, + }) + } + + return threadG.Wait() +} + +// uploadSliceStream 上传单个分片（接受io.ReadSeeker） +func (d *BaiduNetdisk) uploadSliceStream( + ctx context.Context, + uploadUrl string, + path string, + uploadid string, + partseq int, + fileName string, + reader io.ReadSeeker, + size int64, +) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": uploadid, + "partseq": strconv.Itoa(partseq), + } + + b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) + mw := multipart.NewWriter(b) + _, err := mw.CreateFormFile("file", fileName) + if err != nil { + return err + } + headSize := b.Len() + err = mw.Close() + if err != nil { + return err + } + head := bytes.NewReader(b.Bytes()[:headSize]) + tail := bytes.NewReader(b.Bytes()[headSize:]) + rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) + if err != nil { + return err + } + query := req.URL.Query() + for k, v := range params { + query.Set(k, v) + } + req.URL.RawQuery = query.Encode() + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(b.Len()) + size + + client := net.NewHttpClient() + if d.UploadSliceTimeout > 0 { + client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) + } else { + client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT + } + resp, err := client.Do(req) + if err != nil { + // 检测超时或网络错误，标记需要刷新上传 URL + if isUploadURLError(err) { + log.Warnf("[baidu_netdisk] upload slice failed with network error: %v, will refresh upload URL", err) + return errors.Join(err, ErrUploadURLExpired) + } + return err + } + defer resp.Body.Close() + b.Reset() + _, err = b.ReadFrom(resp.Body) + if err != nil { + return err + } + body := b.Bytes() + respStr := string(body) + log.Debugln(respStr) + lower := strings.ToLower(respStr) + // 合并 uploadid 过期检测逻辑 + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() + if errCode != 0 || errNo != 0 { + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) + } + return nil +} + +// isUploadURLError 判断是否为需要刷新上传 URL 的错误 +// 包括：超时、连接被拒绝、连接重置、DNS 解析失败等网络错误 +func isUploadURLError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + // 超时错误 + if strings.Contains(errStr, "timeout") || + strings.Contains(errStr, "deadline exceeded") { + return true + } + // 连接错误 + if strings.Contains(errStr, "connection refused") || + strings.Contains(errStr, "connection reset") || + strings.Contains(errStr, "no such host") || + strings.Contains(errStr, "network is unreachable") { + return true + } + // EOF 错误（连接被服务器关闭） + if strings.Contains(errStr, "eof") || + strings.Contains(errStr, "broken pipe") { + return true + } + return false +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 0e27fb305..75018a708 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -207,7 +207,24 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Li return nil, err } u := fmt.Sprintf("%s&access_token=%s", resp.List[0].Dlink, d.AccessToken) - res, err := base.NoRedirectClient.R().SetHeader("User-Agent", "pan.baidu.com").Head(u) + + // Retry HEAD request with longer timeout to avoid client-side errors + // Create a client with longer timeout (base.NoRedirectClient doesn't have timeout set) + client := base.NoRedirectClient.SetTimeout(60 * time.Second) + var res *resty.Response + maxRetries := 5 + for i := 0; i < maxRetries; i++ { + res, err = client.R(). + SetHeader("User-Agent", "pan.baidu.com"). + Head(u) + if err == nil { + break + } + if i < maxRetries-1 { + log.Warnf("HEAD request failed (attempt %d/%d): %v, retrying...", i+1, maxRetries, err) + time.Sleep(time.Duration(i+1) * 2 * time.Second) // Exponential backoff: 2s, 4s, 6s, 8s + } + } if err != nil { return nil, err } diff --git a/drivers/quark_open/driver.go b/drivers/quark_open/driver.go index f0b8baf09..26a4288cc 100644 --- a/drivers/quark_open/driver.go +++ b/drivers/quark_open/driver.go @@ -8,6 +8,7 @@ import ( "hash" "io" "net/http" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -18,15 +19,23 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) type QuarkOpen struct { model.Storage Addition - config driver.Config - conf Conf + config driver.Config + conf Conf + limiter *rate.Limiter } +// 速率限制常量：夸克开放平台限流，保守设置 +const ( + quarkRateLimit = 2.0 // 每秒2个请求，避免限流 +) + func (d *QuarkOpen) Config() driver.Config { return d.config } @@ -36,6 +45,9 @@ func (d *QuarkOpen) GetAddition() driver.Additional { } func (d *QuarkOpen) Init(ctx context.Context) error { + // 初始化速率限制器 + d.limiter = rate.NewLimiter(rate.Limit(quarkRateLimit), 1) + var resp UserInfoResp _, err := d.request(ctx, "/open/v1/user/info", http.MethodGet, nil, &resp) @@ -52,11 +64,22 @@ func (d *QuarkOpen) Init(ctx context.Context) error { return err } +// waitLimit 等待速率限制 +func (d *QuarkOpen) waitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + func (d *QuarkOpen) Drop(ctx context.Context) error { return nil } func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } files, err := d.GetFiles(ctx, dir.GetID()) if err != nil { return nil, err @@ -67,6 +90,9 @@ func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs } func (d *QuarkOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } data := base.Json{ "fid": file.GetID(), } @@ -143,35 +169,116 @@ func (d *QuarkOpen) Remove(ctx context.Context, obj model.Obj) error { } func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) - var ( - md5 hash.Hash - sha1 hash.Hash - ) - writers := []io.Writer{} - if len(md5Str) != utils.MD5.Width { - md5 = utils.MD5.NewFunc() - writers = append(writers, md5) - } - if len(sha1Str) != utils.SHA1.Width { - sha1 = utils.SHA1.NewFunc() - writers = append(writers, sha1) + if err := d.waitLimit(ctx); err != nil { + return err } + md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) - if len(writers) > 0 { - _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) - if err != nil { - return err - } - if md5 != nil { - md5Str = hex.EncodeToString(md5.Sum(nil)) - } - if sha1 != nil { - sha1Str = hex.EncodeToString(sha1.Sum(nil)) + // 检查是否需要计算hash + needMD5 := len(md5Str) != utils.MD5.Width + needSHA1 := len(sha1Str) != utils.SHA1.Width + + if needMD5 || needSHA1 { + // 检查是否为可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 一次性计算所有hash，避免重复读取 + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + // 使用 RangeRead 分块读取文件，同时计算多个hash + multiWriter := io.MultiWriter(writers...) + size := stream.GetSize() + chunkSize := int64(10 * utils.MB) // 10MB per chunk + buf := make([]byte, chunkSize) + var offset int64 = 0 + + for offset < size { + readSize := min(chunkSize, size-offset) + + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset) + if err != nil { + return fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + + multiWriter.Write(buf[:n]) + offset += int64(n) + + // 更新进度（hash计算占用40%的进度） + up(40 * float64(offset) / float64(size)) + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } + } else { + // 不可重复读取的流（如网络流），需要缓存并计算hash + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) + if err != nil { + return err + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } } } - // pre - pre, err := d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + // pre - 带有 proof fail 重试逻辑 + var pre UpPreResp + var err error + err = retry.Do(func() error { + var preErr error + pre, preErr = d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + if preErr != nil { + // 检查是否为 proof fail 错误 + if strings.Contains(preErr.Error(), "proof") || strings.Contains(preErr.Error(), "43010") { + log.Warnf("[quark_open] Proof verification failed, retrying: %v", preErr) + return preErr // 返回错误触发重试 + } + // 检查是否为限流错误 + if strings.Contains(preErr.Error(), "限流") || strings.Contains(preErr.Error(), "rate") { + log.Warnf("[quark_open] Rate limited, waiting before retry: %v", preErr) + time.Sleep(2 * time.Second) // 额外等待 + return preErr + } + } + return preErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) if err != nil { return err } @@ -181,16 +288,70 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return nil } - // get part info - partInfo := d._getPartInfo(stream, pre.Data.PartSize) - // get upload url info - upUrlInfo, err := d.upUrl(ctx, pre, partInfo) - if err != nil { + // 空文件特殊处理：跳过分片上传，直接调用 upFinish + // 由于夸克 API 对空文件处理不稳定，尝试完成上传，失败则直接成功返回 + if stream.GetSize() == 0 { + log.Infof("[quark_open] Empty file detected, attempting direct finish (task_id: %s)", pre.Data.TaskID) + err = d.upFinish(ctx, pre, []base.Json{}, []string{}) + if err != nil { + // 空文件 upFinish 失败，可能是 API 不支持，直接视为成功 + log.Warnf("[quark_open] Empty file upFinish failed: %v, treating as success", err) + } + up(100) + return nil + } + + // 带重试的分片大小调整逻辑：如果检测到 "part list exceed" 错误，自动翻倍分片大小 + var upUrlInfo UpUrlInfo + var partInfo []base.Json + currentPartSize := pre.Data.PartSize + const maxRetries = 5 + const maxPartSize = 1024 * utils.MB // 1GB 上限 + + for attempt := 0; attempt < maxRetries; attempt++ { + // 计算分片信息 + partInfo = d._getPartInfo(stream, currentPartSize) + + // 尝试获取上传 URL + upUrlInfo, err = d.upUrl(ctx, pre, partInfo) + if err == nil { + // 成功获取上传 URL + log.Infof("[quark_open] Successfully obtained upload URLs with part size: %d MB (%d parts)", + currentPartSize/(1024*1024), len(partInfo)) + break + } + + // 检查是否为分片超限错误 + if strings.Contains(err.Error(), "exceed") { + if attempt < maxRetries-1 { + // 还有重试机会，翻倍分片大小 + newPartSize := currentPartSize * 2 + + // 检查是否超过上限 + if newPartSize > maxPartSize { + return fmt.Errorf("part list exceeded and cannot increase part size (current: %d MB, max: %d MB). File may be too large for Quark API", + currentPartSize/(1024*1024), maxPartSize/(1024*1024)) + } + + log.Warnf("[quark_open] Part list exceeded (attempt %d/%d, %d parts). Retrying with doubled part size: %d MB -> %d MB", + attempt+1, maxRetries, len(partInfo), + currentPartSize/(1024*1024), newPartSize/(1024*1024)) + + currentPartSize = newPartSize + continue // 重试 + } else { + // 已达到最大重试次数 + return fmt.Errorf("part list exceeded after %d retries. Last attempt: part size %d MB, %d parts", + maxRetries, currentPartSize/(1024*1024), len(partInfo)) + } + } + + // 其他错误，直接返回 return err } - // part up - ss, err := streamPkg.NewStreamSectionReader(stream, int(pre.Data.PartSize), &up) + // part up - 使用调整后的 currentPartSize + ss, err := streamPkg.NewStreamSectionReader(stream, int(currentPartSize), &up) if err != nil { return err } @@ -204,30 +365,49 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return ctx.Err() } - offset := int64(i) * pre.Data.PartSize - size := min(pre.Data.PartSize, total-offset) + offset := int64(i) * currentPartSize + size := min(currentPartSize, total-offset) rd, err := ss.GetSectionReader(offset, size) if err != nil { return err } + + // 上传重试逻辑，包含URL刷新 + var etag string err = retry.Do(func() error { rd.Seek(0, io.SeekStart) - etag, err := d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) - if err != nil { - return err + var uploadErr error + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) + + // 检查是否为URL过期错误 + if uploadErr != nil && strings.Contains(uploadErr.Error(), "expire") { + log.Warnf("[quark_open] Upload URL expired for part %d, refreshing...", i) + // 刷新上传URL + newUpUrlInfo, refreshErr := d.upUrl(ctx, pre, partInfo) + if refreshErr != nil { + return fmt.Errorf("failed to refresh upload url: %w", refreshErr) + } + upUrlInfo = newUpUrlInfo + log.Infof("[quark_open] Upload URL refreshed successfully") + + // 使用新URL重试上传 + rd.Seek(0, io.SeekStart) + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) } - etags = append(etags, etag) - return nil + + return uploadErr }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) + ss.FreeSectionReader(rd) if err != nil { return fmt.Errorf("failed to upload part %d: %w", i, err) } + etags = append(etags, etag) up(95 * float64(offset+size) / float64(total)) } diff --git a/drivers/quark_open/meta.go b/drivers/quark_open/meta.go index 3527b52e9..ee1903939 100644 --- a/drivers/quark_open/meta.go +++ b/drivers/quark_open/meta.go @@ -13,8 +13,8 @@ type Addition struct { APIAddress string `json:"api_url_address" default:"https://api.oplist.org/quarkyun/renewapi"` AccessToken string `json:"access_token" required:"false" default:""` RefreshToken string `json:"refresh_token" required:"true"` - AppID string `json:"app_id" required:"true" help:"Keep it empty if you don't have one"` - SignKey string `json:"sign_key" required:"true" help:"Keep it empty if you don't have one"` + AppID string `json:"app_id" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` + SignKey string `json:"sign_key" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` } type Conf struct { diff --git a/drivers/quark_open/util.go b/drivers/quark_open/util.go index 788ca0e99..1a3058375 100644 --- a/drivers/quark_open/util.go +++ b/drivers/quark_open/util.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -283,8 +284,15 @@ func (d *QuarkOpen) getProofRange(proofSeed string, fileSize int64) (*ProofRange func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []base.Json { // 计算分片信息 - partInfo := make([]base.Json, 0) total := stream.GetSize() + + // 确保partSize合理：最小4MB，避免分片过多 + const minPartSize int64 = 4 * utils.MB + if partSize < minPartSize { + partSize = minPartSize + } + + partInfo := make([]base.Json, 0) left := total partNumber := 1 @@ -304,6 +312,7 @@ func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []ba partNumber++ } + log.Infof("[quark_open] Upload plan: file_size=%d, part_size=%d, part_count=%d", total, partSize, len(partInfo)) return partInfo } @@ -315,11 +324,17 @@ func (d *QuarkOpen) upUrl(ctx context.Context, pre UpPreResp, partInfo []base.Js } var resp UpUrlResp + log.Infof("[quark_open] Requesting upload URLs for %d parts (task_id: %s)", len(partInfo), pre.Data.TaskID) + _, err = d.request(ctx, "/open/v1/file/get_upload_urls", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { + // 如果是分片超限错误，记录详细信息 + if strings.Contains(err.Error(), "part list exceed") { + log.Errorf("[quark_open] Part list exceeded limit! Requested %d parts. Please check Quark API documentation for actual limit.", len(partInfo)) + } return upUrlInfo, err } @@ -340,13 +355,43 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("User-Agent", "Go-http-client/1.1") + // ✅ 关键修复：使用更长的超时时间（10分钟） + // 慢速网络下大文件分片上传可能需要很长时间 + client := &http.Client{ + Timeout: 10 * time.Minute, + Transport: base.HttpClient.Transport, + } + // 发送请求 - resp, err := base.HttpClient.Do(req) + resp, err := client.Do(req) if err != nil { return "", err } defer resp.Body.Close() + // 检查是否为URL过期错误（403, 410等状态码） + if resp.StatusCode == 403 || resp.StatusCode == 410 { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("upload url expired (status: %d): %s", resp.StatusCode, string(body)) + } + + // ✅ 关键修复：409 PartAlreadyExist 不是错误！ + // 夸克使用Sequential模式，超时重试时如果分片已存在，说明第一次其实成功了 + if resp.StatusCode == 409 { + body, _ := io.ReadAll(resp.Body) + // 从响应体中提取已存在分片的ETag + if strings.Contains(string(body), "PartAlreadyExist") { + // 尝试从XML响应中提取ETag + if etag := extractEtagFromXML(string(body)); etag != "" { + log.Infof("[quark_open] Part %d already exists (409), using existing ETag: %s", partNumber+1, etag) + return etag, nil + } + // 如果无法提取ETag，返回错误 + log.Warnf("[quark_open] Part %d already exists but cannot extract ETag from response: %s", partNumber+1, string(body)) + return "", fmt.Errorf("part already exists but ETag not found in response") + } + } + if resp.StatusCode != 200 { body, _ := io.ReadAll(resp.Body) return "", fmt.Errorf("up status: %d, error: %s", resp.StatusCode, string(body)) @@ -355,6 +400,23 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber return resp.Header.Get("Etag"), nil } +// extractEtagFromXML 从OSS的XML错误响应中提取ETag +// 示例: "2F796AC486BB2891E3237D8BFDE020B5" +func extractEtagFromXML(xmlBody string) string { + start := strings.Index(xmlBody, "") + if start == -1 { + return "" + } + start += len("") + end := strings.Index(xmlBody[start:], "") + if end == -1 { + return "" + } + etag := xmlBody[start : start+end] + // 移除引号 + return strings.Trim(etag, "\"") +} + func (d *QuarkOpen) upFinish(ctx context.Context, pre UpPreResp, partInfo []base.Json, etags []string) error { // 创建 part_info_list partInfoList := make([]base.Json, len(partInfo)) @@ -417,25 +479,36 @@ func (d *QuarkOpen) generateReqSign(method string, pathname string, signKey stri } func (d *QuarkOpen) refreshToken() error { - refresh, access, err := d._refreshToken() + refresh, access, appID, signKey, err := d._refreshToken() for i := 0; i < 3; i++ { if err == nil { break } else { log.Errorf("[quark_open] failed to refresh token: %s", err) } - refresh, access, err = d._refreshToken() + refresh, access, appID, signKey, err = d._refreshToken() } if err != nil { return err } log.Infof("[quark_open] token exchange: %s -> %s", d.RefreshToken, refresh) d.RefreshToken, d.AccessToken = refresh, access + + // 如果在线API返回了AppID和SignKey，保存它们（不为空时才更新） + if appID != "" && appID != d.AppID { + d.AppID = appID + log.Infof("[quark_open] AppID updated from online API: %s", appID) + } + if signKey != "" && signKey != d.SignKey { + d.SignKey = signKey + log.Infof("[quark_open] SignKey updated from online API") + } + op.MustSaveDriverStorage(d) return nil } -func (d *QuarkOpen) _refreshToken() (string, string, error) { +func (d *QuarkOpen) _refreshToken() (string, string, string, string, error) { if d.UseOnlineAPI && d.APIAddress != "" { u := d.APIAddress var resp RefreshTokenOnlineAPIResp @@ -448,19 +521,20 @@ func (d *QuarkOpen) _refreshToken() (string, string, error) { }). Get(u) if err != nil { - return "", "", err + return "", "", "", "", err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { - return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) + return "", "", "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } - return "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") + return "", "", "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } - return resp.RefreshToken, resp.AccessToken, nil + // 返回所有字段，包括AppID和SignKey + return resp.RefreshToken, resp.AccessToken, resp.AppID, resp.SignKey, nil } // TODO 本地刷新逻辑 - return "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") + return "", "", "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") } // 生成认证 Cookie From a09c5de8b704ed31a03038b01c2bda7dd25850da Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:31:52 +0800 Subject: [PATCH 84/86] fix(core): copy_move depth, alias storage retrieval, sftp symlink, 500 panic - Fix pre-create subdirectory depth from 2 to 1 to avoid deep recursion - Fix srcBasePath bug and remove redundant sleep in preCreateDirectoryTree - Add unit tests for preCreateDirTreeFn - Update storage retrieval method in alias listRoot function - Fix sftp symlink path resolution - Fix 500 panic and NaN issues in openlist driver --- drivers/alias/util.go | 2 +- drivers/openlist/driver.go | 106 +++++++++++- drivers/sftp/types.go | 4 +- internal/fs/copy_move.go | 97 +++++++++++ internal/fs/copy_move_test.go | 317 ++++++++++++++++++++++++++++++++++ 5 files changed, 517 insertions(+), 9 deletions(-) create mode 100644 internal/fs/copy_move_test.go diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 8e5eb8a84..b37854394 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -40,7 +40,7 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model if !withDetails || len(v) != 1 { continue } - remoteDriver, err := op.GetStorageByMountPath(v[0]) + remoteDriver, err := fs.GetStorage(v[0], &fs.GetStoragesArgs{}) if err != nil { continue } diff --git a/drivers/openlist/driver.go b/drivers/openlist/driver.go index 79fc51185..b37d72a06 100644 --- a/drivers/openlist/driver.go +++ b/drivers/openlist/driver.go @@ -14,6 +14,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" @@ -195,6 +197,92 @@ func (d *OpenList) Remove(ctx context.Context, obj model.Obj) error { } func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + // 预计算 hash（如果不存在），使用 RangeRead 不消耗 Reader + // 这样远端驱动不需要再计算，避免 HTTP body 被重复读取 + md5Hash := s.GetHash().GetHash(utils.MD5) + sha1Hash := s.GetHash().GetHash(utils.SHA1) + sha256Hash := s.GetHash().GetHash(utils.SHA256) + sha1_128kHash := s.GetHash().GetHash(utils.SHA1_128K) + preHash := s.GetHash().GetHash(utils.PRE_HASH) + + // 计算所有缺失的 hash，确保最大兼容性 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(s, utils.MD5, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate MD5: %v", err) + md5Hash = "" + } + } + if len(sha1Hash) != utils.SHA1.Width { + var err error + sha1Hash, err = stream.StreamHashFile(s, utils.SHA1, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1: %v", err) + sha1Hash = "" + } + } + if len(sha256Hash) != utils.SHA256.Width { + var err error + sha256Hash, err = stream.StreamHashFile(s, utils.SHA256, 34, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA256: %v", err) + sha256Hash = "" + } + } + + // 计算特殊 hash（用于秒传验证） + // SHA1_128K: 前128KB的SHA1，115网盘使用 + if len(sha1_128kHash) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * 1024 // 128KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + sha1_128kHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1_128K: %v", err) + sha1_128kHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for SHA1_128K: %v", err) + } + } + + // PRE_HASH: 前1024字节的SHA1，阿里云盘使用 + if len(preHash) != utils.PRE_HASH.Width { + const PreHashSize int64 = 1024 // 1KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + preHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate PRE_HASH: %v", err) + preHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for PRE_HASH: %v", err) + } + } + + // 诊断日志：检查流的状态 + if ss, ok := s.(*stream.SeekableStream); ok { + if ss.Reader != nil { + log.Warnf("[openlist] WARNING: SeekableStream.Reader is not nil for file %s, stream may have been consumed!", s.GetName()) + } + } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, @@ -206,14 +294,20 @@ func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStream req.Header.Set("Authorization", d.Token) req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) req.Header.Set("Password", d.MetaPassword) - if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { - req.Header.Set("X-File-Md5", md5) + if len(md5Hash) > 0 { + req.Header.Set("X-File-Md5", md5Hash) + } + if len(sha1Hash) > 0 { + req.Header.Set("X-File-Sha1", sha1Hash) + } + if len(sha256Hash) > 0 { + req.Header.Set("X-File-Sha256", sha256Hash) } - if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { - req.Header.Set("X-File-Sha1", sha1) + if len(sha1_128kHash) > 0 { + req.Header.Set("X-File-Sha1-128k", sha1_128kHash) } - if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { - req.Header.Set("X-File-Sha256", sha256) + if len(preHash) > 0 { + req.Header.Set("X-File-Pre-Hash", preHash) } req.ContentLength = s.GetSize() diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go index 00a32f001..a57076e08 100644 --- a/drivers/sftp/types.go +++ b/drivers/sftp/types.go @@ -48,8 +48,8 @@ func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { Size: _f.Size(), Modified: _f.ModTime(), IsFolder: _f.IsDir(), - Path: target, + Path: path, // Use symlink's own path, not target path } - log.Debugf("[sftp] obj: %+v, is symlink: %v", obj, symlink) + log.Debugf("[sftp] obj: %+v, is symlink: %v, target: %s", obj, symlink, target) return obj, nil } diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index e78fc9be8..5ae92524e 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -17,6 +17,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type taskType uint8 @@ -192,6 +193,20 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) + // Pre-create the destination directory first + t.Status = "ensuring destination directory exists" + if err := op.MakeDir(t.Ctx(), t.DstStorage, dstActualPath); err != nil { + log.Warnf("[copy_move] failed to ensure destination dir [%s]: %v, will continue", dstActualPath, err) + // Continue anyway - the directory might exist but Get failed due to cache issues + } + + // Pre-create subdirectories (up to 1 level deep) to avoid deep recursion issues + // Balances between reducing API calls and maintaining fault tolerance + t.Status = "pre-creating subdirectories" + if err := t.preCreateDirectoryTree(objs, t.SrcActualPath, dstActualPath, 1); err != nil { + log.Warnf("[copy_move] failed to pre-create directory tree: %v, will continue", err) + // Continue anyway - individual directories will be created on-demand + } existedObjs := make(map[string]bool) if t.TaskType == merge { dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) @@ -263,6 +278,88 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } +// preCreateDirectoryTree is a thin method wrapper that resolves the storage-bound +// makeDir / listSrc functions and delegates to the pure preCreateDirTreeFn helper. +func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, srcBasePath, dstBasePath string, maxDepth int) error { + makeDir := func(ctx context.Context, path string) error { + return op.MakeDir(ctx, t.DstStorage, path) + } + listSrc := func(ctx context.Context, path string) ([]model.Obj, error) { + return op.List(ctx, t.SrcStorage, path, model.ListArgs{}) + } + return preCreateDirTreeFn(t.Ctx(), objs, srcBasePath, dstBasePath, maxDepth, makeDir, listSrc) +} + +// preCreateDirTreeFn recursively scans source directory tree and pre-creates +// directories on destination up to maxDepth levels to avoid deep MakeDir recursion issues. +// +// - maxDepth=0 – only create dirs in the current objs list (no recursion) +// - maxDepth=1 – also recurse one level deeper, etc. +// - srcBasePath – current source directory path; passed explicitly through all +// recursion levels so that subdirSrcPath is always correct (do NOT use +// t.SrcActualPath, which is fixed at the top-level path). +// +// makeDir and listSrc are injected to enable testing without a real storage driver. +func preCreateDirTreeFn( + ctx context.Context, + objs []model.Obj, + srcBasePath, dstBasePath string, + maxDepth int, + makeDir func(context.Context, string) error, + listSrc func(context.Context, string) ([]model.Obj, error), +) error { + // First pass: create immediate subdirectories + var subdirs []model.Obj + for _, obj := range objs { + // Check for cancellation + if err := ctx.Err(); err != nil { + return err + } + + if obj.IsDir() { + subdirPath := stdpath.Join(dstBasePath, obj.GetName()) + if err := makeDir(ctx, subdirPath); err != nil { + log.Debugf("[copy_move] failed to pre-create dir [%s]: %v", subdirPath, err) + // Continue with other directories + } + subdirs = append(subdirs, obj) + // No explicit sleep here: drivers that have QPS limits (e.g. 115, BaiduNetDisk) + // implement WaitLimit via a token-bucket rate.Limiter and call it inside their + // MakeDir, so op.MakeDir already blocks at the correct per-driver rate. + } + } + + // Stop recursion if max depth reached + if maxDepth <= 0 { + return nil + } + + // Second pass: recursively scan and create nested subdirectories + for _, subdir := range subdirs { + if err := ctx.Err(); err != nil { + return err + } + + // Build paths relative to srcBasePath (NOT t.SrcActualPath) so that + // deeper recursion levels resolve to the correct source paths. + subdirSrcPath := stdpath.Join(srcBasePath, subdir.GetName()) + subdirDstPath := stdpath.Join(dstBasePath, subdir.GetName()) + + subObjs, err := listSrc(ctx, subdirSrcPath) + if err != nil { + log.Debugf("[copy_move] failed to list subdir [%s] for pre-creation: %v", subdirSrcPath, err) + continue // Skip this subdirectory, will handle when processing + } + + // Recursively create subdirectories with decreased depth + if err := preCreateDirTreeFn(ctx, subObjs, subdirSrcPath, subdirDstPath, maxDepth-1, makeDir, listSrc); err != nil { + return err + } + } + + return nil +} + var ( CopyTaskManager *tache.Manager[*FileTransferTask] MoveTaskManager *tache.Manager[*FileTransferTask] diff --git a/internal/fs/copy_move_test.go b/internal/fs/copy_move_test.go new file mode 100644 index 000000000..5c9f4b539 --- /dev/null +++ b/internal/fs/copy_move_test.go @@ -0,0 +1,317 @@ +package fs + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// ---------- helpers ---------- + +func dirObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: true} +} + +func fileObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: false} +} + +// callRecorder records every path passed to makeDir and listSrc. +type callRecorder struct { + mu sync.Mutex + mkdirs []string + lists []string + // listReturns maps srcPath → objects to return (nil = empty) + listReturns map[string][]model.Obj + // mkdirErr maps dstPath → error to return + mkdirErr map[string]error +} + +func newRecorder() *callRecorder { + return &callRecorder{ + listReturns: make(map[string][]model.Obj), + mkdirErr: make(map[string]error), + } +} + +func (r *callRecorder) makeDir(_ context.Context, path string) error { + r.mu.Lock() + r.mkdirs = append(r.mkdirs, path) + err := r.mkdirErr[path] + r.mu.Unlock() + return err +} + +func (r *callRecorder) listSrc(_ context.Context, path string) ([]model.Obj, error) { + r.mu.Lock() + r.lists = append(r.lists, path) + objs := r.listReturns[path] + r.mu.Unlock() + return objs, nil +} + +func (r *callRecorder) hasMkdir(path string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range r.mkdirs { + if p == path { + return true + } + } + return false +} + +func (r *callRecorder) hasList(path string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range r.lists { + if p == path { + return true + } + } + return false +} + +// ---------- tests ---------- + +// TestPreCreateDirTreeFn_EmptyObjs: no objects → no calls at all. +func TestPreCreateDirTreeFn_EmptyObjs(t *testing.T) { + rec := newRecorder() + err := preCreateDirTreeFn(context.Background(), nil, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("expected 0 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + } + if len(rec.lists) != 0 { + t.Errorf("expected 0 List calls, got %d: %v", len(rec.lists), rec.lists) + } +} + +// TestPreCreateDirTreeFn_OnlyFiles: file objects only → zero MakeDir calls. +func TestPreCreateDirTreeFn_OnlyFiles(t *testing.T) { + objs := []model.Obj{fileObj("a.txt"), fileObj("b.txt")} + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("expected 0 MakeDir calls, got %d", len(rec.mkdirs)) + } +} + +// TestPreCreateDirTreeFn_FlatDirs_MaxDepth0: dirs present, maxDepth=0 → MakeDir +// called for each dir with correct dstPath, NO listSrc calls. +func TestPreCreateDirTreeFn_FlatDirs_MaxDepth0(t *testing.T) { + objs := []model.Obj{dirObj("subA"), fileObj("file.txt"), dirObj("subB")} + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst/parent", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !rec.hasMkdir("/dst/parent/subA") { + t.Error("expected MakeDir(/dst/parent/subA)") + } + if !rec.hasMkdir("/dst/parent/subB") { + t.Error("expected MakeDir(/dst/parent/subB)") + } + if rec.hasMkdir("/dst/parent/file.txt") { + t.Error("MakeDir must NOT be called for a file") + } + if len(rec.lists) != 0 { + t.Errorf("maxDepth=0 must not trigger any List calls, got: %v", rec.lists) + } +} + +// TestPreCreateDirTreeFn_Recursion_CorrectSrcPath is the regression test for the +// srcBasePath bug: with maxDepth=1 the recursive List must use the SUBDIR src path, +// not the original top-level srcBasePath. +func TestPreCreateDirTreeFn_Recursion_CorrectSrcPath(t *testing.T) { + // /src/parent contains [subA(dir), subB(dir)] + // /src/parent/subA contains [subA1(dir)] + // /src/parent/subB contains [] + topObjs := []model.Obj{dirObj("subA"), dirObj("subB")} + rec := newRecorder() + rec.listReturns["/src/parent/subA"] = []model.Obj{dirObj("subA1")} + rec.listReturns["/src/parent/subB"] = []model.Obj{} + + if err := preCreateDirTreeFn(context.Background(), topObjs, "/src/parent", "/dst/parent", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // ── first level dirs must be created + if !rec.hasMkdir("/dst/parent/subA") { + t.Error("expected MakeDir(/dst/parent/subA)") + } + if !rec.hasMkdir("/dst/parent/subB") { + t.Error("expected MakeDir(/dst/parent/subB)") + } + + // ── listSrc must use subdirSrcPath (NOT the whole /src/parent again) + if !rec.hasList("/src/parent/subA") { + t.Error("listSrc must be called with /src/parent/subA, got:", rec.lists) + } + if !rec.hasList("/src/parent/subB") { + t.Error("listSrc must be called with /src/parent/subB, got:", rec.lists) + } + // The original bug would have called listSrc("/src/parent/subA") as + // stdpath.Join(t.SrcActualPath, "subA") where t.SrcActualPath=="/src/parent", + // but in a deeper recursive call (e.g. maxDepth=2) it would have used + // the top-level path incorrectly; verify the nested mkdir used the right dst. + if !rec.hasMkdir("/dst/parent/subA/subA1") { + t.Error("expected MakeDir(/dst/parent/subA/subA1), got mkdirs:", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion: with maxDepth=1 recursion +// goes exactly one level. The nested list returns another dir, but since maxDepth +// reaches 0 that deeper dir must NOT be listed further. +func TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion(t *testing.T) { + topObjs := []model.Obj{dirObj("sub")} + rec := newRecorder() + // sub contains deeper, deeper contains deepest + rec.listReturns["/src/sub"] = []model.Obj{dirObj("deeper")} + rec.listReturns["/src/sub/deeper"] = []model.Obj{dirObj("deepest")} // should NOT be listed + + if err := preCreateDirTreeFn(context.Background(), topObjs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !rec.hasMkdir("/dst/sub") { + t.Error("expected /dst/sub to be created") + } + if !rec.hasMkdir("/dst/sub/deeper") { + t.Error("expected /dst/sub/deeper to be created (within maxDepth=1)") + } + // deepest must NOT be created (would require maxDepth=2) + if rec.hasMkdir("/dst/sub/deeper/deepest") { + t.Error("/dst/sub/deeper/deepest must NOT be created at maxDepth=1") + } + // /src/sub/deeper must NOT be listed (we've hit maxDepth=0 at that point) + if rec.hasList("/src/sub/deeper") { + t.Error("/src/sub/deeper must NOT be listed when maxDepth reaches 0") + } +} + +// TestPreCreateDirTreeFn_ContextCancelled: context cancelled before processing → +// returns ctx.Err, makes zero or partial calls. +func TestPreCreateDirTreeFn_ContextCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // already cancelled + + objs := []model.Obj{dirObj("sub")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled, got: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("no MakeDir should be called after cancellation, got: %v", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_ContextCancelledDuringRecursion: context is cancelled +// during the second-pass recursion loop. +func TestPreCreateDirTreeFn_ContextCancelledDuringRecursion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + // Two dirs; cancel after the first List in recursion + callCount := 0 + listSrc := func(c context.Context, path string) ([]model.Obj, error) { + callCount++ + cancel() // cancel on first list call + return nil, nil + } + objs := []model.Obj{dirObj("sub1"), dirObj("sub2")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, listSrc) + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled after cancellation during recursion, got: %v", err) + } + if callCount > 1 { + t.Errorf("listSrc should have been called at most once before ctx.Err fired, got %d", callCount) + } +} + +// TestPreCreateDirTreeFn_MakeDirErrorNonFatal: a MakeDir failure on one dir must +// not stop processing of subsequent dirs. +func TestPreCreateDirTreeFn_MakeDirErrorNonFatal(t *testing.T) { + objs := []model.Obj{dirObj("subA"), dirObj("subB"), dirObj("subC")} + rec := newRecorder() + rec.mkdirErr["/dst/subA"] = errors.New("quota exceeded") + + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("error should not propagate from MakeDir failure: %v", err) + } + // All three must have been attempted despite the error on subA + for _, p := range []string{"/dst/subA", "/dst/subB", "/dst/subC"} { + if !rec.hasMkdir(p) { + t.Errorf("expected MakeDir(%s) to be called", p) + } + } +} + +// TestPreCreateDirTreeFn_ListErrorNonFatal: a List error for one subdir during +// recursion skips that subdir but continues with the rest. +func TestPreCreateDirTreeFn_ListErrorNonFatal(t *testing.T) { + objs := []model.Obj{dirObj("subA"), dirObj("subB")} + listCallCount := 0 + listSrc := func(_ context.Context, path string) ([]model.Obj, error) { + listCallCount++ + if path == "/src/subA" { + return nil, errors.New("I/O error") + } + return []model.Obj{dirObj("nested")}, nil + } + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, listSrc); err != nil { + t.Fatalf("List error must not be fatal: %v", err) + } + // subB's nested dir should still be processed despite subA's List failure + if !rec.hasMkdir("/dst/subB/nested") { + t.Error("expected /dst/subB/nested to be created despite subA list error, mkdirs:", rec.mkdirs) + } + if listCallCount != 2 { + t.Errorf("both subdirs must be attempted for listing, got %d calls", listCallCount) + } +} + +// TestPreCreateDirTreeFn_MixedObjs: mixed files and dirs; only dirs are processed. +func TestPreCreateDirTreeFn_MixedObjs(t *testing.T) { + objs := []model.Obj{ + fileObj("readme.md"), + dirObj("assets"), + fileObj("main.go"), + dirObj("pkg"), + } + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 2 { + t.Errorf("expected exactly 2 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + } + if !rec.hasMkdir("/dst/assets") || !rec.hasMkdir("/dst/pkg") { + t.Errorf("unexpected mkdirs: %v", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_Timeout: context with a very short deadline cancels execution. +func TestPreCreateDirTreeFn_Timeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + time.Sleep(5 * time.Millisecond) // ensure deadline has passed + + objs := []model.Obj{dirObj("sub")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("expected DeadlineExceeded, got: %v", err) + } +} From 0e725bda52d2af1ed655bbb3b42c07eee0a963ab Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:32:02 +0800 Subject: [PATCH 85/86] feat(frontend): dynamic frontend fetching, CI upgrades, and build infrastructure - Add dynamic frontend pull from GitHub rolling release at startup - Implement frontend caching mechanism with ETag-based validation - Track rolling release by tag commit hash - Unify configurable frontend repo and release channel across build and runtime - Upgrade GitHub Actions to Node24-compatible versions (checkout v5, setup-go v6, etc.) - Update Docker workflow to support frontend version matrix builds - Optimize CI build speed with smart caching --- .github/workflows/beta_release.yml | 7 +- .github/workflows/build.yml | 10 +- .github/workflows/issue_pr_comment.yml | 4 +- .github/workflows/release.yml | 2 +- .github/workflows/release_docker.yml | 16 +- .github/workflows/test_docker.yml | 146 ++++--- .gitignore | 5 +- build.sh | 47 ++- internal/bootstrap/run.go | 2 + internal/conf/config.go | 4 +- internal/conf/var.go | 1 + internal/frontend/fetcher.go | 501 +++++++++++++++++++++++++ internal/frontend/fetcher_test.go | 480 +++++++++++++++++++++++ internal/frontend/watcher.go | 122 ++++++ public/dist/README.md | 1 - server/handles/fsread.go | 6 +- server/static/static.go | 88 ++++- 17 files changed, 1342 insertions(+), 100 deletions(-) create mode 100644 internal/frontend/fetcher.go create mode 100644 internal/frontend/fetcher_test.go create mode 100644 internal/frontend/watcher.go delete mode 100644 public/dist/README.md diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index d5817e6b4..1fd87625a 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -48,7 +48,7 @@ jobs: tag_name: beta - name: Upload assets to github artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: beta changelog path: ${{ github.workspace }}/CHANGELOG.md @@ -110,7 +110,7 @@ jobs: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "1.25.0" @@ -137,6 +137,7 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} env: GOFLAGS: ${{ matrix.goflags }} @@ -182,7 +183,7 @@ jobs: echo "cleaned_target=$CLEANED_TARGET" >> $GITHUB_ENV - name: Upload assets to github artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: beta builds for ${{ env.cleaned_target }} path: ${{ github.workspace }}/build/compress/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3a501ffa..a38d8b715 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: build: strategy: @@ -31,7 +34,7 @@ jobs: id: short-sha - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "1.25.0" @@ -55,7 +58,8 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag - github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=latest + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} output: openlist$ext - name: Verify musl binary is static @@ -69,7 +73,7 @@ jobs: fi - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }} path: build/* diff --git a/.github/workflows/issue_pr_comment.yml b/.github/workflows/issue_pr_comment.yml index 1b51e23b9..bc29e6bf6 100644 --- a/.github/workflows/issue_pr_comment.yml +++ b/.github/workflows/issue_pr_comment.yml @@ -16,7 +16,7 @@ jobs: if: github.event_name == 'issues' steps: - name: Check issue for unchecked tasks and reply - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | let comment = ""; @@ -81,7 +81,7 @@ jobs: if: github.event_name == 'pull_request' steps: - name: Check PR title for required prefix and comment - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const title = context.payload.pull_request.title || ""; diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2fa13af18..76152ef24 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,7 @@ jobs: swap-storage: true - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: '1.25.0' diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 80bdf9e3c..7503a3eda 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -45,13 +45,13 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version: '1.25.0' - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -69,7 +69,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true @@ -85,13 +85,13 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version: '1.25.0' - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -109,7 +109,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ env.ARTIFACT_NAME_LITE }} overwrite: true @@ -147,7 +147,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' @@ -231,7 +231,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: name: ${{ env.ARTIFACT_NAME_LITE }} path: 'build/' diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index 16c299401..33999732e 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -1,150 +1,196 @@ name: Beta Release (Docker) - on: workflow_dispatch: + inputs: + frontend_repo: + description: 'Frontend repo, e.g. Ironboxplus/OpenList-Frontend' + required: false + default: 'OpenListTeam/OpenList-Frontend' + type: string + frontend_channel: + description: 'Frontend release channel to build' + required: false + default: rolling + type: choice + options: + - rolling + - latest + - both push: branches: - main pull_request: branches: - - main + - copy + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: - DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} - GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} - IMAGE_NAME: openlist-git - IMAGE_NAME_DOCKERHUB: openlist + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 + FRONTEND_REPO: ${{ github.event.inputs.frontend_repo || vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} + IMAGE_NAME: openlist REGISTRY: ghcr.io - ARTIFACT_NAME: 'binaries_docker_release' - RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. - IMAGE_PUSH: ${{ github.event_name == 'push' }} - IMAGE_TAGS_BETA: | - type=ref,event=pr - type=raw,value=beta,enable={{is_default_branch}} + ARTIFACT_NAME_PREFIX: 'binaries_docker_release' + # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 + RELEASE_PLATFORMS: 'linux/amd64' + # 👇 关键修改：强制允许推送，不用管是不是 push 事件 + IMAGE_PUSH: 'true' + # 👇 使用默认的前端仓库 (OpenListTeam/OpenList-Frontend) + # FRONTEND_REPO: 'Ironboxplus/OpenList-Frontend' jobs: build_binary: - name: Build Binaries for Docker Release (Beta) + name: Build Binaries (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest + strategy: + matrix: + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["latest","rolling"]') }} steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - name: Setup Go + uses: actions/setup-go@v6 with: go-version: '1.25.0' + cache: true + cache-dependency-path: go.sum + + - name: Get Frontend Cache Version + id: frontend-cache + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WEB_VERSION: ${{ matrix.frontend_channel }} + run: | + frontend_repo="${{ env.FRONTEND_REPO }}" + web_version="$WEB_VERSION" + github_auth_args=() + + if [ -n "$GH_TOKEN" ]; then + github_auth_args=(-H "Authorization: Bearer $GH_TOKEN") + fi + + if [ "$web_version" = "latest" ]; then + frontend_version=$(curl -fsSL "${github_auth_args[@]}" "https://api.github.com/repos/$frontend_repo/releases/latest" | jq -r '.tag_name') + else + frontend_version="$web_version" + fi + + echo "repo=$frontend_repo" >> "$GITHUB_OUTPUT" + echo "version=$frontend_version" >> "$GITHUB_OUTPUT" + echo "Frontend repo: $frontend_repo" + echo "Frontend cache version: $frontend_version" + + - name: Cache Frontend + id: cache-frontend + uses: actions/cache@v5 + with: + path: public/dist + key: frontend-${{ steps.frontend-cache.outputs.repo }}-${{ steps.frontend-cache.outputs.version }} + restore-keys: | + frontend-${{ steps.frontend-cache.outputs.repo }}- - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' - run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash build.sh prepare docker-multiplatform - - name: Build go binary (beta) + - name: Build go binary run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} + WEB_VERSION: ${{ matrix.frontend_channel }} + FRONTEND_REPO: ${{ env.FRONTEND_REPO }} + SKIP_FRONTEND_FETCH: ${{ steps.cache-frontend.outputs.cache-hit == 'true' && 'true' || 'false' }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: - name: ${{ env.ARTIFACT_NAME }} + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} overwrite: true - path: | - build/ - !build/*.tgz - !build/musl-libs/** + path: build/linux/amd64/openlist release_docker: needs: build_binary - name: Release Docker image (Beta) + name: Release Docker (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["latest","rolling"]') }} + # 构建所有变体 image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" base_image_tag: "base" build_arg: "" - tag_favor: "" + image_tag_suffix: "" - image: "ffmpeg" base_image_tag: "ffmpeg" build_arg: INSTALL_FFMPEG=true - tag_favor: "suffix=-ffmpeg,onlatest=true" + image_tag_suffix: "-ffmpeg" - image: "aria2" base_image_tag: "aria2" build_arg: INSTALL_ARIA2=true - tag_favor: "suffix=-aria2,onlatest=true" + image_tag_suffix: "-aria2" - image: "aio" base_image_tag: "aio" build_arg: | INSTALL_FFMPEG=true INSTALL_ARIA2=true - tag_favor: "suffix=-aio,onlatest=true" + image_tag_suffix: "-aio" steps: - name: Checkout uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v7 with: - name: ${{ env.ARTIFACT_NAME }} - path: 'build/' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} + path: 'build/linux/amd64' - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # 👇 只保留 GitHub 登录，删除了 DockerHub 登录 - name: Login to GitHub Container Registry - if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to DockerHub Container Registry - if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} - ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} - tags: ${{ env.IMAGE_TAGS_BETA }} - flavor: | - ${{ matrix.tag_favor }} + tags: | + type=raw,value=front-${{ matrix.frontend_channel }}${{ matrix.image_tag_suffix }} + type=raw,value=latest,enable=${{ matrix.frontend_channel == 'latest' && matrix.image == 'latest' }} - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci - push: ${{ env.IMAGE_PUSH == 'true' }} + push: true build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }},mode=max diff --git a/.gitignore b/.gitignore index 1d71f0d60..d42155110 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .DS_Store output/ /dist/ - +.omx # Binaries for programs and plugins *.exe *.exe~ @@ -31,4 +31,5 @@ output/ /public/dist/* /!public/dist/README.md -.VSCodeCounter \ No newline at end of file +.VSCodeCounter +nul diff --git a/build.sh b/build.sh index b6baca1fe..d37ddad73 100644 --- a/build.sh +++ b/build.sh @@ -18,6 +18,8 @@ if [[ "$*" == *"lite"* ]]; then useLite=true fi +skipFrontendFetch="${SKIP_FRONTEND_FETCH:-false}" + if [ "$1" = "dev" ]; then version="dev" webVersion="rolling" @@ -31,6 +33,10 @@ else webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/$frontendRepo/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi +if [ -n "$WEB_VERSION" ]; then + webVersion="$WEB_VERSION" +fi + echo "backend version: $version" echo "frontend version: $webVersion" if [ "$useLite" = true ]; then @@ -46,6 +52,7 @@ ldflags="\ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$gitCommit' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$version' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=$webVersion' \ +-X 'github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=$frontendRepo' \ " # Keep sqlite driver tag selection centralized to avoid target drift. @@ -97,6 +104,11 @@ AssertStaticBinary() { } FetchWebRolling() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') @@ -110,6 +122,11 @@ FetchWebRolling() { } FetchWebRelease() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/latest\"") release_assets=$(echo "$release_json" | jq -r '.assets[].browser_download_url') @@ -236,8 +253,8 @@ BuildDockerMultiplatform() { docker_lflags="$(GetMuslStaticLdflags)" export CGO_ENABLED=1 - OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds - CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds + OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds + CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} @@ -257,15 +274,17 @@ BuildDockerMultiplatform() { GO_ARM=(6 7) export GOOS=linux export GOARCH=arm - for i in "${!DOCKER_ARM_ARCHES[@]}"; do - docker_arch=${DOCKER_ARM_ARCHES[$i]} - cgo_cc=${CGO_ARGS[$i]} - export GOARM=${GO_ARM[$i]} - export CC=${cgo_cc} - echo "building for $docker_arch" - CGO_LDFLAGS="-static" go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . - AssertStaticBinary "build/${docker_arch%%-*}/${docker_arch##*-}/$appName" - done + # ARM docker variants stay disabled on this branch to keep the workflow x64-only. + # If they are re-enabled later, they should follow the same static-link pattern. + # for i in "${!DOCKER_ARM_ARCHES[@]}"; do + # docker_arch=${DOCKER_ARM_ARCHES[$i]} + # cgo_cc=${CGO_ARGS[$i]} + # export GOARM=${GO_ARM[$i]} + # export CC=${cgo_cc} + # echo "building for $docker_arch" + # CGO_LDFLAGS="-static" go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + # AssertStaticBinary "build/${docker_arch%%-*}/${docker_arch##*-}/$appName" + # done } BuildRelease() { @@ -655,7 +674,11 @@ if [ "$buildType" = "dev" ]; then fi elif [ "$buildType" = "release" -o "$buildType" = "beta" ]; then if [ "$buildType" = "beta" ]; then - FetchWebRolling + if [ "$WEB_VERSION" = "latest" ]; then + FetchWebRelease + else + FetchWebRolling + fi else FetchWebRelease fi diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go index 6740dba65..ff02509ba 100644 --- a/internal/bootstrap/run.go +++ b/internal/bootstrap/run.go @@ -15,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/frontend" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server" "github.com/OpenListTeam/OpenList/v4/server/middlewares" @@ -273,6 +274,7 @@ func Start() { func Shutdown(timeout time.Duration) { utils.Log.Println("Shutdown server...") + frontend.StopWatcher() fs.ArchiveContentUploadTaskManager.RemoveAll() ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() diff --git a/internal/conf/config.go b/internal/conf/config.go index f347380d8..e0c7b52cf 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -118,6 +118,7 @@ type Config struct { TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` DistDir string `json:"dist_dir"` + FrontendRepo string `json:"frontend_repo" env:"FRONTEND_REPO"` Log LogConfig `json:"log" envPrefix:"LOG_"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` MaxBufferLimit int `json:"max_buffer_limitMB" env:"MAX_BUFFER_LIMIT_MB"` @@ -162,7 +163,8 @@ func DefaultConfig(dataDir string) *Config { Host: "http://localhost:7700", Index: "openlist", }, - BleveDir: indexDir, + BleveDir: indexDir, + FrontendRepo: FrontendRepoDefault, Log: LogConfig{ Enable: true, Name: logPath, diff --git a/internal/conf/var.go b/internal/conf/var.go index 972f69997..30dd73de8 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -12,6 +12,7 @@ var ( GitCommit string = "unknown" Version string = "dev" WebVersion string = "rolling" + FrontendRepoDefault string = "OpenListTeam/OpenList-Frontend" ) var ( diff --git a/internal/frontend/fetcher.go b/internal/frontend/fetcher.go new file mode 100644 index 000000000..363d5dfca --- /dev/null +++ b/internal/frontend/fetcher.go @@ -0,0 +1,501 @@ +package frontend + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const ( + defaultFrontendRepo = "OpenListTeam/OpenList-Frontend" + versionFile = ".frontend_version" + distDirName = "dist" +) + +// FetchResult contains the result of a fetch operation +type FetchResult struct { + Version string + Downloaded bool + DistPath string +} + +// GetDistPath returns the path where dynamically fetched frontend dist is stored +func GetDistPath() string { + return filepath.Join(flags.DataDir, "frontend_dist") +} + +// GetVersionFilePath returns the path to the version tracking file +func GetVersionFilePath() string { + return filepath.Join(GetDistPath(), versionFile) +} + +// HasValidDist checks if the dynamic dist directory exists and has an index.html +func HasValidDist() bool { + distPath := GetDistPath() + _, err := os.Stat(filepath.Join(distPath, distDirName, "index.html")) + return err == nil +} + +// ReadCurrentVersion reads the currently cached version from disk +func ReadCurrentVersion() string { + data, err := os.ReadFile(GetVersionFilePath()) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +// writeVersion writes the version string to the version tracking file +func writeVersion(version string) error { + return os.WriteFile(GetVersionFilePath(), []byte(version), 0644) +} + +// FetchFromRolling downloads the frontend dist from the GitHub rolling release +func FetchFromRolling(ctx context.Context) (*FetchResult, error) { + return fetchFromTag(ctx, "rolling", "") +} + +// FetchFromLatest downloads the frontend dist from the GitHub latest release +func FetchFromLatest(ctx context.Context) (*FetchResult, error) { + return fetchFromTag(ctx, "", "") +} + +// githubRelease represents a GitHub release for JSON parsing +type githubRelease struct { + TagName string `json:"tag_name"` + Assets []struct { + BrowserDownloadURL string `json:"browser_download_url"` + Name string `json:"name"` + } `json:"assets"` + PublishedAt string `json:"published_at"` +} + +type githubRef struct { + Object struct { + Type string `json:"type"` + SHA string `json:"sha"` + } `json:"object"` +} + +type githubAnnotatedTag struct { + Object struct { + Type string `json:"type"` + SHA string `json:"sha"` + } `json:"object"` +} + +func getFrontendRepo() string { + if conf.Conf != nil && strings.TrimSpace(conf.Conf.FrontendRepo) != "" { + return strings.TrimSpace(conf.Conf.FrontendRepo) + } + return defaultFrontendRepo +} + +func shortHash(sha string) string { + const shortLen = 12 + if len(sha) > shortLen { + return sha[:shortLen] + } + return sha +} + +func versionIdentifier(tag, commitSHA, fallback string) string { + if strings.TrimSpace(commitSHA) != "" { + return fmt.Sprintf("%s@%s", tag, shortHash(commitSHA)) + } + if strings.TrimSpace(fallback) != "" { + return fallback + } + return tag +} + +func resolveTagCommitSHA(ctx context.Context, client *http.Client, baseURL, tag string) (string, error) { + if strings.TrimSpace(tag) == "" { + return "", fmt.Errorf("empty tag") + } + + apiBase := "https://api.github.com" + if baseURL != "" { + apiBase = strings.TrimRight(baseURL, "/") + } + + repo := getFrontendRepo() + refURL := fmt.Sprintf("%s/repos/%s/git/ref/tags/%s", apiBase, repo, url.PathEscape(tag)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, refURL, nil) + if err != nil { + return "", fmt.Errorf("create ref request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("fetch tag ref: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("tag ref API returned %d: %s", resp.StatusCode, string(body)) + } + + var ref githubRef + if err := json.NewDecoder(resp.Body).Decode(&ref); err != nil { + return "", fmt.Errorf("decode ref JSON: %w", err) + } + + switch ref.Object.Type { + case "commit": + if ref.Object.SHA == "" { + return "", fmt.Errorf("empty commit sha in ref response") + } + return ref.Object.SHA, nil + case "tag": + if ref.Object.SHA == "" { + return "", fmt.Errorf("empty tag sha in ref response") + } + tagObjURL := fmt.Sprintf("%s/repos/%s/git/tags/%s", apiBase, repo, ref.Object.SHA) + tagReq, err := http.NewRequestWithContext(ctx, http.MethodGet, tagObjURL, nil) + if err != nil { + return "", fmt.Errorf("create tag object request: %w", err) + } + tagReq.Header.Set("Accept", "application/vnd.github.v3+json") + tagReq.Header.Set("User-Agent", "OpenList") + + tagResp, err := client.Do(tagReq) + if err != nil { + return "", fmt.Errorf("fetch tag object: %w", err) + } + defer tagResp.Body.Close() + + if tagResp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(tagResp.Body) + return "", fmt.Errorf("tag object API returned %d: %s", tagResp.StatusCode, string(body)) + } + + var tagObj githubAnnotatedTag + if err := json.NewDecoder(tagResp.Body).Decode(&tagObj); err != nil { + return "", fmt.Errorf("decode tag object JSON: %w", err) + } + if tagObj.Object.SHA == "" { + return "", fmt.Errorf("empty object sha in tag object response") + } + return tagObj.Object.SHA, nil + default: + if ref.Object.SHA == "" { + return "", fmt.Errorf("unsupported ref object type %q with empty sha", ref.Object.Type) + } + return ref.Object.SHA, nil + } +} + +// fetchFromTag downloads frontend dist from a GitHub release tag. +// If baseURL is non-empty, it replaces api.github.com (used for testing). +func fetchFromTag(ctx context.Context, tag string, baseURL string) (*FetchResult, error) { + repo := getFrontendRepo() + var apiURL string + if baseURL != "" { + if tag == "" { + apiURL = fmt.Sprintf("%s/repos/%s/releases/latest", baseURL, repo) + } else { + apiURL = fmt.Sprintf("%s/repos/%s/releases/tags/%s", baseURL, repo, tag) + } + } else { + if tag == "" { + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", repo) + } else { + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/releases/tags/%s", repo, tag) + } + } + + client := newHTTPClient() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch release info: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("github API returned %d: %s", resp.StatusCode, string(body)) + } + + var release githubRelease + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + return nil, fmt.Errorf("decode release JSON: %w", err) + } + + // Find the dist tarball URL (non-lite) + var tarURL string + for _, asset := range release.Assets { + if strings.Contains(asset.Name, "openlist-frontend-dist") && + !strings.Contains(asset.Name, "lite") && + strings.HasSuffix(asset.Name, ".tar.gz") { + tarURL = asset.BrowserDownloadURL + break + } + } + if tarURL == "" { + return nil, fmt.Errorf("no frontend dist tarball found in release %s", release.TagName) + } + + commitSHA, err := resolveTagCommitSHA(ctx, client, baseURL, release.TagName) + if err != nil { + utils.Log.Warnf("[frontend] failed to resolve tag %s hash: %v", release.TagName, err) + } + + resolvedVersion := versionIdentifier(release.TagName, commitSHA, tarURL) + + // Use tag+commit-hash as the primary version identifier. + // For rolling releases the tag itself is static, but its target commit moves. + // If hash resolve fails, fallback to tarball URL so updates can still be detected. + currentVersion := ReadCurrentVersion() + if currentVersion == resolvedVersion && HasValidDist() { + utils.Log.Infof("[frontend] version %s already cached, skipping download", resolvedVersion) + return &FetchResult{ + Version: resolvedVersion, + Downloaded: false, + DistPath: filepath.Join(GetDistPath(), distDirName), + }, nil + } + + utils.Log.Infof("[frontend] downloading version %s from %s", resolvedVersion, tarURL) + if err := downloadAndExtract(ctx, client, tarURL); err != nil { + return nil, fmt.Errorf("download and extract: %w", err) + } + + if err := writeVersion(resolvedVersion); err != nil { + utils.Log.Warnf("[frontend] failed to write version file: %v", err) + } + + utils.Log.Infof("[frontend] successfully fetched version %s", resolvedVersion) + return &FetchResult{ + Version: resolvedVersion, + Downloaded: true, + DistPath: filepath.Join(GetDistPath(), distDirName), + }, nil +} + +func downloadAndExtract(ctx context.Context, client *http.Client, url string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("create download request: %w", err) + } + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("download tarball: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download returned status %d", resp.StatusCode) + } + + destDir := GetDistPath() + tmpDir := filepath.Join(destDir, ".tmp-"+fmt.Sprintf("%d", time.Now().UnixNano())) + if err := os.MkdirAll(tmpDir, 0755); err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + if err := extractTarGz(resp.Body, tmpDir); err != nil { + return fmt.Errorf("extract tar.gz: %w", err) + } + + // Determine the source directory: + // If the tarball contains a "dist" subdirectory, use it; + // otherwise, the files are at the root and we use tmpDir directly. + srcDir := tmpDir + if _, err := os.Stat(filepath.Join(tmpDir, distDirName)); err == nil { + srcDir = filepath.Join(tmpDir, distDirName) + } + + // Atomic swap: rename source to final + finalDir := filepath.Join(destDir, distDirName) + oldDir := filepath.Join(destDir, distDirName+".old") + // Remove previous backup if exists + os.RemoveAll(oldDir) + // Move current dist out of the way if it exists + os.Rename(finalDir, oldDir) + // Move new dist into place + if err := os.Rename(srcDir, finalDir); err != nil { + // Rollback + os.RemoveAll(finalDir) + os.Rename(oldDir, finalDir) + return fmt.Errorf("rename new dist: %w", err) + } + os.RemoveAll(oldDir) + + return nil +} + +func extractTarGz(r io.Reader, dest string) error { + gzr, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("gzip reader: %w", err) + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("tar next: %w", err) + } + + // Normalize: strip leading ./ so "./dist" becomes "dist" + name := strings.TrimPrefix(hdr.Name, "./") + if name == "" || name == "." { + continue // skip bare directory entry + } + + target := filepath.Join(dest, name) + + // Security: prevent path traversal + if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("path traversal detected: %s", hdr.Name) + } + + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return err + } + f.Close() + } + } + return nil +} + +// newHTTPClient creates an HTTP client that respects proxy configuration +func newHTTPClient() *http.Client { + transport := &http.Transport{} + if conf.Conf != nil && conf.Conf.ProxyAddress != "" { + if proxyURL := mustParseURL(conf.Conf.ProxyAddress); proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + } + return &http.Client{ + Transport: transport, + Timeout: 5 * time.Minute, + } +} + +func mustParseURL(raw string) *url.URL { + u, err := url.Parse(raw) + if err != nil { + utils.Log.Warnf("[frontend] invalid proxy URL %q: %v", raw, err) + return nil + } + return u +} + +// EnsureDist ensures a valid frontend dist is available, fetching if necessary. +// It first tries the dynamic dist, then falls back to fetching from GitHub. +// Returns the path to the dist directory, or empty string if no dist is available. +func EnsureDist(ctx context.Context) string { + // If user explicitly configured dist_dir, use that + if conf.Conf != nil && conf.Conf.DistDir != "" { + if _, err := os.Stat(filepath.Join(conf.Conf.DistDir, "index.html")); err == nil { + return conf.Conf.DistDir + } + } + + // Check if dynamic dist already exists + if HasValidDist() { + return filepath.Join(GetDistPath(), distDirName) + } + + // If auto-fetch is enabled (and WebVersion is rolling/beta/dev), try fetching + if shouldAutoFetch() { + utils.Log.Infof("[frontend] no local dist found, fetching from rolling release...") + result, err := FetchFromRolling(ctx) + if err != nil { + utils.Log.Warnf("[frontend] failed to fetch from rolling: %v", err) + // Fall through to return empty (embedded dist will be used as fallback) + return "" + } + return result.DistPath + } + + return "" +} + +func shouldAutoFetch() bool { + v := conf.WebVersion + return v == "" || v == "rolling" || v == "beta" || v == "dev" +} + +// Ensure the directory exists for the frontend dist +func init() { + _ = os.MkdirAll(GetDistPath(), 0755) +} + +// Ensure that the sync.Once pattern is used for the fetcher +var ( + fetchMu sync.Mutex + fetchDone bool + fetchResult string +) + +// EnsureDistOnce is a thread-safe version of EnsureDist that only fetches once per process. +// On failure, it does not lock the state so subsequent calls can retry. +func EnsureDistOnce(ctx context.Context) string { + fetchMu.Lock() + defer fetchMu.Unlock() + if fetchDone { + return fetchResult + } + result := EnsureDist(ctx) + if result != "" { + fetchResult = result + fetchDone = true + } + return result +} + +// ResetFetchState resets the fetch state (used for testing or re-fetch) +func ResetFetchState() { + fetchMu.Lock() + defer fetchMu.Unlock() + fetchDone = false + fetchResult = "" +} diff --git a/internal/frontend/fetcher_test.go b/internal/frontend/fetcher_test.go new file mode 100644 index 000000000..14f7a3391 --- /dev/null +++ b/internal/frontend/fetcher_test.go @@ -0,0 +1,480 @@ +package frontend + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +// createTestTarGz creates a tar.gz containing files with given names and contents +func createTestTarGz(t *testing.T, files map[string]string) []byte { + t.Helper() + pr, pw := io.Pipe() + go func() { + defer pw.Close() + gw := gzip.NewWriter(pw) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return + } + if _, err := tw.Write([]byte(content)); err != nil { + return + } + } + }() + data, err := io.ReadAll(pr) + if err != nil { + t.Fatalf("read tar.gz: %v", err) + } + return data +} + +func TestExtractTarGz(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "dist/index.html": "hello", + "dist/assets/app.js": "console.log('app')", + "dist/assets/style.css": "body {}", + "dist/images/logo.svg": "", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err != nil { + t.Fatalf("extractTarGz: %v", err) + } + + for name, expectedContent := range files { + path := filepath.Join(tmpDir, name) + data, err := os.ReadFile(path) + if err != nil { + t.Errorf("read %s: %v", name, err) + continue + } + if string(data) != expectedContent { + t.Errorf("content of %s: got %q, want %q", name, string(data), expectedContent) + } + } +} + +func TestExtractTarGzDotSlash(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "./dist/index.html": "dot-slash", + "./": "", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err != nil { + t.Fatalf("extractTarGz with ./ prefix: %v", err) + } + + data, err := os.ReadFile(filepath.Join(tmpDir, "dist", "index.html")) + if err != nil { + t.Fatalf("read dist/index.html: %v", err) + } + if string(data) != "dot-slash" { + t.Errorf("got %q, want dot-slash content", string(data)) + } +} + +func TestExtractTarGzPathTraversal(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "../../../etc/passwd": "root:x:0:0", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err == nil { + t.Fatal("expected error for path traversal, got nil") + } + if !strings.Contains(err.Error(), "path traversal") { + t.Errorf("expected path traversal error, got: %v", err) + } +} + +func TestHasValidDist(t *testing.T) { + if HasValidDist() { + t.Log("HasValidDist returned true (may have existing dist from previous runs)") + } +} + +func TestWriteAndReadVersion(t *testing.T) { + _ = os.MkdirAll(GetDistPath(), 0755) + versionPath := GetVersionFilePath() + + origData, origErr := os.ReadFile(versionPath) + defer func() { + if origErr == nil { + _ = os.WriteFile(versionPath, origData, 0644) + } else { + _ = os.Remove(versionPath) + } + }() + + testVersion := "v1.0.0-test" + if err := writeVersion(testVersion); err != nil { + t.Fatalf("writeVersion: %v", err) + } + + got := ReadCurrentVersion() + if got != testVersion { + t.Errorf("ReadCurrentVersion: got %q, want %q", got, testVersion) + } +} + +func TestShouldAutoFetch(t *testing.T) { + origVersion := conf.WebVersion + defer func() { conf.WebVersion = origVersion }() + + tests := []struct { + version string + want bool + }{ + {"", true}, + {"rolling", true}, + {"beta", true}, + {"dev", true}, + {"v3.0.0", false}, + {"latest", false}, + } + + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + conf.WebVersion = tt.version + if got := shouldAutoFetch(); got != tt.want { + t.Errorf("shouldAutoFetch(%q) = %v, want %v", tt.version, got, tt.want) + } + }) + } +} + +func TestFetchFromRollingIntegration(t *testing.T) { + files := map[string]string{ + "./dist/index.html": "integration", + "./dist/assets/test.js": "console.log('test')", + } + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/OpenListTeam/OpenList-Frontend/releases/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling-test", + "assets": [{ + "name": "openlist-frontend-dist.tar.gz", + "browser_download_url": "%s/download/frontend.tar.gz" + }] + }`, ts.URL))) + case "/repos/OpenListTeam/OpenList-Frontend/git/ref/tags/rolling-test": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "0123456789abcdef0123456789abcdef01234567" + } + }`)) + case "/download/frontend.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(200) + w.Write(tarData) + default: + w.WriteHeader(404) + } + })) + defer ts.Close() + + ResetFetchState() + + destDir := GetDistPath() + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + ctx := context.Background() + result, err := fetchFromTag(ctx, "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag: %v", err) + } + + if result.Version != "rolling-test@0123456789ab" { + t.Errorf("version: got %q, want %q", result.Version, "rolling-test@0123456789ab") + } + if !result.Downloaded { + t.Error("expected Downloaded=true") + } + + idx, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(idx) != "integration" { + t.Errorf("index.html: got %q", string(idx)) + } + + ver := ReadCurrentVersion() + expectedVer := "rolling-test@0123456789ab" + if ver != expectedVer { + t.Errorf("version file: got %q, want %q", ver, expectedVer) + } +} + +func TestLegacyConfigJSONGetsDefaultFrontendRepo(t *testing.T) { + cfg := conf.DefaultConfig("data") + if err := json.Unmarshal([]byte(`{"site_url":"https://example.com"}`), cfg); err != nil { + t.Fatalf("unmarshal legacy config: %v", err) + } + if cfg.FrontendRepo != defaultFrontendRepo { + t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, defaultFrontendRepo) + } +} + +func TestDefaultConfigUsesBuiltFrontendRepoDefault(t *testing.T) { + orig := conf.FrontendRepoDefault + conf.FrontendRepoDefault = "Ironboxplus/OpenList-Frontend" + defer func() { conf.FrontendRepoDefault = orig }() + + cfg := conf.DefaultConfig("data") + if cfg.FrontendRepo != "Ironboxplus/OpenList-Frontend" { + t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, "Ironboxplus/OpenList-Frontend") + } +} + +func TestExistingConfigJSONKeepsFrontendRepo(t *testing.T) { + cfg := conf.DefaultConfig("data") + if err := json.Unmarshal([]byte(`{"frontend_repo":"Ironboxplus/OpenList-Frontend"}`), cfg); err != nil { + t.Fatalf("unmarshal config with frontend_repo: %v", err) + } + if cfg.FrontendRepo != "Ironboxplus/OpenList-Frontend" { + t.Fatalf("FrontendRepo: got %q", cfg.FrontendRepo) + } +} + +func TestFetchFromTagUsesConfiguredFrontendRepo(t *testing.T) { + origConf := conf.Conf + if origConf == nil { + conf.Conf = &conf.Config{} + } else { + confCopy := *origConf + conf.Conf = &confCopy + } + defer func() { conf.Conf = origConf }() + conf.Conf.FrontendRepo = "Ironboxplus/OpenList-Frontend" + + files := map[string]string{ + "./dist/index.html": "custom-repo", + } + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/Ironboxplus/OpenList-Frontend/releases/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling-custom", + "assets": [{ + "name": "openlist-frontend-dist.tar.gz", + "browser_download_url": "%s/download/custom.tar.gz" + }] + }`, ts.URL))) + case "/repos/Ironboxplus/OpenList-Frontend/git/ref/tags/rolling-custom": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "fedcba9876543210fedcba9876543210fedcba98" + } + }`)) + case "/download/custom.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(tarData) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + ResetFetchState() + _ = os.MkdirAll(GetDistPath(), 0o755) + _ = os.RemoveAll(filepath.Join(GetDistPath(), distDirName)) + _ = os.Remove(GetVersionFilePath()) + + result, err := fetchFromTag(context.Background(), "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag(custom repo): %v", err) + } + if result.Version != "rolling-custom@fedcba987654" { + t.Fatalf("version: got %q, want %q", result.Version, "rolling-custom@fedcba987654") + } + + data, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(data) != "custom-repo" { + t.Fatalf("index.html: got %q", string(data)) + } +} + +func TestResolveTagCommitSHA_AnnotatedTag(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/OpenListTeam/OpenList-Frontend/git/ref/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{ + "object": { + "type": "tag", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }`)) + case "/repos/OpenListTeam/OpenList-Frontend/git/tags/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + sha, err := resolveTagCommitSHA(context.Background(), ts.Client(), ts.URL, "rolling") + if err != nil { + t.Fatalf("resolveTagCommitSHA: %v", err) + } + if sha != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" { + t.Fatalf("sha: got %q, want %q", sha, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + } +} + +func TestVersionIdentifierFallback(t *testing.T) { + tests := []struct { + name string + tag string + sha string + fallback string + want string + }{ + {name: "hash preferred", tag: "rolling", sha: "0123456789abcdef", fallback: "fallback", want: "rolling@0123456789ab"}, + {name: "fallback url", tag: "rolling", sha: "", fallback: "http://example.com/dist.tar.gz", want: "http://example.com/dist.tar.gz"}, + {name: "tag only", tag: "rolling", sha: "", fallback: "", want: "rolling"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := versionIdentifier(tt.tag, tt.sha, tt.fallback) + if got != tt.want { + t.Fatalf("versionIdentifier() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestResolveTagCommitSHA_RealGitHubWithProxy10808(t *testing.T) { + if os.Getenv("OPENLIST_REAL_GITHUB_TEST") != "1" { + t.Skip("set OPENLIST_REAL_GITHUB_TEST=1 to run real GitHub integration test (proxy 127.0.0.1:10808 recommended)") + } + + if os.Getenv("HTTP_PROXY") == "" && os.Getenv("http_proxy") == "" { + _ = os.Setenv("HTTP_PROXY", "http://127.0.0.1:10808") + } + if os.Getenv("HTTPS_PROXY") == "" && os.Getenv("https_proxy") == "" { + _ = os.Setenv("HTTPS_PROXY", "http://127.0.0.1:10808") + } + + client := newHTTPClient() + sha, err := resolveTagCommitSHA(context.Background(), client, "", "rolling") + if err != nil { + t.Fatalf("resolveTagCommitSHA(real): %v", err) + } + + matched, _ := regexp.MatchString("^[0-9a-f]{40}$", sha) + if !matched { + t.Fatalf("sha format invalid: %q", sha) + } + + // Optional sanity: ensure API can fetch release JSON in real scenario + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "https://api.github.com/repos/OpenListTeam/OpenList-Frontend/releases/tags/rolling", nil) + if err != nil { + t.Fatalf("create release request: %v", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + resp, err := client.Do(req) + if err != nil { + t.Fatalf("fetch release: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("release API status=%d body=%s", resp.StatusCode, string(body)) + } + + var release map[string]any + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + t.Fatalf("decode release: %v", err) + } + if release["tag_name"] == nil { + t.Fatalf("release tag_name missing") + } +} + +func TestEnsureDistOnceFailureDoesNotLock(t *testing.T) { + ResetFetchState() + _ = os.MkdirAll(GetDistPath(), 0755) + + destDir := GetDistPath() + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + origVersion := conf.WebVersion + conf.WebVersion = "v3.0.0" // shouldAutoFetch returns false + defer func() { conf.WebVersion = origVersion }() + + ctx := context.Background() + + result := EnsureDistOnce(ctx) + if result != "" { + t.Errorf("expected empty result, got %q", result) + } + + // Second call should also work (not locked by previous failure) + result2 := EnsureDistOnce(ctx) + if result2 != "" { + t.Errorf("expected empty result on retry, got %q", result2) + } +} diff --git a/internal/frontend/watcher.go b/internal/frontend/watcher.go new file mode 100644 index 000000000..1a0102fd1 --- /dev/null +++ b/internal/frontend/watcher.go @@ -0,0 +1,122 @@ +package frontend + +import ( + "context" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const defaultCheckInterval = 30 * time.Minute + +// Watcher periodically checks for new frontend versions and fetches them. +type Watcher struct { + interval time.Duration + stopCh chan struct{} + stopped bool + mu sync.Mutex + onUpdated func() +} + +// NewWatcher creates a new frontend watcher. +// onUpdated is called when a new version is fetched (used to reload static files). +func NewWatcher(onUpdated func()) *Watcher { + return &Watcher{ + interval: defaultCheckInterval, + stopCh: make(chan struct{}), + onUpdated: onUpdated, + } +} + +// SetInterval changes the check interval. Must be called before Start. +func (w *Watcher) SetInterval(d time.Duration) { + w.mu.Lock() + defer w.mu.Unlock() + w.interval = d +} + +// Start begins the periodic check loop in a background goroutine. +func (w *Watcher) Start() { + w.mu.Lock() + interval := w.interval + w.mu.Unlock() + + go func() { + utils.Log.Infof("[frontend] watcher started, checking every %s", interval) + // Check immediately on start, then periodically + w.check() + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-w.stopCh: + utils.Log.Infof("[frontend] watcher stopped") + return + case <-ticker.C: + w.check() + } + } + }() +} + +// Stop signals the watcher to stop. +func (w *Watcher) Stop() { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped { + return + } + w.stopped = true + close(w.stopCh) +} + +func (w *Watcher) check() { + if !shouldAutoFetch() { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + result, err := FetchFromRolling(ctx) + if err != nil { + utils.Log.Warnf("[frontend] watcher check failed: %v", err) + return + } + + if result.Downloaded { + utils.Log.Infof("[frontend] watcher fetched new version: %s", result.Version) + if w.onUpdated != nil { + w.onUpdated() + } + } +} + +// globalWatcher is the singleton watcher instance +var ( + globalWatcher *Watcher + watcherMu sync.Mutex +) + +// StartWatcher starts the global frontend watcher. +func StartWatcher(onUpdated func()) { + watcherMu.Lock() + defer watcherMu.Unlock() + if globalWatcher != nil { + return + } + globalWatcher = NewWatcher(onUpdated) + globalWatcher.Start() +} + +// StopWatcher stops the global frontend watcher. +func StopWatcher() { + watcherMu.Lock() + defer watcherMu.Unlock() + if globalWatcher != nil { + globalWatcher.Stop() + globalWatcher = nil + } +} diff --git a/public/dist/README.md b/public/dist/README.md deleted file mode 100644 index d8709fb57..000000000 --- a/public/dist/README.md +++ /dev/null @@ -1 +0,0 @@ -## Put dist of frontend here. \ No newline at end of file diff --git a/server/handles/fsread.go b/server/handles/fsread.go index a90fc1082..8a67e4e59 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -230,6 +230,10 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { for _, obj := range objs { thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) + hashInfo := obj.GetHash().Export() + if hashInfo == nil { + hashInfo = make(map[*utils.HashType]string) + } resp = append(resp, ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), @@ -237,7 +241,7 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), + HashInfo: hashInfo, Sign: common.Sign(obj, parent, encrypt), Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), diff --git a/server/static/static.go b/server/static/static.go index 29f97ff74..3850b6d11 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -1,6 +1,7 @@ package static import ( + "context" "encoding/json" "errors" "fmt" @@ -8,10 +9,14 @@ import ( "io/fs" "net/http" "os" + "path/filepath" "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/frontend" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/public" @@ -32,21 +37,60 @@ type Manifest struct { Icons []ManifestIcon `json:"icons"` } -var static fs.FS +// reloadableFS wraps fs.FS with thread-safe swapping. +// This allows gin StaticFS routes (which capture the fs.FS at registration time) +// to serve updated files after a watcher-triggered reload. +type reloadableFS struct { + mu sync.RWMutex + current fs.FS +} + +func (r *reloadableFS) Open(name string) (fs.File, error) { + r.mu.RLock() + current := r.current + r.mu.RUnlock() + return current.Open(name) +} + +func (r *reloadableFS) swap(f fs.FS) { + r.mu.Lock() + r.current = f + r.mu.Unlock() +} + +var staticFS = &reloadableFS{} func initStatic() { utils.Log.Debug("Initializing static file system...") - if conf.Conf.DistDir == "" { - dist, err := fs.Sub(public.Public, "dist") - if err != nil { - utils.Log.Fatalf("failed to read dist dir: %v", err) - } - static = dist - utils.Log.Debug("Using embedded dist directory") + // 1. User explicitly configured dist_dir + if conf.Conf.DistDir != "" { + staticFS.swap(os.DirFS(conf.Conf.DistDir)) + utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) + return + } + // 2. Try dynamic dist (fetched from rolling release) + if frontend.HasValidDist() { + distPath := filepath.Join(frontend.GetDistPath(), "dist") + staticFS.swap(os.DirFS(distPath)) + utils.Log.Infof("Using dynamically fetched dist: %s", distPath) + return + } + // 3. Try auto-fetching from rolling (short timeout to avoid blocking startup) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + distPath := frontend.EnsureDistOnce(ctx) + cancel() + if distPath != "" { + staticFS.swap(os.DirFS(distPath)) + utils.Log.Infof("Using auto-fetched dist: %s", distPath) return } - static = os.DirFS(conf.Conf.DistDir) - utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) + // 4. Final fallback to embedded dist + dist, err := fs.Sub(public.Public, "dist") + if err != nil { + utils.Log.Fatalf("failed to read dist dir: %v", err) + } + staticFS.swap(dist) + utils.Log.Debug("Using embedded dist directory") } func replaceStrings(content string, replacements map[string]string) string { @@ -74,7 +118,7 @@ func initIndex(siteConfig SiteConfig) { utils.Log.Info("Successfully fetched index.html from CDN") } else { utils.Log.Debug("Reading index.html from static files system...") - indexFile, err := static.Open("index.html") + indexFile, err := staticFS.Open("index.html") if err != nil { if errors.Is(err, fs.ErrNotExist) { utils.Log.Fatalf("index.html not exist, you may forget to put dist of frontend to public/dist") @@ -131,13 +175,21 @@ func UpdateIndex() { utils.Log.Debug("Index.html update completed") } +// ReloadStatic reloads the static files from disk (called by the watcher after an update) +func ReloadStatic() { + utils.Log.Info("[static] reloading static files after frontend update...") + siteConfig := getSiteConfig() + initStatic() + initIndex(siteConfig) +} + func ManifestJSON(c *gin.Context) { // Get site configuration to ensure consistent base path handling siteConfig := getSiteConfig() - + // Get site title from settings siteTitle := setting.GetStr(conf.SiteTitle) - + // Get logo from settings, use the first line (light theme logo) logoSetting := setting.GetStr(conf.Logo) logoUrl := strings.Split(logoSetting, "\n")[0] @@ -167,7 +219,7 @@ func ManifestJSON(c *gin.Context) { c.Header("Content-Type", "application/json") c.Header("Cache-Control", "public, max-age=3600") // cache for 1 hour - + if err := json.NewEncoder(c.Writer).Encode(manifest); err != nil { utils.Log.Errorf("Failed to encode manifest.json: %v", err) c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to generate manifest"}) @@ -180,8 +232,12 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { siteConfig := getSiteConfig() initStatic() initIndex(siteConfig) + + // Start the frontend watcher for periodic updates + frontend.StartWatcher(ReloadStatic) + folders := []string{"assets", "images", "streamer", "static"} - + if conf.Conf.Cdn == "" { utils.Log.Debug("Setting up static file serving...") r.Use(func(c *gin.Context) { @@ -192,7 +248,7 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { } }) for _, folder := range folders { - sub, err := fs.Sub(static, folder) + sub, err := fs.Sub(staticFS, folder) if err != nil { utils.Log.Fatalf("can't find folder: %s", folder) } From bc1438124d4d62780cc0df95829b244bb86a2fca Mon Sep 17 00:00:00 2001 From: cyk Date: Sat, 25 Apr 2026 21:32:12 +0800 Subject: [PATCH 86/86] chore: add project docs and update dependencies (115-sdk-go fork) - Add CLAUDE.md for project guidance and development instructions - Add compatibility report - Use Ironboxplus/115-sdk-go v0.2.5 fork in go.mod replace directive --- CLAUDE.md | 346 ++++++++++++++++++++++++++++++++++++++++ COMPATIBILITY_REPORT.md | 204 +++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 4 files changed, 553 insertions(+), 3 deletions(-) create mode 100644 CLAUDE.md create mode 100644 COMPATIBILITY_REPORT.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..c3ffdded0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,346 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Core Development Principles + +1. **最小代码改动原则** (Minimum code changes): Make the smallest change necessary to achieve the goal +2. **不缓存整个文件原则** (No full file caching for seekable streams): For SeekableStream, use RangeRead instead of caching entire file +3. **必要情况下可以多遍上传原则** (Multi-pass upload when necessary): If rapid upload fails, fall back to normal upload + +## Build and Development Commands + +```bash +# Development +go run main.go # Run backend server (default port 5244) +air # Hot reload during development (uses .air.toml) +./build.sh dev # Build development version with frontend +./build.sh release # Build release version + +# Testing +go test ./... # Run all tests + +# Docker +docker-compose up # Run with docker-compose +docker build -f Dockerfile . # Build docker image +``` + +**Build Script Details** (`build.sh`): +- Fetches frontend from OpenListTeam/OpenList-Frontend releases +- Injects version info via ldflags: `-X "github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$(date +'%F %T %z')"` +- Supports `dev`, `beta`, and release builds +- Downloads prebuilt frontend distribution automatically + +**Go Version**: Requires Go 1.23.4+ + +## Architecture Overview + +### Driver System (Storage Abstraction) + +OpenList uses a **driver pattern** to support 70+ cloud storage providers. Each driver implements the core `Driver` interface. + +**Location**: `drivers/*/` + +**Core Interfaces** (`internal/driver/driver.go`): +- `Reader`: List directories, generate download links (REQUIRED) +- `Writer`: Upload, delete, move files (optional) +- `ArchiveDriver`: Extract archives (optional) +- `LinkCacheModeResolver`: Custom cache TTL strategies (optional) + +**Driver Registration Pattern**: +```go +// In drivers/your_driver/meta.go +var config = driver.Config{ + Name: "YourDriver", + LocalSort: false, + NoCache: false, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &YourDriver{} + }) +} +``` + +**Adding a New Driver**: +1. Copy `drivers/template/` to `drivers/your_driver/` +2. Implement `List()` and `Link()` methods (required) +3. Define `Addition` struct with configuration fields using struct tags: + - `json:"field_name"` - JSON field name + - `type:"select"` - Input type (select, string, text, bool, number) + - `required:"true"` - Required field + - `options:"a,b,c"` - Dropdown options + - `default:"value"` - Default value +4. Register driver in `init()` function + +**Example Driver Structure**: +```go +type YourDriver struct { + model.Storage + Addition + client *YourClient +} + +func (d *YourDriver) Init(ctx context.Context) error { + // Initialize client, login, etc. +} + +func (d *YourDriver) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + // Return list of files/folders +} + +func (d *YourDriver) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + // Return download URL or RangeReader +} +``` + +### Request Flow + +``` +HTTP Request (Gin Router) + ↓ +Middleware (Auth, CORS, Logging) + ↓ +Handler (server/handles/) + ↓ +fs.List/Get/Link (mount path → storage path conversion) + ↓ +op.List/Get/Link (caching, driver lookup) + ↓ +Driver.List/Link (storage-specific API calls) + ↓ +Response (JSON / Proxy / Redirect) +``` + +### Internal Package Structure + +| Package | Purpose | +|---------|---------| +| `bootstrap/` | Initialization sequence: config, DB, storages, servers | +| `conf/` | Configuration management | +| `db/` | Database models (SQLite/MySQL/Postgres) | +| `driver/` | Driver interface definitions | +| `fs/` | Mount path abstraction (converts `/mount/path` to storage + path) | +| `op/` | Core operations with caching and driver management | +| `stream/` | Streaming, range readers, link refresh, rate limiting | +| `model/` | Data models (Obj, Link, Storage, User) | +| `cache/` | Multi-level caching (directories, links, users, settings) | +| `net/` | HTTP utilities, proxy config, download manager | + +### Link Generation and Caching + +**Link Types**: +1. **Direct URL** (`link.URL`): Simple redirect to storage provider +2. **RangeReader** (`link.RangeReader`): Custom streaming implementation +3. **Refreshable Link** (`link.Refresher`): Auto-refresh on expiration + +**Cache System** (`internal/op/cache.go`): +- **Directory Cache**: Stores file listings with configurable TTL +- **Link Cache**: Stores download URLs (30min default) +- **User Cache**: Authentication data (1hr default) +- **Custom Policies**: Pattern-based TTL via `pattern:ttl` format + +**Cache Key Pattern**: `{storageMountPath}/{relativePath}` + +**Invalidation**: Recursive tree deletion for directory operations + +### Range Reader and Streaming + +**Location**: `internal/stream/` + +**Purpose**: Handle partial content requests (HTTP 206), multi-threaded downloads, and link refresh during streaming. + +**Key Components**: + +1. **RangeReaderIF**: Core interface for range-based reading + ```go + type RangeReaderIF interface { + RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) + } + ``` + +2. **RefreshableRangeReader**: Wraps RangeReader with automatic link refresh + - Detects expired links via error strings or HTTP status codes (401, 403, 410, 500) + - Calls `link.Refresher(ctx)` to get new link + - Resumes download from current byte position + - Max 3 refresh attempts to prevent infinite loops + +3. **Multi-threaded Downloader** (`internal/net/downloader.go`): + - Splits file into parts based on `Concurrency` and `PartSize` + - Downloads parts in parallel + - Assembles final stream + +**Stream Types and Reader Management**: + +⚠️ **CRITICAL**: SeekableStream.Reader must NEVER be created early! + +- **FileStream**: One-time sequential stream (e.g., HTTP body) + - `Reader` is set at creation and consumed sequentially + - Cannot be rewound or re-read + +- **SeekableStream**: Reusable stream with RangeRead capability + - Has `rangeReader` for creating new readers on-demand + - `Reader` should ONLY be created when actually needed for sequential reading + - **DO NOT create Reader early** - use lazy initialization via `generateReader()` + +**Common Pitfall - Early Reader Creation**: +```go +// ❌ WRONG: Creating Reader early +if _, ok := rr.(*model.FileRangeReader); ok { + rc, _ := rr.RangeRead(ctx, http_range.Range{Length: -1}) + fs.Reader = rc // This will be consumed by intermediate operations! +} + +// ✅ CORRECT: Let generateReader() create it on-demand +// Reader will be created only when Read() is called +return &SeekableStream{FileStream: fs, rangeReader: rr}, nil +``` + +**Why This Matters**: +- Hash calculation uses `StreamHashFile()` which reads the file via RangeRead +- If Reader is created early, it may be at EOF when HTTP upload actually needs it +- Result: `http: ContentLength=X with Body length 0` error + +**Hash Calculation for Uploads**: +```go +// For SeekableStream: Use RangeRead to avoid consuming Reader +if _, ok := file.(*SeekableStream); ok { + hash, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + // StreamHashFile uses RangeRead internally, Reader remains unused +} + +// For FileStream: Must cache first, then calculate hash +_, hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) +``` + +**Link Refresh Pattern**: +```go +// In op.Link(), a refresher is automatically attached +link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + // Get fresh link from storage driver + file, err := GetUnwrap(refreshCtx, storage, path) + newLink, err := storage.Link(refreshCtx, file, args) + return newLink, file, nil +} + +// RefreshableRangeReader uses this during streaming +if IsLinkExpiredError(err) && r.link.Refresher != nil { + newLink, _, err := r.link.Refresher(ctx) + // Resume from current position +} +``` + +**Proxy Function** (`server/common/proxy.go`): + +Handles multiple scenarios: +1. Multi-threaded download (`link.Concurrency > 0`) +2. Direct RangeReader (`link.RangeReader != nil`) +3. Refreshable link (`link.Refresher != nil`) ← Wraps with RefreshableRangeReader +4. Transparent proxy (forwards to `link.URL`) + +### Startup Sequence + +**Location**: `internal/bootstrap/run.go` + +Order of initialization: +1. `InitConfig()` - Load config, environment variables +2. `Log()` - Initialize logging +3. `InitDB()` - Connect to database +4. `data.InitData()` - Initialize default data +5. `LoadStorages()` - Load and initialize all storage drivers +6. `InitTaskManager()` - Start background tasks +7. `Start()` - Start HTTP/HTTPS/WebDAV/FTP/SFTP servers + +## Common Patterns + +### Error Handling + +Use custom errors from `internal/errs/`: +- `errs.NotImplement` - Feature not implemented +- `errs.ObjectNotFound` - File/folder not found +- `errs.NotFolder` - Path is not a directory +- `errs.StorageNotInit` - Storage driver not initialized + +**Link Expiry Detection**: +```go +// Checks error string for keywords: "expired", "invalid signature", "token expired" +// Also checks HTTP status: 401, 403, 410, 500 +if stream.IsLinkExpiredError(err) { + // Refresh link +} +``` + +### Saving Driver State + +When updating tokens or credentials: +```go +d.AccessToken = newToken +op.MustSaveDriverStorage(d) // Persists to database +``` + +### Rate Limiting + +Use `rate.Limiter` for API rate limits: +```go +type YourDriver struct { + limiter *rate.Limiter +} + +func (d *YourDriver) Init(ctx context.Context) error { + d.limiter = rate.NewLimiter(rate.Every(time.Second), 1) // 1 req/sec +} + +func (d *YourDriver) List(...) { + d.limiter.Wait(ctx) + // Make API call +} +``` + +### Context Cancellation + +Always respect context cancellation in long operations: +```go +select { +case <-ctx.Done(): + return nil, ctx.Err() +default: + // Continue operation +} +``` + +## Important Conventions + +**Naming**: +- Drivers: lowercase with underscores (e.g., `baidu_netdisk`, `aliyundrive_open`) +- Packages: lowercase (e.g., `internal/op`) +- Interfaces: PascalCase with suffix (e.g., `Reader`, `Writer`) + +**Driver Configuration Fields**: +- Use `driver.RootPath` or `driver.RootID` for root folder +- Add `omitempty` to optional JSON fields +- Use descriptive help text in struct tags + +**Retries and Timeouts**: +- Use `github.com/avast/retry-go` for retry logic +- Set reasonable timeouts on HTTP clients (default 30s in `base.RestyClient`) +- For unstable APIs, implement exponential backoff + +**Logging**: +- Use `logrus` via `log` package +- Levels: `log.Debugf`, `log.Infof`, `log.Warnf`, `log.Errorf` +- Include driver name in logs: `log.Infof("[driver_name] message")` + +## Project Context + +OpenList is a community-driven fork of AList, focused on: +- Long-term governance and trust +- Support for 70+ cloud storage providers +- Web UI for file management +- Multi-protocol support (HTTP, WebDAV, FTP, SFTP, S3) +- Offline downloads (Aria2, Transmission) +- Full-text search +- Archive extraction + +**License**: AGPL-3.0 diff --git a/COMPATIBILITY_REPORT.md b/COMPATIBILITY_REPORT.md new file mode 100644 index 000000000..0b8be3b52 --- /dev/null +++ b/COMPATIBILITY_REPORT.md @@ -0,0 +1,204 @@ +# Rebase兼容性分析报告 + +## 提交概览 +共引入 **21个commits**，主要涉及以下模块： + +### 核心功能改动 + +#### 1. **链接刷新机制** (`internal/stream/util.go`) +**Commits**: +- `4c33ffa4` feat(link): add link refresh capability for expired download links +- `f38fe180` fix(stream): 修复链接过期检测逻辑，避免将上下文取消视为链接过期 +- `7cf362c6` fix(stream): 更新过期链接检查逻辑，支持所有4xx客户端错误 +- `03fbaf1c` refactor(stream): 移除过时的链接刷新逻辑，添加自愈读取器以处理0字节读取 + +**核心代码**: +```go +// 新增常量 +MAX_LINK_REFRESH_COUNT = 50 // 链接最大刷新次数 +MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead重试次数（从3提升到5） + +// 新增函数 +IsLinkExpiredError(err error) bool // 判断是否为链接过期错误 + +// 新增结构 +RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // 防止无限循环 +} + +selfHealingReadCloser struct { + // 检测0字节读取，自动刷新链接 +} +``` + +**功能说明**: +1. **链接过期检测**: 识别多种云盘的过期错误（expired, token expired, access denied, 4xx状态码等） +2. **自动刷新**: 检测到过期时自动调用Refresher获取新链接，最多刷新50次 +3. **自愈机制**: 处理某些云盘返回200但内容为空的情况（0字节读取检测） +4. **并发安全**: 使用sync.Mutex保护共享状态 +5. **Context隔离**: 刷新时使用WithoutCancel避免用户取消操作影响刷新 + +**潜在风险**: +- ✅ Context.WithoutCancel需要Go 1.21+ +- ✅ 并发场景下的锁竞争 +- ✅ refreshCount可能在某些场景下不递增导致无限循环 + +--- + +#### 2. **目录预创建优化** (`internal/fs/copy_move.go`) +**Commit**: `ce0da112` fix(copy_move): 将预创建子目录的深度从2级调整为1级 + +**核心代码**: +```go +func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath string, maxDepth int) error { + // 第一轮：创建直接子目录 + for _, obj := range objs { + if obj.IsDir() { + subdirPath := stdpath.Join(dstBasePath, obj.GetName()) + op.MakeDir(t.Ctx(), t.DstStorage, subdirPath) + subdirs = append(subdirs, obj) + } + } + + // 停止递归条件 + if maxDepth <= 0 { + return nil + } + + // 第二轮：递归创建嵌套目录 + for _, subdir := range subdirs { + subObjs := op.List(...) + preCreateDirectoryTree(subObjs, subdirDstPath, maxDepth-1) + } +} +``` + +**功能说明**: +1. **深度控制**: 默认maxDepth=1，只预创建2级目录（当前+子级） +2. **防止深度递归**: 避免在大型项目中递归过深导致栈溢出或性能问题 +3. **错误容忍**: MakeDir失败时继续处理其他目录 +4. **Context感知**: 每次循环检查ctx.Err()支持取消操作 + +**潜在风险**: +- ✅ op.MakeDir和op.List调用需要存储初始化 +- ✅ 大量目录时的性能问题 +- ✅ Context取消时的资源清理 + +--- + +#### 3. **网络优化** (`drivers/`, `internal/net/`) +**Commits**: +- `b9dafa65` feat(network): 增加对慢速网络的支持，调整超时和重试机制 +- `bce47884` fix(driver): 增加夸克分片大小调整逻辑，支持重试机制 +- `0b8471f6` feat(quark_open): 添加速率限制和重试逻辑 + +**功能说明**: +1. 提升RangeRead重试次数: 3 → 5 +2. 调整网络超时参数 +3. 添加分片上传重试逻辑 + +--- + +#### 4. **驱动修复** +**Commits**: +- `da2812c0` fix(google_drive): 更新Put方法以支持可重复读取流和不可重复读取流的MD5校验 +- `5a6bad90` feat(google_drive): 添加文件夹创建的锁机制和重试逻辑 +- `a54b2388` feat(google_drive): 添加处理重复文件名的功能 +- `9ef22ec9` fix(driver): fix file copy failure to 123pan due to incorrect etag +- `0ead87ef` fix(alias): update storage retrieval method in listRoot function +- `311f6246` fix: 修复500 panic和NaN问题 + +--- + +## 兼容性评估 + +### ✅ 编译兼容性 +- 构建成功，无语法错误 +- 依赖版本无冲突 + +### ✅ API兼容性 +- 新增函数不破坏现有接口 +- RefreshableRangeReader实现model.RangeReaderIF接口 +- 向后兼容旧代码 + +### ⚠️ 运行时兼容性 +**需要验证的场景**: +1. **并发安全**: RefreshableRangeReader的并发读取 +2. **资源泄漏**: Context取消时goroutine是否正确退出 +3. **边界条件**: + - refreshCount达到50次的行为 + - 0字节读取检测的准确性 + - maxDepth=0时的目录创建 +4. **错误处理**: + - nil Refresher时的处理 + - 链接刷新失败时的回退机制 +5. **性能**: + - 大文件下载时的刷新开销 + - 深层目录结构的预创建性能 + +--- + +## 测试需求 + +### 必须测试的场景 + +#### Stream包测试 +1. **IsLinkExpiredError准确性** + - 各种云盘的过期错误格式 + - Context取消不应判断为过期 + - HTTP 4xx/5xx的区分 + +2. **RefreshableRangeReader可靠性** + - 正常读取流程 + - 自动刷新触发和成功 + - 达到最大刷新次数 + - 并发读取安全性 + - Context取消的正确处理 + +3. **selfHealingReadCloser** + - 0字节读取检测 + - 刷新重试机制 + - 资源正确关闭 + +#### FS包测试 +1. **preCreateDirectoryTree** + - 深度控制正确性(0, 1, 2级) + - 大量目录的性能 + - Context取消的响应 + - 错误容忍性 + +--- + +## 风险等级: **中等** + +**原因**: +- ✅ 新功能设计合理，有明确的边界和错误处理 +- ⚠️ 并发场景需要充分测试 +- ⚠️ 链接刷新逻辑复杂，需要验证各种边界情况 +- ⚠️ 依赖op包的函数需要正确的初始化 + +--- + +## 推送建议: **通过测试后可推送** + +**前置条件**: +1. 完成全面的单元测试（见下方测试代码） +2. 验证并发安全性 +3. 确认Context取消不会导致资源泄漏 +4. 性能测试通过（大文件、深层目录） + +**建议测试命令**: +```bash +# 单元测试 +go test ./internal/stream ./internal/fs -v -count=1 -race + +# 压力测试 +go test ./internal/stream -run Stress -v -count=10 + +# 完整测试套件 +go test ./... -short -count=1 +``` diff --git a/go.mod b/go.mod index cd86a8147..cf8089ca2 100644 --- a/go.mod +++ b/go.mod @@ -312,4 +312,4 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -// replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.5 diff --git a/go.sum b/go.sum index 758741249..1a1288506 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= +github.com/Ironboxplus/115-sdk-go v0.2.5 h1:8giRpk9TwDT/5oe6F1H6h5+OwMMkM/vSYJqZomsmp20= +github.com/Ironboxplus/115-sdk-go v0.2.5/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= @@ -29,8 +31,6 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/OpenListTeam/115-sdk-go v0.2.3 h1:nDNz0GxgliW+nT2Ds486k/rp/GgJj7Ngznc98ZBUwZo= -github.com/OpenListTeam/115-sdk-go v0.2.3/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs=