diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index 06752f537..7dadfc3f4 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -13,7 +13,7 @@ body: attributes: label: 请确认以下事项 description: | - 您必须勾选以下内容，否则您的问题可能会被直接关闭。 + 您必须阅读并检查以下内容，否则您的问题一定会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | @@ -34,6 +34,8 @@ body: 我认为此问题必须由`OpenList`处理，而非第三方。 - label: | 我已确认这个问题在最新版本中没有被修复。 + - label: | + 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue - type: input id: version @@ -60,19 +62,21 @@ body: validations: required: true - type: textarea - id: config + id: logs attributes: - label: 配置文件内容（必填） + label: 日志（必填） description: | - 请提供您的`OpenList`应用的配置文件，并截图相关存储配置。(可隐藏隐私字段) + 请复制粘贴错误日志，或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug) validations: required: true - type: textarea - id: logs + id: config attributes: - label: 日志（可选） + label: 配置文件内容（必填） description: | - 请复制粘贴错误日志，或者截图。(可隐藏隐私字段) [查看方法](https://doc.oplist.org/faq/howto#%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%9A%E4%BD%8Dbug) + 请提供您的`OpenList`应用的配置文件，并截图相关存储配置。(可隐藏隐私字段) + validations: + required: true - type: textarea id: reproduction attributes: diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index 5d263c3c5..5d5d91d00 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -13,7 +13,7 @@ body: attributes: label: Please confirm the following description: | - You must check all the following, otherwise your issue may be closed directly. + You must read and check all the following, otherwise your issue will definitely be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | @@ -34,6 +34,8 @@ body: I believe this issue must be handled by `OpenList` and not by a third party. - label: | I confirm this issue is not fixed in the latest version. + - label: | + I have not read these checkboxes and therefore I just ticked them all, Please close this issue - type: input id: version @@ -60,19 +62,21 @@ body: validations: required: true - type: textarea - id: config + id: logs attributes: - label: Configuration File Content (required) + label: Logs (required) description: | - Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields) + Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs) validations: required: true - type: textarea - id: logs + id: config attributes: - label: Logs (optional) + label: Configuration File Content (required) description: | - Please copy and paste any relevant log output or screenshots. (You may mask sensitive fields) [Guide](https://doc.oplist.org/faq/howto#how-to-quickly-locate-bugs) + Please provide your `OpenList` application's configuration file and a screenshot of the relevant storage configuration. (You may mask sensitive fields) + validations: + required: true - type: textarea id: reproduction attributes: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e74e59631..f1687eabf 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,11 +2,13 @@ Provide a general summary of your changes in the Title above. The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`. If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body. + For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`. --> ## Description / 描述 diff --git a/.github/workflows/issue_pr_comment.yml b/.github/workflows/issue_pr_comment.yml index 44f6e0aff..f15acd424 100644 --- a/.github/workflows/issue_pr_comment.yml +++ b/.github/workflows/issue_pr_comment.yml @@ -20,7 +20,7 @@ jobs: with: script: | const issueBody = context.payload.issue.body || ""; - const unchecked = /- \[ \] /.test(issueBody); + const unchecked = /- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); let comment = "感谢您联系OpenList。我们会尽快回复您。\n"; comment += "Thanks for contacting OpenList. We will reply to you as soon as possible.\n\n"; if (unchecked) { @@ -47,12 +47,14 @@ jobs: with: script: | const title = context.payload.pull_request.title || ""; - const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\): /i.test(title); + const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\)!?: /i.test(title); if (!ok) { let comment = "⚠️ PR 标题需以 `feat(): `, `docs(): `, `fix(): `, `style(): `, `refactor(): `, `chore(): ` 其中之一开头，例如：`feat(component): 新增功能`。\n"; comment += "⚠️ The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`.\n\n"; comment += "如果跨多个组件，请使用主要组件作为前缀，并在标题中枚举、描述中说明。\n"; comment += "If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body.\n\n"; + comment += "如果是破坏性变更，请在类型后添加 `!`，例如 `feat(component)!: 破坏性变更`。\n"; + comment += "For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`.\n\n"; await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, diff --git a/.gitignore b/.gitignore index 1d71f0d60..d56e0e161 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ output/ /public/dist/* /!public/dist/README.md -.VSCodeCounter \ No newline at end of file +.VSCodeCounter/OpenList +/OpenList diff --git a/README.md b/README.md index 405377f18..c4f29462e 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,9 @@ Thank you for your support and understanding of the OpenList project. - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([China](https://www.teambition.com), [International](https://us.teambition.com)) - - [x] [Mediatrack](https://www.mediatrack.cn) - [x] [MediaFire](https://www.mediafire.com) + - [x] [Mediatrack](https://www.mediatrack.cn) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) @@ -121,12 +122,17 @@ Thank you for your support and understanding of the OpenList project. ## Demo -N/A (to be rebuilt) +- 🌎 [Global Demo](https://demo.oplist.org) +- 🇨🇳 [CN Demo](https://demo.oplist.org.cn) ## Discussion Please refer to [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) for raising general questions, ***Issues* is for bug reports and feature requests only.** +## Sponsor + +[![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") + ## License The `OpenList` is open-source software licensed under the [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) license. diff --git a/README_cn.md b/README_cn.md index 4d58f31f6..adac3d0b9 100644 --- a/README_cn.md +++ b/README_cn.md @@ -64,8 +64,9 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 - [x] [又拍云对象存储](https://www.upyun.com/products/file-storage) - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([中国](https://www.teambition.com), [国际](https://us.teambition.com)) - - [x] [分秒帧](https://www.mediatrack.cn) - [x] [MediaFire](https://www.mediafire.com) + - [x] [分秒帧](https://www.mediatrack.cn) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [和彩云](https://yun.139.com)（个人、家庭、群组） - [x] [YandexDisk](https://disk.yandex.com) - [x] [百度网盘](http://pan.baidu.com) @@ -121,12 +122,17 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目，遵循 AGPL-3 ## 演示 -N/A（待重建） +- 🇨🇳 [国内演示站](https://demo.oplist.org.cn) +- 🌎 [海外演示站](https://demo.oplist.org) ## 讨论 如有一般性问题请前往 [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) 讨论区，***Issues* 仅用于错误报告和功能请求。** +## 赞助者 + +[![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") + ## 许可证 `OpenList` 是基于 [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) 许可证的开源软件。 diff --git a/README_ja.md b/README_ja.md index 3eac9a691..52c1a01c0 100644 --- a/README_ja.md +++ b/README_ja.md @@ -65,6 +65,7 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい - [x] [WebDAV](https://en.wikipedia.org/wiki/WebDAV) - [x] Teambition([中国](https://www.teambition.com), [国際](https://us.teambition.com)) - [x] [Mediatrack](https://www.mediatrack.cn) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com)（個人、家族、グループ） - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) @@ -121,12 +122,17 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい ## デモ -N/A（再構築中） +- 🌎 [グローバルデモ](https://demo.oplist.org) +- 🇨🇳 [CNデモ](https://demo.oplist.org.cn) ## ディスカッション 一般的な質問は [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) をご利用ください。***Issues* はバグ報告と機能リクエスト専用です。** +## スポンサー + +[![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") + ## ライセンス 「OpenList」は [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) ライセンスの下で公開されているオープンソースソフトウェアです。 diff --git a/README_nl.md b/README_nl.md index 469e86028..8b9e62ec7 100644 --- a/README_nl.md +++ b/README_nl.md @@ -66,6 +66,7 @@ Dank u voor uw ondersteuning en begrip - [x] Teambition([China](https://www.teambition.com), [Internationaal](https://us.teambition.com)) - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com) (Persoonlijk, Familie, Groep) - [x] [YandexDisk](https://disk.yandex.com) - [x] [BaiduNetdisk](http://pan.baidu.com) @@ -121,12 +122,17 @@ Dank u voor uw ondersteuning en begrip ## Demo -N.v.t. (wordt opnieuw opgebouwd) +- 🌎 [Global Demo](https://demo.oplist.org) +- 🇨🇳 [CN Demo](https://demo.oplist.org.cn) ## Discussie Stel algemene vragen in [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions), ***Issues* zijn alleen voor bugmeldingen en feature requests.** +## Sponsoren + +[![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.") + ## Licentie `OpenList` is open-source software onder de [AGPL-3.0](https://www.gnu.org/licenses/agpl-3.0.txt) licentie. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..ff558d64e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,89 @@ +# Security Policy + +## Supported Versions + +Only the latest stable release receives security patches. We strongly recommend always keeping OpenList up to date. + +| Version | Supported | +| -------------------- | ------------------ | +| Latest stable (v4.x) | :white_check_mark: | +| Older versions | :x: | + +## Reporting a Vulnerability + +**Please do NOT report security vulnerabilities through public GitHub Issues.** + +If you discover a security vulnerability in OpenList, please report it responsibly by using one of the following channels: + +- **GitHub Private Security Advisory** (preferred): [Submit here](https://github.com/OpenListTeam/OpenList/security/advisories/new) +- **Telegram**: Contact a maintainer privately via [@OpenListTeam](https://t.me/OpenListTeam) + +When reporting, please include as much of the following as possible: + +- A description of the vulnerability and its potential impact +- The affected version(s) +- Step-by-step instructions to reproduce the issue +- Any proof-of-concept code or screenshots (if applicable) +- Suggested mitigation or fix (optional but appreciated) + +## Security Best Practices for Users + +To keep your OpenList instance secure: + +- Always update to the latest release. +- Use a strong, unique admin password and change it after first login. +- Enable HTTPS (TLS) for your deployment — do **not** expose OpenList over plain HTTP on the public internet. +- Limit exposed ports using a reverse proxy (e.g., Nginx, Caddy). +- Set up access controls and avoid enabling guest access unless necessary. +- Regularly review mounted storage permissions and revoke unused API tokens. +- When using Docker, avoid running the container as root if possible. + +## Acknowledgments + +We sincerely thank all security researchers and community members who responsibly disclose vulnerabilities and help make OpenList safer for everyone. + +--- + +# 安全政策 + +## 支持的版本 + +我们仅对最新稳定版本提供安全补丁。强烈建议始终保持 OpenList 为最新版本。 + +| 版本 | 是否支持 | +| ------------------ | ------------------ | +| 最新稳定版（v4.x） | :white_check_mark: | +| 旧版本 | :x: | + +## 报告漏洞 + +**请勿通过公开的 GitHub Issues 报告安全漏洞。** + +如果您在 OpenList 中发现安全漏洞，请通过以下渠道之一负责任地进行报告： + +- **GitHub 私密安全公告**（推荐）：[点击提交](https://github.com/OpenListTeam/OpenList/security/advisories/new) +- **Telegram**：通过 [@OpenListTeam](https://t.me/OpenListTeam) 私信联系维护者 + +报告时，请尽量提供以下信息： + +- 漏洞描述及其潜在影响 +- 受影响的版本 +- 复现问题的详细步骤 +- 概念验证代码或截图（如有） +- 建议的缓解措施或修复方案（可选，但非常欢迎） + +## 用户安全最佳实践 + +为保障您的 OpenList 实例安全： + +- 始终更新至最新版本。 +- 使用强且唯一的管理员密码，并在首次登录后立即修改。 +- 为您的部署启用 HTTPS（TLS）—— **请勿**在公网上以明文 HTTP 方式暴露 OpenList。 +- 使用反向代理（如 Nginx、Caddy）限制对外暴露的端口。 +- 配置访问控制，非必要情况下不要开启访客访问。 +- 定期检查已挂载存储的权限，并撤销未使用的 API 令牌。 +- 使用 Docker 部署时，尽可能避免以 root 用户运行容器。 + +## 致谢 + +我们衷心感谢所有负责任地披露漏洞、帮助 OpenList 变得更加安全的安全研究人员和社区成员。 diff --git a/cmd/admin.go b/cmd/admin.go index 447c49703..5e09959cf 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -20,8 +21,8 @@ var AdminCmd = &cobra.Command{ Aliases: []string{"password"}, Short: "Show admin user's info and some operations about admin user's password", Run: func(cmd *cobra.Command, args []string) { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("failed get admin user: %+v", err) @@ -61,8 +62,8 @@ var ShowTokenCmd = &cobra.Command{ Use: "token", Short: "Show admin token", Run: func(cmd *cobra.Command, args []string) { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() token := setting.GetStr(conf.Token) utils.Log.Infof("show admin token from CLI") fmt.Println("Admin token:", token) @@ -70,8 +71,8 @@ var ShowTokenCmd = &cobra.Command{ } func setAdminPassword(pwd string) { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("failed get admin user: %+v", err) diff --git a/cmd/cancel2FA.go b/cmd/cancel2FA.go index 809d32e37..3ddd7f838 100644 --- a/cmd/cancel2FA.go +++ b/cmd/cancel2FA.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/spf13/cobra" @@ -16,8 +17,8 @@ var Cancel2FACmd = &cobra.Command{ Use: "cancel2fa", Short: "Delete 2FA of admin user", Run: func(cmd *cobra.Command, args []string) { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("failed to get admin user: %+v", err) diff --git a/cmd/common.go b/cmd/common.go index 6835a0f5c..e334fc1fa 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -6,24 +6,16 @@ import ( "strconv" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" - "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/data" - "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) func Init() { - bootstrap.InitConfig() - bootstrap.Log() - bootstrap.InitDB() - data.InitData() - bootstrap.InitStreamLimit() - bootstrap.InitIndex() - bootstrap.InitUpgradePatch() + bootstrap.Init() } func Release() { - db.Close() + bootstrap.Release() } var pid = -1 diff --git a/cmd/crypt.go b/cmd/crypt.go index 10b3352cd..12f51f5cd 100644 --- a/cmd/crypt.go +++ b/cmd/crypt.go @@ -1,19 +1,17 @@ package cmd import ( - log "github.com/sirupsen/logrus" - "io" "os" "path" "path/filepath" "strings" - "github.com/spf13/cobra" - rcCrypt "github.com/rclone/rclone/backend/crypt" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/obscure" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) // encryption and decryption command format for Crypt driver diff --git a/cmd/flags/config.go b/cmd/flags/config.go index f74e2cb4b..0409731e8 100644 --- a/cmd/flags/config.go +++ b/cmd/flags/config.go @@ -2,6 +2,7 @@ package flags var ( DataDir string + ConfigPath string Debug bool NoPrefix bool Dev bool diff --git a/cmd/lang.go b/cmd/lang.go index a03b3c561..c18877cd7 100644 --- a/cmd/lang.go +++ b/cmd/lang.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "os" - "reflect" "strings" _ "github.com/OpenListTeam/OpenList/v4/drivers" @@ -69,15 +68,33 @@ func writeFile(name string, data interface{}) { log.Errorf("failed to unmarshal json: %+v", err) return } - if reflect.DeepEqual(oldData, newData) { + if mergeJson(newData, oldData) { log.Infof("%s.json no changed, skip", name) } else { log.Infof("%s.json changed, update file", name) //log.Infof("old: %+v\nnew:%+v", oldData, data) - utils.WriteJsonToFile(fmt.Sprintf("lang/%s.json", name), newData, true) + utils.WriteJsonToFile(fmt.Sprintf("lang/%s.json", name), oldData, true) } } +func mergeJson(source, target map[string]interface{}) bool { + equal := true + for k, v := range source { + tgtV, tgtOk := target[k] + if !tgtOk { + equal = false + target[k] = v + } else { + srcMap, srcIsMap := v.(map[string]interface{}) + tgtMap, tgtIsMap := tgtV.(map[string]interface{}) + if srcIsMap && tgtIsMap { + equal = mergeJson(srcMap, tgtMap) && equal + } + } + } + return equal +} + func generateDriversJson() { drivers := make(Drivers) drivers["drivers"] = make(KV[interface{}]) diff --git a/cmd/root.go b/cmd/root.go index 86e45a356..42121aac7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,7 +27,8 @@ func Execute() { } func init() { - RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data folder") + RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data directory (relative paths are resolved against the current working directory)") + RootCmd.PersistentFlags().StringVar(&flags.ConfigPath, "config", "", "path to config.json (relative to current working directory; defaults to [data directory]/config.json, where [data directory] is set by --data)") RootCmd.PersistentFlags().BoolVar(&flags.Debug, "debug", false, "start with debug mode") RootCmd.PersistentFlags().BoolVar(&flags.NoPrefix, "no-prefix", false, "disable env prefix") RootCmd.PersistentFlags().BoolVar(&flags.Dev, "dev", false, "start with dev mode") diff --git a/cmd/server.go b/cmd/server.go index 3758009f6..26441f916 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -1,32 +1,13 @@ package cmd import ( - "context" - "errors" - "fmt" - "net" - "net/http" "os" "os/signal" - "strconv" - "sync" "syscall" "time" - "github.com/OpenListTeam/OpenList/v4/cmd/flags" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" - "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/fs" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/server" - "github.com/OpenListTeam/OpenList/v4/server/middlewares" - "github.com/OpenListTeam/sftpd-openlist" - ftpserver "github.com/fclairamb/ftpserverlib" - "github.com/gin-gonic/gin" - log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "golang.org/x/net/http2" - "golang.org/x/net/http2/h2c" ) // ServerCmd represents the server command @@ -36,142 +17,9 @@ var ServerCmd = &cobra.Command{ Long: `Start the server at the specified address the address is defined in config file`, Run: func(cmd *cobra.Command, args []string) { - Init() - if conf.Conf.DelayedStart != 0 { - utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart) - time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second) - } - bootstrap.InitOfflineDownloadTools() - bootstrap.LoadStorages() - bootstrap.InitTaskManager() - if !flags.Debug && !flags.Dev { - gin.SetMode(gin.ReleaseMode) - } - r := gin.New() - - // gin log - if conf.Conf.Log.Filter.Enable { - r.Use(middlewares.FilteredLogger()) - } else { - r.Use(gin.LoggerWithWriter(log.StandardLogger().Out)) - } - r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out)) - - server.Init(r) - var httpHandler http.Handler = r - if conf.Conf.Scheme.EnableH2c { - httpHandler = h2c.NewHandler(r, &http2.Server{}) - } - var httpSrv, httpsSrv, unixSrv *http.Server - if conf.Conf.Scheme.HttpPort != -1 { - httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort) - fmt.Printf("start HTTP server @ %s\n", httpBase) - utils.Log.Infof("start HTTP server @ %s", httpBase) - httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler} - go func() { - err := httpSrv.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - utils.Log.Fatalf("failed to start http: %s", err.Error()) - } - }() - } - if conf.Conf.Scheme.HttpsPort != -1 { - httpsBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpsPort) - fmt.Printf("start HTTPS server @ %s\n", httpsBase) - utils.Log.Infof("start HTTPS server @ %s", httpsBase) - httpsSrv = &http.Server{Addr: httpsBase, Handler: r} - go func() { - err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) - if err != nil && !errors.Is(err, http.ErrServerClosed) { - utils.Log.Fatalf("failed to start https: %s", err.Error()) - } - }() - } - if conf.Conf.Scheme.UnixFile != "" { - fmt.Printf("start unix server @ %s\n", conf.Conf.Scheme.UnixFile) - utils.Log.Infof("start unix server @ %s", conf.Conf.Scheme.UnixFile) - unixSrv = &http.Server{Handler: httpHandler} - go func() { - listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile) - if err != nil { - utils.Log.Fatalf("failed to listen unix: %+v", err) - } - // set socket file permission - mode, err := strconv.ParseUint(conf.Conf.Scheme.UnixFilePerm, 8, 32) - if err != nil { - utils.Log.Errorf("failed to parse socket file permission: %+v", err) - } else { - err = os.Chmod(conf.Conf.Scheme.UnixFile, os.FileMode(mode)) - if err != nil { - utils.Log.Errorf("failed to chmod socket file: %+v", err) - } - } - err = unixSrv.Serve(listener) - if err != nil && !errors.Is(err, http.ErrServerClosed) { - utils.Log.Fatalf("failed to start unix: %s", err.Error()) - } - }() - } - if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { - s3r := gin.New() - s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) - server.InitS3(s3r) - s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) - fmt.Printf("start S3 server @ %s\n", s3Base) - utils.Log.Infof("start S3 server @ %s", s3Base) - go func() { - var err error - if conf.Conf.S3.SSL { - httpsSrv = &http.Server{Addr: s3Base, Handler: s3r} - err = httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) - } - if !conf.Conf.S3.SSL { - httpSrv = &http.Server{Addr: s3Base, Handler: s3r} - err = httpSrv.ListenAndServe() - } - if err != nil && !errors.Is(err, http.ErrServerClosed) { - utils.Log.Fatalf("failed to start s3 server: %s", err.Error()) - } - }() - } - var ftpDriver *server.FtpMainDriver - var ftpServer *ftpserver.FtpServer - if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable { - var err error - ftpDriver, err = server.NewMainDriver() - if err != nil { - utils.Log.Fatalf("failed to start ftp driver: %s", err.Error()) - } else { - fmt.Printf("start ftp server on %s\n", conf.Conf.FTP.Listen) - utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen) - go func() { - ftpServer = ftpserver.NewFtpServer(ftpDriver) - err = ftpServer.ListenAndServe() - if err != nil { - utils.Log.Fatalf("problem ftp server listening: %s", err.Error()) - } - }() - } - } - var sftpDriver *server.SftpDriver - var sftpServer *sftpd.SftpServer - if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable { - var err error - sftpDriver, err = server.NewSftpDriver() - if err != nil { - utils.Log.Fatalf("failed to start sftp driver: %s", err.Error()) - } else { - fmt.Printf("start sftp server on %s", conf.Conf.SFTP.Listen) - utils.Log.Infof("start sftp server on %s", conf.Conf.SFTP.Listen) - go func() { - sftpServer = sftpd.NewSftpServer(sftpDriver) - err = sftpServer.RunServer() - if err != nil { - utils.Log.Fatalf("problem sftp server listening: %s", err.Error()) - } - }() - } - } + bootstrap.Init() + defer bootstrap.Release() + bootstrap.Start() // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) @@ -180,60 +28,7 @@ the address is defined in config file`, // kill -9 is syscall. SIGKILL but can"t be catch, so don't need add it signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) <-quit - utils.Log.Println("Shutdown server...") - fs.ArchiveContentUploadTaskManager.RemoveAll() - Release() - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - var wg sync.WaitGroup - if conf.Conf.Scheme.HttpPort != -1 { - wg.Add(1) - go func() { - defer wg.Done() - if err := httpSrv.Shutdown(ctx); err != nil { - utils.Log.Fatal("HTTP server shutdown err: ", err) - } - }() - } - if conf.Conf.Scheme.HttpsPort != -1 { - wg.Add(1) - go func() { - defer wg.Done() - if err := httpsSrv.Shutdown(ctx); err != nil { - utils.Log.Fatal("HTTPS server shutdown err: ", err) - } - }() - } - if conf.Conf.Scheme.UnixFile != "" { - wg.Add(1) - go func() { - defer wg.Done() - if err := unixSrv.Shutdown(ctx); err != nil { - utils.Log.Fatal("Unix server shutdown err: ", err) - } - }() - } - if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil && ftpDriver != nil { - wg.Add(1) - go func() { - defer wg.Done() - ftpDriver.Stop() - if err := ftpServer.Stop(); err != nil { - utils.Log.Fatal("FTP server shutdown err: ", err) - } - }() - } - if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable && sftpServer != nil && sftpDriver != nil { - wg.Add(1) - go func() { - defer wg.Done() - if err := sftpServer.Close(); err != nil { - utils.Log.Fatal("SFTP server shutdown err: ", err) - } - }() - } - wg.Wait() - utils.Log.Println("Server exit") + bootstrap.Shutdown(1 * time.Second) }, } diff --git a/cmd/storage.go b/cmd/storage.go index c744dace5..6190feb38 100644 --- a/cmd/storage.go +++ b/cmd/storage.go @@ -8,6 +8,7 @@ import ( "os" "strconv" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/charmbracelet/bubbles/table" @@ -30,8 +31,8 @@ var disableStorageCmd = &cobra.Command{ return fmt.Errorf("mount path is required") } mountPath := args[0] - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() storage, err := db.GetStorageByMountPath(mountPath) if err != nil { return fmt.Errorf("failed to query storage: %+v", err) @@ -69,8 +70,8 @@ var deleteStorageCmd = &cobra.Command{ } } - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() err = db.DeleteStorageById(uint(id)) if err != nil { return fmt.Errorf("failed to delete storage by id: %+v", err) @@ -123,8 +124,8 @@ var listStorageCmd = &cobra.Command{ Use: "list", Short: "List all storages", RunE: func(cmd *cobra.Command, args []string) error { - Init() - defer Release() + bootstrap.Init() + defer bootstrap.Release() storages, _, err := db.GetStorages(1, -1) if err != nil { return fmt.Errorf("failed to query storages: %+v", err) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 22db74e38..d4f5741d0 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -68,8 +68,7 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, err } userAgent := args.Header.Get("User-Agent") - downloadInfo, err := d. - DownloadWithUA(file.(*FileObj).PickCode, userAgent) + downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } @@ -252,8 +251,8 @@ func (d *Pan115) GetDetails(ctx context.Context) (*model.StorageDetails, error) } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: uint64(info.SpaceInfo.AllTotal.Size), - FreeSpace: uint64(info.SpaceInfo.AllRemain.Size), + TotalSpace: info.SpaceInfo.AllTotal.Size, + UsedSpace: info.SpaceInfo.AllUse.Size, }, }, nil } diff --git a/drivers/115/meta.go b/drivers/115/meta.go index 6a32715e7..02c180c30 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -15,10 +15,9 @@ type Addition struct { } var config = driver.Config{ - Name: "115 Cloud", - DefaultRoot: "0", - // OnlyProxy: true, - // NoOverwriteUpload: true, + Name: "115 Cloud", + DefaultRoot: "0", + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/115/types.go b/drivers/115/types.go index 28a8ced30..3477ffed0 100644 --- a/drivers/115/types.go +++ b/drivers/115/types.go @@ -22,6 +22,10 @@ func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + type UploadResult struct { driver.BasicResp Data struct { diff --git a/drivers/115/util.go b/drivers/115/util.go index b000436b2..7ae375b75 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "net/url" "strconv" "strings" @@ -22,11 +21,9 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/aliyun/aliyun-oss-go-sdk/oss" - cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" - crypto "github.com/SheltonZhu/115driver/pkg/crypto/m115" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/pkg/errors" ) @@ -108,60 +105,6 @@ func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } -func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { - key := crypto.GenerateKey() - result := driver115.DownloadResp{} - params, err := utils.Json.Marshal(map[string]string{"pick_code": pickCode}) - if err != nil { - return nil, err - } - - data := crypto.Encode(params, key) - - bodyReader := strings.NewReader(url.Values{"data": []string{data}}.Encode()) - reqUrl := fmt.Sprintf("%s?t=%s", driver115.AndroidApiDownloadGetUrl, driver115.Now().String()) - req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Cookie", d.Cookie) - req.Header.Set("User-Agent", ua) - - resp, err := d.client.Client.GetClient().Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if err := utils.Json.Unmarshal(body, &result); err != nil { - return nil, err - } - - if err = result.Err(string(body)); err != nil { - return nil, err - } - - b, err := crypto.Decode(string(result.EncodedData), key) - if err != nil { - return nil, err - } - - downloadInfo := struct { - Url string `json:"url"` - }{} - if err := utils.Json.Unmarshal(b, &downloadInfo); err != nil { - return nil, err - } - - info := &driver115.DownloadInfo{} - info.PickCode = pickCode - info.Header = resp.Request.Header - info.Url.Url = downloadInfo.Url - return info, nil -} - func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { userID := strconv.FormatInt(c.client.UserID, 10) userIDMd5 := md5.Sum([]byte(userID)) @@ -309,7 +252,8 @@ func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSPar // UploadByMultipart upload by mutipart blocks func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.UploadOSSParams, fileSize int64, s model.FileStreamer, - dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption) (*UploadResult, error) { + dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption, +) (*UploadResult, error) { var ( chunks []oss.FileChunk parts []oss.UploadPart diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index edab65abe..ec76a6bc8 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -53,6 +53,12 @@ func (d *Open115) Init(ctx context.Context) error { if d.Addition.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.Addition.LimitRate), 1) } + if d.PageSize <= 0 { + d.PageSize = 200 + } else if d.PageSize > 1150 { + d.PageSize = 1150 + } + return nil } @@ -69,7 +75,7 @@ func (d *Open115) Drop(ctx context.Context) error { func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var res []model.Obj - pageSize := int64(200) + pageSize := int64(d.PageSize) offset := int64(0) for { if err := d.WaitLimit(ctx); err != nil { @@ -131,23 +137,6 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) }, nil } -func (d *Open115) GetObjInfo(ctx context.Context, path string) (model.Obj, error) { - if err := d.WaitLimit(ctx); err != nil { - return nil, err - } - resp, err := d.client.GetFolderInfoByPath(ctx, path) - if err != nil { - return nil, err - } - return &Obj{ - Fid: resp.FileID, - Fn: resp.FileName, - Fc: resp.FileCategory, - Sha1: resp.Sha1, - Pc: resp.PickCode, - }, nil -} - func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { return nil, err @@ -187,7 +176,7 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ FileID: srcObj.GetID(), - FileNma: newName, + FileName: newName, }) if err != nil { return nil, err @@ -342,18 +331,18 @@ func (d *Open115) GetDetails(ctx context.Context) (*model.StorageDetails, error) if err != nil { return nil, err } - total, err := userInfo.RtSpaceInfo.AllTotal.Size.Int64() + total, err := ParseInt64(userInfo.RtSpaceInfo.AllTotal.Size) if err != nil { return nil, err } - free, err := userInfo.RtSpaceInfo.AllRemain.Size.Int64() + used, err := ParseInt64(userInfo.RtSpaceInfo.AllUse.Size) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: uint64(total), - FreeSpace: uint64(free), + TotalSpace: total, + UsedSpace: used, }, }, nil } diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go index c24b9993a..ed908e2e6 100644 --- a/drivers/115_open/meta.go +++ b/drivers/115_open/meta.go @@ -12,13 +12,15 @@ type Addition struct { OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` + PageSize int64 `json:"page_size" type:"number" default:"200" help:"list api per page size of 115open driver"` AccessToken string `json:"access_token" required:"true"` RefreshToken string `json:"refresh_token" required:"true"` } var config = driver.Config{ - Name: "115 Open", - DefaultRoot: "0", + Name: "115 Open", + DefaultRoot: "0", + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index 3c847e057..3575678c2 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -107,16 +107,16 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, if err != nil { return err } - rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd) err = retry.Do(func() error { rd.Seek(0, io.SeekStart) - part, err := bucket.UploadPart(imur, rateLimitedRd, partSize, int(i)) + part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), partSize, int(i)) if err != nil { return err } parts[i-1] = part return nil }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) diff --git a/drivers/115_open/util.go b/drivers/115_open/util.go index 1d7ab0fff..a7a333c91 100644 --- a/drivers/115_open/util.go +++ b/drivers/115_open/util.go @@ -1,3 +1,15 @@ package _115_open -// do others that not defined in Driver interface \ No newline at end of file +import "encoding/json" + +func ParseInt64(v json.Number) (int64, error) { + i, err := v.Int64() + if err == nil { + return i, nil + } + f, e1 := v.Float64() + if e1 == nil { + return int64(f), nil + } + return int64(0), err +} diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index 00fa623e6..fe8b7733a 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -3,6 +3,7 @@ package _115_share import ( "context" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -49,9 +50,16 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr if err := d.WaitLimit(ctx); err != nil { return nil, err } - + var ua string + // TODO: will use user agent from header + // if args.Header != nil { + // ua = args.Header.Get("User-Agent") + // } + if ua == "" { + ua = base.UserAgentNT + } files := make([]driver115.ShareFile, 0) - fileResp, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) + fileResp, err := d.client.GetShareSnapWithUA(ua, d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) if err != nil { return nil, err } @@ -77,7 +85,14 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if err := d.WaitLimit(ctx); err != nil { return nil, err } - downloadInfo, err := d.client.DownloadByShareCode(d.ShareCode, d.ReceiveCode, file.GetID()) + var ua string + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + downloadInfo, err := d.client.DownloadByShareCodeWithUA(ua, d.ShareCode, d.ReceiveCode, file.GetID()) if err != nil { return nil, err } diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go index 082d9d462..f9575d493 100644 --- a/drivers/115_share/utils.go +++ b/drivers/115_share/utils.go @@ -5,6 +5,7 @@ import ( "strconv" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" driver115 "github.com/SheltonZhu/115driver/pkg/driver" @@ -20,6 +21,7 @@ type FileObj struct { FileName string isDir bool FileID string + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -54,6 +56,10 @@ func (f *FileObj) GetPath() string { return "" } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + func transFunc(sf driver115.ShareFile) (model.Obj, error) { timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) if err != nil { @@ -74,15 +80,14 @@ func transFunc(sf driver115.ShareFile) (model.Obj, error) { FileName: string(sf.FileName), isDir: isDir, FileID: fileID, + ThumbURL: sf.ThumbURL, }, nil } -var UserAgent = driver115.UA115Browser - func (d *Pan115Share) login() error { var err error opts := []driver115.Option{ - driver115.UA(UserAgent), + driver115.UA(base.UserAgentNT), } d.client = driver115.New(opts...) if _, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, ""); err != nil { diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 135ebc51a..abf0724db 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -41,7 +41,9 @@ func (d *Pan123) GetAddition() driver.Additional { } func (d *Pan123) Init(ctx context.Context) error { - _, err := d.Request(UserInfo, http.MethodGet, nil, nil) + _, err := d.Request(UserInfo, http.MethodGet, func(req *resty.Request) { + req.SetHeader("platform", "web") + }, nil) return err } @@ -258,11 +260,10 @@ func (d *Pan123) GetDetails(ctx context.Context) (*model.StorageDetails, error) if err != nil { return nil, err } - total := userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: total, - FreeSpace: total - userInfo.Data.SpaceUsed, + TotalSpace: userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp, + UsedSpace: userInfo.Data.SpaceUsed, }, }, nil } diff --git a/drivers/123/meta.go b/drivers/123/meta.go index 613ba3ad6..5b19ae82e 100644 --- a/drivers/123/meta.go +++ b/drivers/123/meta.go @@ -12,13 +12,15 @@ type Addition struct { //OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"` //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` AccessToken string - UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"` + UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"` + Platform string `json:"platform" type:"string" default:"web" help:"the platform header value, sent with API requests"` } var config = driver.Config{ Name: "123Pan", DefaultRoot: "0", LocalSort: true, + PreferProxy: true, } func init() { @@ -27,6 +29,7 @@ func init() { return &Pan123{ Addition: Addition{ UploadThread: 3, + Platform: "web", }, } }) diff --git a/drivers/123/types.go b/drivers/123/types.go index dc77b77f1..dc76c26fa 100644 --- a/drivers/123/types.go +++ b/drivers/123/types.go @@ -127,9 +127,54 @@ type UserInfoResp struct { Data struct { Uid int64 `json:"UID"` Nickname string `json:"Nickname"` - SpaceUsed uint64 `json:"SpaceUsed"` - SpacePermanent uint64 `json:"SpacePermanent"` - SpaceTemp uint64 `json:"SpaceTemp"` + SpaceUsed int64 `json:"SpaceUsed"` + SpacePermanent int64 `json:"SpacePermanent"` + SpaceTemp int64 `json:"SpaceTemp"` FileCount int `json:"FileCount"` } `json:"data"` } + +type offlineResolveResp struct { + Data struct { + List []struct { + Result int `json:"result"` + ID int64 `json:"id"` + ErrCode int `json:"err_code"` + ErrMsg string `json:"err_msg"` + Files []struct { + ID int64 `json:"id"` + } `json:"files"` + } `json:"list"` + } `json:"data"` +} + +type offlineSubmitResp struct { + Data struct { + TaskList []struct { + TaskID int64 `json:"task_id"` + Result int `json:"result"` + } `json:"task_list"` + } `json:"data"` +} + +type offlineTaskListResp struct { + Data struct { + HasRun bool `json:"has_run"` + List []offlineTask `json:"list"` + Total int `json:"total"` + } `json:"data"` +} + +type offlineTask struct { + TaskID int64 `json:"task_id"` + Name string `json:"name"` + Status int `json:"status"` + Size int64 `json:"size"` + ThirdTask string `json:"third_task_id"` + Downloaded int64 `json:"downloaded"` + Progress float64 `json:"progress"` + UploadIDR int64 `json:"upload_idr"` + UploadName string `json:"upload_name"` + Type string `json:"type"` + Speed int64 `json:"speed"` +} diff --git a/drivers/123/upload.go b/drivers/123/upload.go index a4ae988d9..cc149cfe3 100644 --- a/drivers/123/upload.go +++ b/drivers/123/upload.go @@ -125,27 +125,18 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi curSize = lastChunkSize } var reader io.ReadSeeker - var rateLimitedRd io.Reader threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { - if reader == nil { - var err error - reader, err = ss.GetSectionReader(offset, curSize) - if err != nil { - return err - } - rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader) - } - return nil + Before: func(ctx context.Context) (err error) { + reader, err = ss.GetSectionReader(offset, curSize) + return }, - Do: func(ctx context.Context) error { + Do: func(ctx context.Context) (err error) { reader.Seek(0, io.SeekStart) uploadUrl := s3PreSignedUrls.Data.PreSignedUrls[strconv.Itoa(cur)] if uploadUrl == "" { return fmt.Errorf("upload url is empty, s3PreSignedUrls: %+v", s3PreSignedUrls) } - reader.Seek(0, io.SeekStart) - req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, rateLimitedRd) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadUrl, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } @@ -157,7 +148,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi } defer res.Body.Close() if res.StatusCode == http.StatusForbidden { - singleflight.AnyGroup.Do(fmt.Sprintf("Pan123.newUpload_%p", threadG), func() (any, error) { + _, err, _ = singleflight.AnyGroup.Do(fmt.Sprintf("Pan123.newUpload_%p", threadG), func() (any, error) { newS3PreSignedUrls, err := getS3UploadUrl(ctx, upReq, cur, end) if err != nil { return nil, err @@ -177,7 +168,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi } return fmt.Errorf("upload s3 chunk %d failed, status code: %d, body: %s", cur, res.StatusCode, body) } - progress := 10.0 + 85.0*float64(threadG.Success())/float64(chunkCount) + progress := 100 * float64(threadG.Success()+1) / float64(chunkCount+1) up(progress) return nil }, diff --git a/drivers/123/util.go b/drivers/123/util.go index 9afe46e36..1a2ff0173 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -16,6 +16,7 @@ import ( "encoding/json" "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" @@ -47,9 +48,16 @@ const ( S3Auth = MainApi + "/file/s3_upload_object/auth" UploadCompleteV2 = MainApi + "/file/upload_complete/v2" S3Complete = MainApi + "/file/s3_complete_multipart_upload" + + OfflineResolve = MainApi + "/v2/offline_download/task/resolve" + OfflineSubmit = MainApi + "/v2/offline_download/task/submit" + OfflineTaskList = MainApi + "/offline_download/task/list" + OfflineTaskDelete = MainApi + "/offline_download/task/delete" // AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) +var ErrOfflineTaskNotFound = errors.New("offline task not found") + func signPath(path string, os string, version string) (k string, v string) { table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'} random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64())) @@ -242,10 +250,10 @@ do: "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, - "user-agent": "123pan/v2.4.7(Android_10.0;Oppo)", - "platform": "android", - "app-version": "69", - "x-app-version": "2.4.7", + "platform": d.Platform, + "user-agent": "123pan/v2.4.7(Android_10.0;Oppo)", + "app-version": "69", + "x-app-version": "2.4.7", //"user-agent": base.UserAgent, }) if callback != nil { @@ -306,6 +314,115 @@ do: return body, nil } +func (d *Pan123) OfflineDownload(ctx context.Context, uri string, dstDir model.Obj) (int64, error) { + var resolveResp offlineResolveResp + _, err := d.Request(OfflineResolve, http.MethodPost, func(req *resty.Request) { + req.SetContext(ctx).SetBody(base.Json{ + "urls": uri, + }) + }, &resolveResp) + if err != nil { + return 0, err + } + if len(resolveResp.Data.List) == 0 { + return 0, fmt.Errorf("offline resolve failed: empty response") + } + if resolveResp.Data.List[0].Result != 0 { + msg := resolveResp.Data.List[0].ErrMsg + if msg == "" { + msg = "offline resolve failed" + } + return 0, fmt.Errorf("%s", msg) + } + resourceID := resolveResp.Data.List[0].ID + if resourceID == 0 { + return 0, fmt.Errorf("offline resolve failed: empty resource id") + } + selectFileIDs := make([]int64, 0, len(resolveResp.Data.List[0].Files)) + for _, f := range resolveResp.Data.List[0].Files { + if f.ID > 0 { + selectFileIDs = append(selectFileIDs, f.ID) + } + } + if len(selectFileIDs) == 0 { + return 0, fmt.Errorf("offline resolve failed: empty file list") + } + uploadDir, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid destination dir id: %s", dstDir.GetID()) + } + + var submitResp offlineSubmitResp + _, err = d.Request(OfflineSubmit, http.MethodPost, func(req *resty.Request) { + req.SetContext(ctx).SetBody(base.Json{ + "resource_list": []base.Json{ + { + "resource_id": resourceID, + "select_file_id": selectFileIDs, + }, + }, + "upload_dir": uploadDir, + }) + }, &submitResp) + if err != nil { + return 0, err + } + if len(submitResp.Data.TaskList) == 0 { + return 0, fmt.Errorf("offline submit failed: empty task list") + } + if submitResp.Data.TaskList[0].Result != 0 { + return 0, fmt.Errorf("offline submit failed") + } + if submitResp.Data.TaskList[0].TaskID == 0 { + return 0, fmt.Errorf("offline submit failed: empty task id") + } + return submitResp.Data.TaskList[0].TaskID, nil +} + +func (d *Pan123) GetOfflineTask(ctx context.Context, taskID int64) (*offlineTask, error) { + if taskID == 0 { + return nil, fmt.Errorf("invalid task id") + } + page := 1 + pageSize := 100 + statusArr := []int{0, 1, 2, 3} + for { + var listResp offlineTaskListResp + _, err := d.Request(OfflineTaskList, http.MethodPost, func(req *resty.Request) { + req.SetContext(ctx).SetBody(base.Json{ + "current_page": page, + "page_size": pageSize, + "status_arr": statusArr, + }) + }, &listResp) + if err != nil { + return nil, err + } + for i := range listResp.Data.List { + if listResp.Data.List[i].TaskID == taskID { + return &listResp.Data.List[i], nil + } + } + if len(listResp.Data.List) == 0 || page*pageSize >= listResp.Data.Total { + break + } + page++ + } + return nil, ErrOfflineTaskNotFound +} + +func (d *Pan123) DeleteOfflineTasks(ctx context.Context, taskIDs []int64) error { + if len(taskIDs) == 0 { + return nil + } + _, err := d.Request(OfflineTaskDelete, http.MethodPost, func(req *resty.Request) { + req.SetContext(ctx).SetBody(base.Json{ + "task_ids": taskIDs, + }) + }, nil) + return err +} + func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([]File, error) { page := 1 total := 0 diff --git a/drivers/123_link/driver.go b/drivers/123_link/driver.go index af8cc2164..dec09816b 100644 --- a/drivers/123_link/driver.go +++ b/drivers/123_link/driver.go @@ -39,6 +39,10 @@ func (d *Pan123Link) Drop(ctx context.Context) error { return nil } +func (Addition) GetRootPath() string { + return "/" +} + func (d *Pan123Link) Get(ctx context.Context, path string) (model.Obj, error) { node := GetNodeFromRootByPath(d.root, path) return nodeToObj(node, path) diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 62b4b95c5..e20140277 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -18,6 +18,7 @@ type Open123 struct { model.Storage Addition UID uint64 + tm *tokenManager } func (d *Open123) Config() driver.Config { @@ -33,6 +34,24 @@ func (d *Open123) Init(ctx context.Context) error { d.UploadThread = 3 } + if d.RefreshToken != "" { + // refresh token 直接主动刷新 + d.AccessToken = "" + d.tm = &tokenManager{} + } else { + // 避免个人 token 刷新产生的多个登录，被动刷新 + // 默认过期时间90天，jwt exp 不可靠 + d.tm = &tokenManager{ + // accessToken: d.AccessToken, + expiredAt: time.Now().Add(90 * 24 * time.Hour), + } + } + + _, err := d.getAccessToken(false) + if err != nil { + return fmt.Errorf("init get access token error: %w", err) + } + return nil } @@ -162,6 +181,22 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return nil, fmt.Errorf("parse parentFileID error: %v", err) } + + // 尝试 SHA1 秒传 + sha1Hash := file.GetHash().GetHash(utils.SHA1) + if len(sha1Hash) == utils.SHA1.Width { + resp, err := d.sha1Reuse(parentFileId, file.GetName(), sha1Hash, file.GetSize(), 2) + if err == nil && resp.Data.Reuse { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: resp.Data.FileID, + Type: 2, + SHA1: sha1Hash, + }, nil + } + } + // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) if len(etag) < utils.MD5.Width { @@ -219,16 +254,22 @@ func (d *Open123) GetDetails(ctx context.Context) (*model.StorageDetails, error) if err != nil { return nil, err } - total := userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp - free := total - userInfo.Data.SpaceUsed return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: total, - FreeSpace: free, + TotalSpace: userInfo.Data.SpacePermanent + userInfo.Data.SpaceTemp, + UsedSpace: userInfo.Data.SpaceUsed, }, }, nil } +func (d *Open123) OfflineDownload(ctx context.Context, url string, dir model.Obj, callback string) (int, error) { + return d.createOfflineDownloadTask(ctx, url, dir.GetID(), callback) +} + +func (d *Open123) OfflineDownloadProcess(ctx context.Context, taskID int) (float64, int, error) { + return d.queryOfflineDownloadStatus(ctx, taskID) +} + var ( _ driver.Driver = (*Open123)(nil) _ driver.PutResult = (*Open123)(nil) diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go index db4ccc187..5481ef356 100644 --- a/drivers/123_open/meta.go +++ b/drivers/123_open/meta.go @@ -13,7 +13,7 @@ type Addition struct { ClientID string `json:"ClientID" required:"false"` ClientSecret string `json:"ClientSecret" required:"false"` - // 直接写入AccessToken + // 直接写入AccessToken, AccessToken有过期时间，不建议直接填写 AccessToken string `json:"AccessToken" required:"false"` // 用户名+密码方式登录的AccessToken可以兼容 @@ -35,6 +35,7 @@ var config = driver.Config{ Name: "123 Open", DefaultRoot: "0", LocalSort: true, + PreferProxy: true, } func init() { diff --git a/drivers/123_open/token.go b/drivers/123_open/token.go new file mode 100644 index 000000000..3c5c416c9 --- /dev/null +++ b/drivers/123_open/token.go @@ -0,0 +1,115 @@ +package _123_open + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +var ( + AccessToken = "https://open-api.123pan.com/api/v1/access_token" + RefreshToken = "https://open-api.123pan.com/api/v1/oauth2/access_token" +) + +type tokenManager struct { + // accessToken string + expiredAt time.Time + mu sync.Mutex + blockRefresh bool +} + +func (d *Open123) getAccessToken(forceRefresh bool) (string, error) { + tm := d.tm + tm.mu.Lock() + defer tm.mu.Unlock() + if tm.blockRefresh { + return "", errors.New("Authentication expired") + } + if !forceRefresh && d.AccessToken != "" && time.Now().Before(tm.expiredAt.Add(-5*time.Minute)) { + return d.AccessToken, nil + } + if err := d.flushAccessToken(); err != nil { + // token expired and failed to refresh, block further refresh attempts + tm.blockRefresh = true + return "", err + } + return d.AccessToken, nil +} + +func (d *Open123) flushAccessToken() error { + // directly send request to avoid deadlock + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "authorization": "Bearer " + d.AccessToken, + "platform": "open_platform", + "Content-Type": "application/json", + }) + + if d.ClientID != "" { + if d.RefreshToken != "" { + var resp RefreshTokenResp + req.SetQueryParam("client_id", d.ClientID) + if d.ClientSecret != "" { + req.SetQueryParam("client_secret", d.ClientSecret) + } + req.SetQueryParam("grant_type", "refresh_token") + req.SetQueryParam("refresh_token", d.RefreshToken) + req.SetResult(&resp) + res, err := req.Execute(http.MethodPost, RefreshToken) + if err != nil { + return err + } + body := res.Body() + var baseResp BaseResp + if err = json.Unmarshal(body, &baseResp); err != nil { + return err + } + if baseResp.Code != 0 { + return fmt.Errorf("get access token failed: %s", baseResp.Message) + } + + d.AccessToken = resp.AccessToken + // add token expire time + d.tm.expiredAt = time.Now().Add(time.Duration(resp.ExpiresIn) * time.Second) + d.RefreshToken = resp.RefreshToken + op.MustSaveDriverStorage(d) + d.tm.blockRefresh = false + return nil + } else if d.ClientSecret != "" { + var resp AccessTokenResp + req.SetBody(base.Json{ + "clientID": d.ClientID, + "clientSecret": d.ClientSecret, + }) + req.SetResult(&resp) + res, err := req.Execute(http.MethodPost, AccessToken) + if err != nil { + return err + } + body := res.Body() + var baseResp BaseResp + if err = json.Unmarshal(body, &baseResp); err != nil { + return err + } + if baseResp.Code != 0 { + return fmt.Errorf("get access token failed: %s", baseResp.Message) + } + d.AccessToken = resp.Data.AccessToken + // parse token expire time + d.tm.expiredAt, err = time.Parse(time.RFC3339, resp.Data.ExpiredAt) + if err != nil { + return fmt.Errorf("parse expire time failed: %w", err) + } + op.MustSaveDriverStorage(d) + d.tm.blockRefresh = false + return nil + } + } + return errors.New("no valid authentication method available") +} diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go index 70257d84f..7d586c8b0 100644 --- a/drivers/123_open/types.go +++ b/drivers/123_open/types.go @@ -19,6 +19,7 @@ func (a *ApiInfo) Require() { a.token <- struct{}{} } } + func (a *ApiInfo) Release() { if a.qps > 0 { time.AfterFunc(time.Second, func() { @@ -26,13 +27,16 @@ func (a *ApiInfo) Release() { }) } } + func (a *ApiInfo) SetQPS(qps int) { a.qps = qps a.token = make(chan struct{}, qps) } + func (a *ApiInfo) NowLen() int { return len(a.token) } + func InitApiInfo(url string, qps int) *ApiInfo { return &ApiInfo{ url: url, @@ -54,9 +58,13 @@ type File struct { Category int `json:"category"` Status int `json:"status"` Trashed int `json:"trashed"` + SHA1 string } func (f File) GetHash() utils.HashInfo { + if len(f.SHA1) == utils.SHA1.Width && len(f.Etag) != utils.MD5.Width { + return utils.NewHashInfo(utils.SHA1, f.SHA1) + } return utils.NewHashInfo(utils.MD5, f.Etag) } @@ -133,9 +141,9 @@ type UserInfoResp struct { // HeadImage string `json:"headImage"` // Passport string `json:"passport"` // Mail string `json:"mail"` - SpaceUsed uint64 `json:"spaceUsed"` - SpacePermanent uint64 `json:"spacePermanent"` - SpaceTemp uint64 `json:"spaceTemp"` + SpaceUsed int64 `json:"spaceUsed"` + SpacePermanent int64 `json:"spacePermanent"` + SpaceTemp int64 `json:"spaceTemp"` // SpaceTempExpr int64 `json:"spaceTempExpr"` // Vip bool `json:"vip"` // DirectTraffic int64 `json:"directTraffic"` @@ -185,3 +193,26 @@ type UploadCompleteResp struct { FileID int64 `json:"fileID"` } `json:"data"` } + +type SHA1ReuseResp struct { + BaseResp + Data struct { + FileID int64 `json:"fileID"` + Reuse bool `json:"reuse"` + } `json:"data"` +} + +type OfflineDownloadResp struct { + BaseResp + Data struct { + TaskID int `json:"taskID"` + } `json:"data"` +} + +type OfflineDownloadProcessResp struct { + BaseResp + Data struct { + Process float64 `json:"process"` + Status int `json:"status"` + } `json:"data"` +} diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go index 73395fcfd..0e03684e9 100644 --- a/drivers/123_open/upload.go +++ b/drivers/123_open/upload.go @@ -73,25 +73,20 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes // 表单 b := bytes.NewBuffer(make([]byte, 0, 2048)) threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { - if reader == nil { - var err error - // 每个分片一个reader - reader, err = ss.GetSectionReader(offset, size) - if err != nil { - return err - } - // 计算当前分片的MD5 + Before: func(ctx context.Context) (err error) { + reader, err = ss.GetSectionReader(offset, size) + return + }, + Do: func(ctx context.Context) (err error) { + reader.Seek(0, io.SeekStart) + if sliceMD5 == "" { + // 把耗时的计算放在这里，避免阻塞其他协程 sliceMD5, err = utils.HashReader(utils.MD5, reader) if err != nil { return err } + reader.Seek(0, io.SeekStart) } - return nil - }, - Do: func(ctx context.Context) error { - // 重置分片reader位置，因为HashReader、上一次失败已经读取到分片EOF - reader.Seek(0, io.SeekStart) b.Reset() w := multipart.NewWriter(b) @@ -121,6 +116,10 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes head := bytes.NewReader(b.Bytes()[:headSize]) tail := bytes.NewReader(b.Bytes()[headSize:]) rateLimitedRd = driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail)) + token, err := d.getAccessToken(false) + if err != nil { + return err + } // 创建请求并设置header req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadDomain+"/upload/v2/file/slice", rateLimitedRd) if err != nil { @@ -128,7 +127,7 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes } // 设置请求头 - req.Header.Add("Authorization", "Bearer "+d.AccessToken) + req.Header.Add("Authorization", "Bearer "+token) req.Header.Add("Content-Type", w.FormDataContentType()) req.Header.Add("Platform", "open_platform") @@ -140,12 +139,13 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes if res.StatusCode != 200 { return fmt.Errorf("slice %d upload failed, status code: %d", partNumber, res.StatusCode) } - var resp BaseResp - respBody, err := io.ReadAll(res.Body) + b.Reset() + _, err = b.ReadFrom(res.Body) if err != nil { return err } - err = json.Unmarshal(respBody, &resp) + var resp BaseResp + err = json.Unmarshal(b.Bytes(), &resp) if err != nil { return err } @@ -153,7 +153,7 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes return fmt.Errorf("slice %d upload failed: %s", partNumber, resp.Message) } - progress := 10.0 + 85.0*float64(threadG.Success())/float64(uploadNums) + progress := 100 * float64(threadG.Success()+1) / float64(uploadNums+1) up(progress) return nil }, @@ -183,3 +183,21 @@ func (d *Open123) complete(preuploadID string) (*UploadCompleteResp, error) { } return &resp, nil } + +// SHA1 秒传 +func (d *Open123) sha1Reuse(parentFileID int64, filename string, sha1Hash string, size int64, duplicate int) (*SHA1ReuseResp, error) { + var resp SHA1ReuseResp + _, err := d.Request(UploadSHA1Reuse, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "parentFileID": parentFileID, + "filename": filename, + "sha1": strings.ToLower(sha1Hash), + "size": size, + "duplicate": duplicate, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} diff --git a/drivers/123_open/util.go b/drivers/123_open/util.go index 17da9a66f..1b6eea2da 100644 --- a/drivers/123_open/util.go +++ b/drivers/123_open/util.go @@ -13,7 +13,6 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/go-resty/resty/v2" "github.com/google/uuid" log "github.com/sirupsen/logrus" @@ -22,26 +21,31 @@ import ( var ( // 不同情况下获取的AccessTokenQPS限制不同 如下模块化易于拓展 Api = "https://open-api.123pan.com" - AccessToken = InitApiInfo(Api+"/api/v1/access_token", 1) - RefreshToken = InitApiInfo(Api+"/api/v1/oauth2/access_token", 1) - UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) - FileList = InitApiInfo(Api+"/api/v2/file/list", 3) - DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 5) - DirectLink = InitApiInfo(Api+"/api/v1/direct-link/url", 5) - Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) - Move = InitApiInfo(Api+"/api/v1/file/move", 1) - Rename = InitApiInfo(Api+"/api/v1/file/name", 1) - Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) - UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) - UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) + UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) + FileList = InitApiInfo(Api+"/api/v2/file/list", 3) + DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 5) + DirectLink = InitApiInfo(Api+"/api/v1/direct-link/url", 5) + Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) + Move = InitApiInfo(Api+"/api/v1/file/move", 1) + Rename = InitApiInfo(Api+"/api/v1/file/name", 1) + Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) + UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) + UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) + UploadSHA1Reuse = InitApiInfo(Api+"/upload/v2/file/sha1_reuse", 2) + + OfflineDownload = InitApiInfo(Api+"/api/v1/offline/download", 1) + OfflineDownloadProcess = InitApiInfo(Api+"/api/v1/offline/download/process", 5) ) func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - retryToken := true for { + token, err := d.getAccessToken(false) + if err != nil { + return nil, err + } req := base.RestyClient.R() req.SetHeaders(map[string]string{ - "authorization": "Bearer " + d.AccessToken, + "authorization": "Bearer " + token, "platform": "open_platform", "Content-Type": "application/json", }) @@ -71,9 +75,9 @@ func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCall if baseResp.Code == 0 { return body, nil - } else if baseResp.Code == 401 && retryToken { - retryToken = false - if err := d.flushAccessToken(); err != nil { + } else if baseResp.Code == 401 { + // 强制刷新Token, 有小概率会 race condition 导致多次刷新Token，但不影响正确运行 + if _, err := d.getAccessToken(true); err != nil { return nil, err } } else if baseResp.Code == 429 { @@ -85,42 +89,6 @@ func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCall } } -func (d *Open123) flushAccessToken() error { - if d.ClientID != "" { - if d.RefreshToken != "" { - var resp RefreshTokenResp - _, err := d.Request(RefreshToken, http.MethodPost, func(req *resty.Request) { - req.SetQueryParam("client_id", d.ClientID) - if d.ClientSecret != "" { - req.SetQueryParam("client_secret", d.ClientSecret) - } - req.SetQueryParam("grant_type", "refresh_token") - req.SetQueryParam("refresh_token", d.RefreshToken) - }, &resp) - if err != nil { - return err - } - d.AccessToken = resp.AccessToken - d.RefreshToken = resp.RefreshToken - op.MustSaveDriverStorage(d) - } else if d.ClientSecret != "" { - var resp AccessTokenResp - _, err := d.Request(AccessToken, http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "clientID": d.ClientID, - "clientSecret": d.ClientSecret, - }) - }, &resp) - if err != nil { - return err - } - d.AccessToken = resp.Data.AccessToken - op.MustSaveDriverStorage(d) - } - } - return nil -} - func (d *Open123) SignURL(originURL, privateKey string, uid uint64, validDuration time.Duration) (newURL string, err error) { // 生成Unix时间戳 ts := time.Now().Add(validDuration).Unix() @@ -277,3 +245,34 @@ func (d *Open123) trash(fileId int64) error { return nil } + +func (d *Open123) createOfflineDownloadTask(ctx context.Context, url string, dirID, callback string) (taskID int, err error) { + body := base.Json{ + "url": url, + "dirID": dirID, + } + if len(callback) > 0 { + body["callBackUrl"] = callback + } + var resp OfflineDownloadResp + _, err = d.Request(OfflineDownload, http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &resp) + if err != nil { + return 0, err + } + return resp.Data.TaskID, nil +} + +func (d *Open123) queryOfflineDownloadStatus(ctx context.Context, taskID int) (process float64, status int, err error) { + var resp OfflineDownloadProcessResp + _, err = d.Request(OfflineDownloadProcess, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "taskID": strconv.Itoa(taskID), + }) + }, &resp) + if err != nil { + return .0, 0, err + } + return resp.Data.Process, resp.Data.Status, nil +} diff --git a/drivers/123_share/meta.go b/drivers/123_share/meta.go index 12c620d7a..e40965fe9 100644 --- a/drivers/123_share/meta.go +++ b/drivers/123_share/meta.go @@ -19,6 +19,7 @@ var config = driver.Config{ LocalSort: true, NoUpload: true, DefaultRoot: "0", + PreferProxy: true, } func init() { diff --git a/drivers/139/driver.go b/drivers/139/driver.go index fbbc7f8e0..4e3ea3e92 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -14,6 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -28,6 +29,7 @@ type Yun139 struct { Account string ref *Yun139 PersonalCloudHost string + RootPath string } func (d *Yun139) Config() driver.Config { @@ -41,7 +43,16 @@ func (d *Yun139) GetAddition() driver.Additional { func (d *Yun139) Init(ctx context.Context) error { if d.ref == nil { if len(d.Authorization) == 0 { - return fmt.Errorf("authorization is empty") + if d.Username != "" && d.Password != "" { + log.Infof("139yun: authorization is empty, trying to login with password.") + newAuth, err := d.loginWithPassword() + log.Debugf("newAuth: Ok: %s", newAuth) + if err != nil { + return fmt.Errorf("login with password failed: %w", err) + } + } else { + return fmt.Errorf("authorization is empty and username/password is not provided") + } } err := d.refreshToken() if err != nil { @@ -92,7 +103,22 @@ func (d *Yun139) Init(ctx context.Context) error { if len(d.Addition.RootFolderID) == 0 { d.RootFolderID = d.CloudID } + _, err := d.groupGetFiles(d.RootFolderID) + if err != nil { + return err + } case MetaFamily: + if len(d.Addition.RootFolderID) == 0 { + // Attempt to obtain data.path as the root via a query and persist it. + if root, err := d.getFamilyRootPath(d.CloudID); err == nil && root != "" { + d.RootFolderID = root + op.MustSaveDriverStorage(d) + } + } + _, err := d.familyGetFiles(d.RootFolderID) + if err != nil { + return err + } default: return errs.NotImplement } @@ -279,6 +305,42 @@ func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, return nil, err } return srcObj, nil + case MetaFamily: + pathname := "/isbo/openApi/createBatchOprTask" + var contentList []string + var catalogList []string + if srcObj.IsDir() { + catalogList = append(catalogList, path.Join(srcObj.GetPath(), srcObj.GetID())) + } else { + contentList = append(contentList, path.Join(srcObj.GetPath(), srcObj.GetID())) + } + + body := base.Json{ + "catalogList": catalogList, + "accountInfo": base.Json{ + "accountName": d.getAccount(), + "accountType": "1", + }, + "contentList": contentList, + "destCatalogID": dstDir.GetID(), + "destGroupID": d.CloudID, + "destPath": path.Join(dstDir.GetPath(), dstDir.GetID()), + "destType": 0, + "srcGroupID": d.CloudID, + "srcType": 0, + "taskType": 3, + } + + var resp CreateBatchOprTaskResp + _, err := d.isboPost(pathname, body, &resp) + if err != nil { + return nil, err + } + log.Debugf("[139] Move MetaFamily CreateBatchOprTaskResp.Result.ResultCode: %s", resp.Result.ResultCode) + if resp.Result.ResultCode != "0" { + return nil, fmt.Errorf("failed to move in family cloud: %s", resp.Result.ResultDesc) + } + return srcObj, nil default: return nil, errs.NotImplement } @@ -353,19 +415,27 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e var data base.Json var pathname string if srcObj.IsDir() { - // 网页接口不支持重命名家庭云文件夹 - // data = base.Json{ - // "catalogType": 3, - // "catalogID": srcObj.GetID(), - // "catalogName": newName, - // "commonAccountInfo": base.Json{ - // "account": d.getAccount(), - // "accountType": 1, - // }, - // "path": srcObj.GetPath(), - // } - // pathname = "/orchestration/familyCloud-rebuild/photoContent/v1.0/modifyCatalogInfo" - return errs.NotImplement + pathname = "/modifyCloudDocV2" + data = base.Json{ + "catalogType": 3, + "cloudID": d.CloudID, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": "1", + }, + "docLibName": newName, + "docLibraryID": srcObj.GetID(), + "path": path.Join(srcObj.GetPath(), srcObj.GetID()), + } + var resp ModifyCloudDocV2Resp + _, err = d.andAlbumRequest(pathname, data, &resp) + if err != nil { + return err + } + if resp.Result.ResultCode != "0" { + return fmt.Errorf("failed to rename family folder: %s", resp.Result.ResultDesc) + } + return nil } else { data = base.Json{ "contentID": srcObj.GetID(), @@ -421,6 +491,33 @@ func (d *Yun139) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" _, err = d.post(pathname, data, nil) + case MetaGroup: + err = d.handleMetaGroupCopy(ctx, srcObj, dstDir) + case MetaFamily: + pathname := "/copyContentCatalog" + var sourceContentIDs []string + var sourceCatalogIDs []string + if srcObj.IsDir() { + sourceCatalogIDs = append(sourceCatalogIDs, srcObj.GetID()) + } else { + sourceContentIDs = append(sourceContentIDs, srcObj.GetID()) + } + + body := base.Json{ + "commonAccountInfo": base.Json{ + "accountType": "1", + "accountUserId": d.ref.UserDomainID, + }, + "destCatalogID": dstDir.GetID(), + "destCloudID": d.CloudID, + "sourceCatalogIDs": sourceCatalogIDs, + "sourceCloudID": d.CloudID, + "sourceContentIDs": sourceContentIDs, + } + + var resp base.Json // Assuming a generic JSON response for success/failure + _, err = d.andAlbumRequest(pathname, body, &resp) + // For now, we assume no error means success. default: err = errs.NotImplement } @@ -680,6 +777,8 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return nil case MetaPersonal: fallthrough + case MetaGroup: + fallthrough case MetaFamily: // 处理冲突 // 获取文件列表 @@ -727,12 +826,17 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr }, } pathname := "/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest" - if d.isFamily() { + if d.isFamily() || d.Addition.Type == MetaGroup { + uploadPath := path.Join(dstDir.GetPath(), dstDir.GetID()) + // if dstDir is root folder + if dstDir.GetID() == d.RootFolderID { + uploadPath = d.RootPath + } data = d.newJson(base.Json{ "fileCount": 1, "manualRename": 2, "operation": 0, - "path": path.Join(dstDir.GetPath(), dstDir.GetID()), + "path": uploadPath, "seqNo": random.String(32), // 序列号不能为空 "totalSize": reportSize, "uploadContentList": []base.Json{{ @@ -744,6 +848,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr pathname = "/orchestration/familyCloud-rebuild/content/v1.0/getFileUploadURL" } var resp UploadResp + log.Debugf("[139] upload request body: %+v", data) _, err = d.post(pathname, data, &resp) if err != nil { return err @@ -839,42 +944,42 @@ func (d *Yun139) GetDetails(ctx context.Context) (*model.StorageDetails, error) if d.UserDomainID == "" { return nil, errs.NotImplement } - var total, free uint64 + var total, used int64 if d.isFamily() { diskInfo, err := d.getFamilyDiskInfo(ctx) if err != nil { return nil, err } - totalMb, err := strconv.ParseUint(diskInfo.Data.DiskSize, 10, 64) + totalMb, err := strconv.ParseInt(diskInfo.Data.DiskSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert disk size into integer: %+v", err) } - usedMb, err := strconv.ParseUint(diskInfo.Data.UsedSize, 10, 64) + usedMb, err := strconv.ParseInt(diskInfo.Data.UsedSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert used size into integer: %+v", err) } total = totalMb * 1024 * 1024 - free = total - (usedMb * 1024 * 1024) + used = usedMb * 1024 * 1024 } else { diskInfo, err := d.getPersonalDiskInfo(ctx) if err != nil { return nil, err } - totalMb, err := strconv.ParseUint(diskInfo.Data.DiskSize, 10, 64) + totalMb, err := strconv.ParseInt(diskInfo.Data.DiskSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert disk size into integer: %+v", err) } - freeMb, err := strconv.ParseUint(diskInfo.Data.FreeDiskSize, 10, 64) + freeMb, err := strconv.ParseInt(diskInfo.Data.FreeDiskSize, 10, 64) if err != nil { return nil, fmt.Errorf("failed convert free size into integer: %+v", err) } total = totalMb * 1024 * 1024 - free = freeMb * 1024 * 1024 + used = total - (freeMb * 1024 * 1024) } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: used, }, }, nil } diff --git a/drivers/139/meta.go b/drivers/139/meta.go index b226e240f..91d54fd30 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -8,6 +8,9 @@ import ( type Addition struct { //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true" secret:"true"` + MailCookies string `json:"mail_cookies" required:"true" type:"text" help:"Cookies from mail.139.com used for login authentication."` driver.RootID Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` CloudID string `json:"cloud_id"` diff --git a/drivers/139/types.go b/drivers/139/types.go index 118655deb..499cdbc4d 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -329,3 +329,62 @@ type FamilyDiskInfoResp struct { DiskSize string `json:"diskSize"` } `json:"data"` } + +type AndAlbumUploadResp struct { + Result struct { + ResultCode string `json:"resultCode"` + ResultDesc string `json:"resultDesc"` + } `json:"result"` + UploadResult struct { + UploadTaskID string `json:"uploadTaskID"` + RedirectionURL string `json:"redirectionUrl"` + NewContentIDList []struct { + ContentID string `json:"contentID"` + ContentName string `json:"contentName"` + } `json:"newContentIDList"` + } `json:"uploadResult"` +} + +type ModifyCloudDocV2Req struct { + CatalogType int `json:"catalogType"` + CloudID string `json:"cloudID"` + CommonAccountInfo struct { + Account string `json:"account"` + AccountType string `json:"accountType"` + } `json:"commonAccountInfo"` + DocLibName string `json:"docLibName"` + DocLibraryID string `json:"docLibraryID"` + Path string `json:"path"` +} + +type ModifyCloudDocV2Resp struct { + Result struct { + ResultCode string `json:"resultCode"` + ResultDesc string `json:"resultDesc"` + } `json:"result"` +} + +type CreateBatchOprTaskReq struct { + CatalogList []string `json:"catalogList"` + CommonAccountInfo struct { + Account string `json:"account"` + AccountType string `json:"accountType"` + } `json:"commonAccountInfo"` + ContentList []string `json:"contentList"` + DestCatalogID string `json:"destCatalogID"` + DestGroupID string `json:"destGroupID"` + DestPath string `json:"destPath"` + DestType int `json:"destType"` + SourceCatalogType int `json:"sourceCatalogType"` + SourceCloudID string `json:"sourceCloudID"` + SourceType int `json:"sourceType"` + TaskType int `json:"taskType"` +} + +type CreateBatchOprTaskResp struct { + Result struct { + ResultCode string `json:"resultCode"` + ResultDesc string `json:"resultDesc"` + } `json:"result"` + TaskID string `json:"taskID"` +} diff --git a/drivers/139/util.go b/drivers/139/util.go index 9a9d2c6d0..2c7ac242d 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -1,14 +1,22 @@ package _139 import ( + "bytes" "context" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + crypto_rand "crypto/rand" + "crypto/sha1" "encoding/base64" + "encoding/hex" "errors" "fmt" "io" "net/http" "net/url" "path" + "regexp" "sort" "strconv" "strings" @@ -25,6 +33,11 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + KEY_HEX_1 = "73634235495062495331515373756c734e7253306c673d3d" // 第一层 AES 解密密钥 + KEY_HEX_2 = "7150714477323633586746674c337538" // 第二层 AES 解密密钥 +) + // do others that not defined in Driver interface func (d *Yun139) isFamily() bool { return d.Type == "family" @@ -96,12 +109,16 @@ func (d *Yun139) refreshToken() error { SetBody(reqBody). SetResult(&resp). Post(url) - if err != nil { - return err - } - if resp.Return != "0" { - return fmt.Errorf("failed to refresh token: %s", resp.Desc) + if err != nil || resp.Return != "0" { + log.Warnf("139yun: failed to refresh token with old token: %v, desc: %s. trying to login with password.", err, resp.Desc) + newAuth, loginErr := d.loginWithPassword() + log.Debugf("newAuth: Ok: %s", newAuth) + if loginErr != nil { + return fmt.Errorf("failed to login with password after refresh failed: %w", loginErr) + } + return nil } + d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + ":" + splits[1] + ":" + resp.Token)) op.MustSaveDriverStorage(d) return nil @@ -146,10 +163,29 @@ func (d *Yun139) request(url string, method string, callback base.ReqCallback, r var e BaseResp req.SetResult(&e) + log.Debugf("[139] request: %s %s, body: %s", method, url, string(body)) res, err := req.Execute(method, url) - log.Debugln(res.String()) + if err != nil { + log.Debugf("[139] request error: %v", err) + return nil, err + } + log.Debugf("[139] response body: %s", res.String()) if !e.Success { - return nil, errors.New(e.Message) + // Always try to unmarshal to the specific response type first if 'resp' is provided. + if resp != nil { + err = utils.Json.Unmarshal(res.Body(), resp) + if err != nil { + log.Debugf("[139] failed to unmarshal response to specific type: %v", err) + return nil, err // Return unmarshal error + } + if createBatchOprTaskResp, ok := resp.(*CreateBatchOprTaskResp); ok { + log.Debugf("[139] CreateBatchOprTaskResp.Result.ResultCode: %s", createBatchOprTaskResp.Result.ResultCode) + if createBatchOprTaskResp.Result.ResultCode == "0" { + goto SUCCESS_PROCESS + } + } + } + return nil, errors.New(e.Message) // Fallback to original error if not handled } if resp != nil { err = utils.Json.Unmarshal(res.Body(), resp) @@ -157,6 +193,7 @@ func (d *Yun139) request(url string, method string, callback base.ReqCallback, r return nil, err } } +SUCCESS_PROCESS: return res.Body(), nil } @@ -311,6 +348,9 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { return nil, err } path := resp.Data.Path + if catalogID == d.RootFolderID { + d.RootPath = path + } for _, catalog := range resp.Data.CloudCatalogList { f := model.Object{ ID: catalog.CatalogID, @@ -366,6 +406,9 @@ func (d *Yun139) groupGetFiles(catalogID string) ([]model.Obj, error) { return nil, err } path := resp.Data.GetGroupContentResult.ParentCatalogID + if catalogID == d.RootFolderID { + d.RootPath = path + } for _, catalog := range resp.Data.GetGroupContentResult.CatalogList { f := model.Object{ ID: catalog.CatalogID, @@ -494,11 +537,13 @@ func (d *Yun139) personalRequest(pathname string, method string, callback base.R var e BaseResp req.SetResult(&e) + log.Debugf("[139] personal request: %s %s, body: %s", method, url, string(body)) res, err := req.Execute(method, url) if err != nil { + log.Debugf("[139] personal request error: %v", err) return nil, err } - log.Debugln(res.String()) + log.Debugf("[139] personal response body: %s", res.String()) if !e.Success { return nil, errors.New(e.Message) } @@ -517,6 +562,13 @@ func (d *Yun139) personalPost(pathname string, data interface{}, resp interface{ }, resp) } +func (d *Yun139) isboPost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + url := "https://group.yun.139.com/hcy/mutual/adapter" + pathname + return d.request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, resp) +} + func getPersonalTime(t string) time.Time { stamp, err := time.ParseInLocation("2006-01-02T15:04:05.999-07:00", t, utils.CNLoc) if err != nil { @@ -703,3 +755,645 @@ func (d *Yun139) getFamilyDiskInfo(ctx context.Context) (*FamilyDiskInfoResp, er } return &resp, nil } + +func getMd5(dataStr string) string { + hash := md5.Sum([]byte(dataStr)) + return fmt.Sprintf("%x", hash) +} + +func (d *Yun139) step1_password_login() (string, error) { + log.Debugf("--- 执行步骤 1: 登录 API ---") + loginURL := "https://mail.10086.cn/Login/Login.ashx" + + // 密码 SHA1 哈希 + hashedPassword := sha1Hash(fmt.Sprintf("fetion.com.cn:%s", d.Password)) + log.Debugf("DEBUG: 原始密码: %s", d.Password) + log.Debugf("DEBUG: SHA1 输入: fetion.com.cn:%s", d.Password) + log.Debugf("DEBUG: 生成的 Password 哈希: %s", hashedPassword) + + cguid := strconv.FormatInt(time.Now().UnixMilli(), 10) // 随机生成 cguid + + loginHeaders := map[string]string{ + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "accept-language": "zh-CN,zh;q=0.9,zh-TW;q=0.8,en-US;q=0.7,en;q=0.6,en-GB;q=0.5", + "cache-control": "max-age=0", + "content-type": "application/x-www-form-urlencoded", + "dnt": "1", + "origin": "https://mail.10086.cn", + "priority": "u=0, i", + "referer": fmt.Sprintf("https://mail.10086.cn/default.html?&s=1&v=0&u=%s&m=1&ec=S001&resource=indexLogin&clientid=1003&auto=on&cguid=%s&mtime=45", base64.StdEncoding.EncodeToString([]byte(d.Username)), cguid), + "sec-ch-ua": "\"Microsoft Edge\";v=\"141\", \"Not?A_Brand\";v=\"8\", \"Chromium\";v=\"141\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"Windows\"", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "same-origin", + "sec-fetch-user": "?1", + "upgrade-insecure-requests": "1", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0", + "Cookie": d.MailCookies, + } + + loginData := url.Values{} + loginData.Set("UserName", d.Username) + loginData.Set("passOld", "") + loginData.Set("auto", "on") + loginData.Set("Password", hashedPassword) + loginData.Set("webIndexPagePwdLogin", "1") + loginData.Set("pwdType", "1") + loginData.Set("clientId", "1003") + loginData.Set("authType", "2") + + log.Debugf("DEBUG: 登录请求 URL: %s", loginURL) + log.Debugf("DEBUG: 登录请求 Headers: %+v", loginHeaders) + log.Debugf("DEBUG: 登录请求 Body: %s", loginData.Encode()) + + // 设置客户端不跟随重定向 + client := base.RestyClient.SetRedirectPolicy(resty.NoRedirectPolicy()) + res, err := client.R(). + SetHeaders(loginHeaders). + SetFormDataFromValues(loginData). + Post(loginURL) + + if err != nil { + // 如果是重定向错误，则不作为失败处理，因为我们禁止了自动重定向 + if res != nil && res.StatusCode() >= 300 && res.StatusCode() < 400 { + log.Debugf("DEBUG: 登录响应 Status Code: %d (Redirect)", res.StatusCode()) + } else { + return "", fmt.Errorf("step1 login request failed: %w", err) + } + } else { + log.Debugf("DEBUG: 登录响应 Status Code: %d", res.StatusCode()) + } + // 恢复客户端的默认重定向策略，以免影响后续请求 + base.RestyClient.SetRedirectPolicy(resty.FlexibleRedirectPolicy(10)) + log.Debugf("DEBUG: 登录响应 Headers: %+v", res.Header()) + + var sid, extractedCguid string + + // 从 Location 头部提取 sid 和 cguid + locationHeader := res.Header().Get("Location") + if locationHeader != "" { + sidMatch := regexp.MustCompile(`sid=([^&]+)`).FindStringSubmatch(locationHeader) + cguidMatch := regexp.MustCompile(`cguid=([^&]+)`).FindStringSubmatch(locationHeader) + if len(sidMatch) > 1 { + sid = sidMatch[1] + log.Debugf("DEBUG: 从 Location 提取到 sid: %s", sid) + } + if len(cguidMatch) > 1 { + extractedCguid = cguidMatch[1] + log.Debugf("DEBUG: 从 Location 提取到 cguid: %s", extractedCguid) + } + } + + // 如果 Location 中没有，尝试从 Set-Cookie 中提取 + if sid == "" || extractedCguid == "" { + setCookieHeaders := res.Header().Values("Set-Cookie") + for _, cookieStr := range setCookieHeaders { + ssoSidMatch := regexp.MustCompile(`Os_SSo_Sid=([^;]+)`).FindStringSubmatch(cookieStr) + cookieCguidMatch := regexp.MustCompile(`cguid=([^;]+)`).FindStringSubmatch(cookieStr) + if len(ssoSidMatch) > 1 && sid == "" { + sid = ssoSidMatch[1] + log.Debugf("DEBUG: 从 Set-Cookie 提取到 sid: %s", sid) + } + if len(cookieCguidMatch) > 1 && extractedCguid == "" { + extractedCguid = cookieCguidMatch[1] + log.Debugf("DEBUG: 从 Set-Cookie 提取到 cguid: %s", extractedCguid) + } + } + } + + if sid == "" || extractedCguid == "" { + return "", errors.New("failed to extract sid or cguid from login response") + } + + // 提取并记录 cookies + loginUrlObj, _ := url.Parse(loginURL) + cookies := base.RestyClient.GetClient().Jar.Cookies(loginUrlObj) + var cookieStrings []string + for _, cookie := range cookies { + cookieStrings = append(cookieStrings, cookie.Name+"="+cookie.Value) + } + cookieStr := strings.Join(cookieStrings, "; ") + log.Debugf("DEBUG: 提取到的 Cookies: %s", cookieStr) + d.MailCookies = cookieStr + + return sid, nil +} + +func (d *Yun139) step2_get_single_token(sid string) (string, error) { + log.Debugf("\n--- 执行步骤 2: 换artifact API ---") + cguid := strconv.FormatInt(time.Now().UnixMilli(), 10) + + exchangeArtifactURL := fmt.Sprintf("https://smsrebuild1.mail.10086.cn/setting/s?func=%s&sid=%s&cguid=%s", url.QueryEscape("umc:getArtifact"), sid, cguid) + + // 从 MailCookies 中提取 RMKEY + var rmkey string + cookies := strings.Split(d.MailCookies, ";") + for _, cookie := range cookies { + cookie = strings.TrimSpace(cookie) + if strings.HasPrefix(cookie, "RMKEY=") { + rmkey = cookie + break + } + } + if rmkey == "" { + return "", errors.New("RMKEY not found in MailCookies") + } + + exchangePassidHeaders := map[string]string{ + "Host": "smsrebuild1.mail.10086.cn", + "Cookie": rmkey, + "Content-Type": "text/xml; charset=utf-8", + "Accept-Encoding": "gzip", + "User-Agent": "okhttp/4.12.0", + } + + log.Debugf("DEBUG: 换passid 请求 URL: %s", exchangeArtifactURL) + log.Debugf("DEBUG: 换passid 请求 Headers: %+v", exchangePassidHeaders) + + res, err := base.RestyClient.R(). + SetHeaders(exchangePassidHeaders). + Post(exchangeArtifactURL) + + if err != nil { + return "", fmt.Errorf("step2 exchange artifact request failed: %w", err) + } + + log.Debugf("DEBUG: 换passid 响应 Status Code: %d", res.StatusCode()) + log.Debugf("DEBUG: 换passid 响应 Headers: %+v", res.Header()) + log.Debugf("DEBUG: 换passid 响应 Body: %s...", res.String()[:min(len(res.String()), 500)]) + + dycpwd := jsoniter.Get(res.Body(), "var", "artifact").ToString() + if dycpwd == "" { + return "", errors.New("failed to extract dycpwd from artifact exchange response") + } + log.Debugf("DEBUG: 提取到 dycpwd: %s", dycpwd) + + return dycpwd, nil +} + +// --- 辅助函数：加密/解密 --- + +// sha1Hash 计算 SHA1 哈希值，返回十六进制字符串。 +func sha1Hash(data string) string { + h := sha1.New() + h.Write([]byte(data)) + return hex.EncodeToString(h.Sum(nil)) +} + +// pkcs7_pad PKCS7 填充 +func pkcs7_pad(data []byte, blockSize int) []byte { + padding := blockSize - len(data)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + return append(data, padtext...) +} + +// pkcs7_unpad PKCS7 去填充 +func pkcs7_unpad(data []byte) ([]byte, error) { + length := len(data) + if length == 0 { + return nil, errors.New("pkcs7: data is empty") + } + unpadding := int(data[length-1]) + if unpadding > length { + return nil, errors.New("pkcs7: invalid padding") + } + return data[:(length - unpadding)], nil +} + +// aes_ecb_decrypt AES/ECB/Pkcs7 解密，输入为十六进制字符串。 +func aes_ecb_decrypt(ciphertext []byte, key []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + + if len(ciphertext)%block.BlockSize() != 0 { + return nil, errors.New("AES ECB decrypt: ciphertext is not a multiple of the block size") + } + + decrypted := make([]byte, len(ciphertext)) + blockSize := block.BlockSize() + + for bs, be := 0, blockSize; bs < len(ciphertext); bs, be = bs+blockSize, be+blockSize { + block.Decrypt(decrypted[bs:be], ciphertext[bs:be]) + } + + return pkcs7_unpad(decrypted) +} + +// 以下提供 camelCase 的 AES CBC 加解密，供文件中其它位置调用（并支持传入 IV）。 +func aesCbcEncrypt(plaintext []byte, key []byte, iv []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + if len(iv) != block.BlockSize() { + return nil, fmt.Errorf("aesCbcEncrypt: iv length %d does not match block size %d", len(iv), block.BlockSize()) + } + padded := pkcs7_pad(plaintext, block.BlockSize()) + ciphertext := make([]byte, len(padded)) + mode := cipher.NewCBCEncrypter(block, iv) + mode.CryptBlocks(ciphertext, padded) + return ciphertext, nil +} + +func aesCbcDecrypt(ciphertext []byte, key []byte, iv []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + if len(iv) != block.BlockSize() { + return nil, fmt.Errorf("aesCbcDecrypt: iv length %d does not match block size %d", len(iv), block.BlockSize()) + } + if len(ciphertext)%block.BlockSize() != 0 { + return nil, errors.New("aesCbcDecrypt: ciphertext is not a multiple of the block size") + } + decrypted := make([]byte, len(ciphertext)) + mode := cipher.NewCBCDecrypter(block, iv) + mode.CryptBlocks(decrypted, ciphertext) + return pkcs7_unpad(decrypted) +} + +// sortedJsonStringify 对 JSON 对象进行排序并字符串化。 +func sortedJsonStringify(obj interface{}) (string, error) { + if obj == nil { + return "null", nil + } + + switch v := obj.(type) { + case string: + // 尝试解析为 JSON，如果成功则递归处理 + var parsed interface{} + if err := jsoniter.Unmarshal([]byte(v), &parsed); err == nil { + return sortedJsonStringify(parsed) + } + // 如果不是 JSON 字符串，则直接返回 JSON 字符串化的结果 + return jsoniter.MarshalToString(v) + case int, float64, bool: + return fmt.Sprintf("%v", v), nil + case []interface{}: + var items []string + for _, item := range v { + s, err := sortedJsonStringify(item) + if err != nil { + return "", err + } + items = append(items, s) + } + return fmt.Sprintf("[%s]", strings.Join(items, ",")), nil + case map[string]interface{}: + sortedKeys := make([]string, 0, len(v)) + for key := range v { + sortedKeys = append(sortedKeys, key) + } + sort.Strings(sortedKeys) + + var pairs []string + for _, key := range sortedKeys { + value := v[key] + s, err := sortedJsonStringify(value) + if err != nil { + return "", err + } + // Use jsoniter.MarshalToString for the key to ensure it's quoted correctly + keyStr, err := jsoniter.MarshalToString(key) + if err != nil { + return "", err + } + pairs = append(pairs, fmt.Sprintf("%s:%s", keyStr, s)) + } + return fmt.Sprintf("{%s}", strings.Join(pairs, ",")), nil + default: + // Fallback for other types, e.g., numbers, booleans, or unhandled complex types + // Use jsoniter's default marshalling for these + return jsoniter.MarshalToString(v) + } +} + +// yun139EncryptedRequest handles the common encrypted request/response flow. +func (d *Yun139) yun139EncryptedRequest(url string, body interface{}, headers map[string]string, aesKeyHex string, resp interface{}) ([]byte, error) { + // 1. Decode AES key + aesKey, err := hex.DecodeString(aesKeyHex) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to decode AES key: %w", err) + } + + // 2. Marshal and sort the request body + sortedJson, err := sortedJsonStringify(body) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to marshal and sort body: %w", err) + } + log.Debugf("yun139EncryptedRequest: Request Body (plaintext): %s", sortedJson) + + // 3. Encrypt the body using AES/CBC + iv := make([]byte, 16) // 16 bytes for AES-128 + if _, err := crypto_rand.Read(iv); err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to generate IV: %w", err) + } + encryptedBody, err := aesCbcEncrypt([]byte(sortedJson), aesKey, iv) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to encrypt body: %w", err) + } + payload := base64.StdEncoding.EncodeToString(append(iv, encryptedBody...)) + + // 4. Make the request + res, err := base.RestyClient.R(). + SetHeaders(headers). + SetBody(payload). + Post(url) + + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: http request failed: %w", err) + } + + if res.StatusCode() != 200 { + return nil, fmt.Errorf("yun139EncryptedRequest: unexpected status code %d: %s", res.StatusCode(), res.String()) + } + + // 5. Decrypt the response + respBody := res.Body() + var decryptedBytes []byte + + if len(respBody) > 0 && respBody[0] == '{' { + log.Warnf("yun139EncryptedRequest: received a plain JSON response, not an encrypted string. Body: %s", string(respBody)) + decryptedBytes = respBody + } else { + decodedResp, err := base64.StdEncoding.DecodeString(string(respBody)) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: response base64 decode failed: %w. Body: '%s'", err, string(respBody)) + } + + if len(decodedResp) < 16 { + return nil, fmt.Errorf("yun139EncryptedRequest: decoded response is too short to be encrypted. Length: %d", len(decodedResp)) + } + + respIv := decodedResp[:16] + respCiphertext := decodedResp[16:] + + decryptedBytes, err = aesCbcDecrypt(respCiphertext, aesKey, respIv) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: response aes decrypt failed: %w", err) + } + } + + log.Debugf("yun139EncryptedRequest: Response Body (decrypted): %s", string(decryptedBytes)) + + // 6. Unmarshal to the final response struct + if resp != nil { + err = utils.Json.Unmarshal(decryptedBytes, resp) + if err != nil { + return nil, fmt.Errorf("yun139EncryptedRequest: failed to unmarshal decrypted response: %w", err) + } + } + + return decryptedBytes, nil +} + +func (d *Yun139) step3_third_party_login(dycpwd string) (string, error) { + log.Debugf("\n--- 执行步骤 3: 单点登录 API ---") + ssoLoginURL := "https://user-njs.yun.139.com/user/thirdlogin" + + // 构建原始请求体 + ssoRequestBodyRaw := base.Json{ + "clientkey_decrypt": "l3TryM&Q+X7@dzwk)qP", + "clienttype": "886", + "cpid": "507", + "dycpwd": dycpwd, + "extInfo": base.Json{"ifOpenAccount": "0"}, + "loginMode": "0", + "msisdn": d.Username, + "pintype": "13", + "secinfo": strings.ToUpper(sha1Hash(fmt.Sprintf("fetion.com.cn:%s", dycpwd))), + "version": "20250901", + } + + ssoLoginHeaders := map[string]string{ + "hcy-cool-flag": "1", + "x-huawei-channelSrc": "10246600", + "x-sdk-channelSrc": "", + "x-MM-Source": "0", + "x-UserAgent": "android|23116PN5BC|android15|1.2.6|||1440x3200|10246600", + "x-DeviceInfo": "4|127.0.0.1|5|1.2.6|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|||", + "Content-Type": "text/plain;charset=UTF-8", + "Host": "user-njs.yun.139.com", + "Connection": "Keep-Alive", + "Accept-Encoding": "gzip", + "User-Agent": "okhttp/3.12.2", + } + + // 使用通用加密请求函数 + decryptedLayer1StrBytes, err := d.yun139EncryptedRequest(ssoLoginURL, ssoRequestBodyRaw, ssoLoginHeaders, KEY_HEX_1, nil) + if err != nil { + return "", fmt.Errorf("step3 encrypted request failed: %w", err) + } + + hexInner := jsoniter.Get(decryptedLayer1StrBytes, "data").ToString() + if hexInner == "" { + return "", errors.New("missing data field in first layer decryption result") + } + log.Debugf("DEBUG: 第一层解密提取到 hex_inner: %s...", hexInner[:min(len(hexInner), 50)]) + + // 第二层解密 + key2, err := hex.DecodeString(KEY_HEX_2) + if err != nil { + return "", fmt.Errorf("failed to decode KEY_HEX_2: %w", err) + } + hexInnerBytes, err := hex.DecodeString(hexInner) + if err != nil { + return "", fmt.Errorf("failed to decode hex_inner: %w", err) + } + finalJsonStrBytes, err := aes_ecb_decrypt(hexInnerBytes, key2) + if err != nil { + return "", fmt.Errorf("step3 response layer2 aes ecb decrypt failed: %w", err) + } + log.Debugf("DEBUG: 最终解密结果: %s", string(finalJsonStrBytes)) + + // 提取 authToken + authToken := jsoniter.Get(finalJsonStrBytes, "authToken").ToString() + if authToken == "" { + return "", errors.New("failed to extract authToken from final decryption result") + } + log.Debugf("DEBUG: 提取到 authToken: %s", authToken) + + // 提取 account 和 userDomainId + account := jsoniter.Get(finalJsonStrBytes, "account").ToString() + userDomainId := jsoniter.Get(finalJsonStrBytes, "userDomainId").ToString() + + if account == "" || userDomainId == "" { + return "", errors.New("failed to extract account or userDomainId from final decryption result") + } + + d.UserDomainID = userDomainId + newAuthorization := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("pc:%s:%s", account, authToken))) + return newAuthorization, nil +} + +func (d *Yun139) loginWithPassword() (string, error) { + if d.Username == "" || d.Password == "" || d.MailCookies == "" { + return "", errors.New("username, password or mail_cookies is empty") + } + + passId, err := d.step1_password_login() + if err != nil { + return "", err + } + log.Infof("Step 1 success, passId: %s", passId) + + token, err := d.step2_get_single_token(passId) + if err != nil { + return "", err + } + log.Infof("Step 2 success, token: %s", token) + + newAuth, err := d.step3_third_party_login(token) + if err != nil { + return "", err + } + log.Infof("Step 3 success, new authorization generated.") + + d.Authorization = newAuth // Ensure Authorization is also updated before saving + op.MustSaveDriverStorage(d) + return newAuth, nil +} + +func (d *Yun139) andAlbumRequest(pathname string, body interface{}, resp interface{}) ([]byte, error) { + url := "https://group.yun.139.com/hcy/family/adapter/andAlbum/openApi" + pathname + + headers := map[string]string{ + "Host": "group.yun.139.com", + "authorization": "Basic " + d.getAuthorization(), + "x-svctype": "2", + "hcy-cool-flag": "1", + "api-version": "v2", + "x-huawei-channelsrc": "10246600", + "x-sdk-channelsrc": "", + "x-mm-source": "0", + "x-deviceinfo": "1|127.0.0.1|1|12.3.2|Xiaomi|23116PN5BC||02-00-00-00-00-00|android 15|1440x3200|android|zh||||032|0|", //重要参数 + "content-type": "application/json; charset=utf-8", + "user-agent": "okhttp/4.11.0", + "accept-encoding": "gzip", + } + + return d.yun139EncryptedRequest(url, body, headers, KEY_HEX_1, resp) +} + +func (d *Yun139) handleMetaGroupCopy(ctx context.Context, srcObj, dstDir model.Obj) error { + pathname := "/copyContentCatalog" + var sourceContentIDs []string + var sourceCatalogIDs []string + if srcObj.IsDir() { + sourceCatalogIDs = append(sourceCatalogIDs, path.Join("root:/", srcObj.GetPath(), srcObj.GetID())) + } else { + sourceContentIDs = append(sourceContentIDs, path.Join("root:/", srcObj.GetPath(), srcObj.GetID())) + } + + destCatalogID := path.Join("root:/", dstDir.GetPath(), dstDir.GetID()) + log.Debugf("[139Yun Group Copy] srcObj ID: %s, srcObj Path: %s, dstDir ID: %s, dstDir Path: %s, destCatalogID: %s", srcObj.GetID(), srcObj.GetPath(), dstDir.GetID(), dstDir.GetPath(), destCatalogID) + + body := base.Json{ + "commonAccountInfo": base.Json{ + "accountType": "1", + "accountUserId": d.UserDomainID, + }, + "destCatalogID": destCatalogID, + "destCloudID": d.CloudID, + "sourceCatalogIDs": sourceCatalogIDs, + "sourceCloudID": d.CloudID, + "sourceContentIDs": sourceContentIDs, + } + + var resp base.Json + _, err := d.andAlbumRequest(pathname, body, &resp) + return err +} + +// getGroupRootByCloudID 查询 group 上层信息，优先返回 parentCatalogID，回退到 catalogList[0].path +func (d *Yun139) getGroupRootByCloudID(cloudID string) (string, error) { + pathname := "/orchestration/group-rebuild/catalog/v1.0/queryGroupContentList" + body := base.Json{ + "groupID": cloudID, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + "pageInfo": base.Json{ + "pageNum": 1, + "pageSize": 1, + }, + } + var resp base.Json + _, err := d.post(pathname, body, &resp) + if err != nil { + return "", err + } + dataObj, _ := resp["data"].(map[string]interface{}) + if dataObj == nil { + return "", fmt.Errorf("invalid group response data") + } + if gcr, ok := dataObj["getGroupContentResult"].(map[string]interface{}); ok { + if pid, ok := gcr["parentCatalogID"].(string); ok && pid != "" { + return pid, nil + } + if cl, ok := gcr["catalogList"].([]interface{}); ok && len(cl) > 0 { + if first, ok := cl[0].(map[string]interface{}); ok { + if p, ok := first["path"].(string); ok && p != "" { + return p, nil + } + } + } + } + return "", fmt.Errorf("no root found in group response") +} + +// getFamilyRootPath 查询 family 的上层 path（data.path） +// 返回值已去除前缀 "root:/"（或 "root:"），直接返回纯 ID 或 path 部分，便于持久化为 RootFolderID。 +func (d *Yun139) getFamilyRootPath(cloudID string) (string, error) { + // 使用 v1.2 接口（代码日志中已有该请求），pageSize 取 1 足够获取 path 字段 + pathname := "/orchestration/familyCloud-rebuild/content/v1.2/queryContentList" + body := base.Json{ + "catalogID": "", + "catalogType": 3, + "cloudID": cloudID, + "cloudType": 1, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + "contentSortType": 0, + "pageInfo": base.Json{ + "pageNum": 1, + "pageSize": 1, + }, + "sortDirection": 1, + } + var resp base.Json + _, err := d.post(pathname, body, &resp) + if err != nil { + return "", err + } + dataObj, _ := resp["data"].(map[string]interface{}) + if dataObj == nil { + return "", fmt.Errorf("invalid family response data") + } + // helper to strip "root:/" or "root:" prefix + stripRoot := func(s string) string { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "root:/") + s = strings.TrimPrefix(s, "root:") + return s + } + if p, ok := dataObj["path"].(string); ok && p != "" { + return stripRoot(p), nil + } + // 回退：有时 path 在 cloudCatalogList.catalogList 中 + if cl, ok := dataObj["cloudCatalogList"].([]interface{}); ok && len(cl) > 0 { + if first, ok := cl[0].(map[string]interface{}); ok { + if p, ok := first["path"].(string); ok && p != "" { + return stripRoot(p), nil + } + } + } + return "", fmt.Errorf("no path found in family response") +} diff --git a/drivers/189/driver.go b/drivers/189/driver.go index c12c010c3..ffd5c82f6 100644 --- a/drivers/189/driver.go +++ b/drivers/189/driver.go @@ -202,7 +202,7 @@ func (d *Cloud189) GetDetails(ctx context.Context) (*model.StorageDetails, error return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: capacityInfo.CloudCapacityInfo.TotalSize, - FreeSpace: capacityInfo.CloudCapacityInfo.FreeSize, + UsedSpace: capacityInfo.CloudCapacityInfo.UsedSize, }, }, nil } diff --git a/drivers/189/types.go b/drivers/189/types.go index 4aee16e1d..178a3a5c6 100644 --- a/drivers/189/types.go +++ b/drivers/189/types.go @@ -72,15 +72,15 @@ type CapacityResp struct { ResMessage string `json:"res_message"` Account string `json:"account"` CloudCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - MailUsedSize uint64 `json:"mail189UsedSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + MailUsedSize int64 `json:"mail189UsedSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"cloudCapacityInfo"` FamilyCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } diff --git a/drivers/189/util.go b/drivers/189/util.go index 6e0682ea7..bb9a6adb4 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -107,7 +107,7 @@ import ( // res, err = d.client.R(). // SetHeaders(map[string]string{ // "lt": lt, -// "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", +// "User-Agent": base.UserAgentNT, // "Referer": "https://open.e.189.cn/", // "accept": "application/json;charset=UTF-8", // }).SetFormData(map[string]string{ diff --git a/drivers/189_tv/driver.go b/drivers/189_tv/driver.go index 6ee69c7e3..3b4aa51f6 100644 --- a/drivers/189_tv/driver.go +++ b/drivers/189_tv/driver.go @@ -284,18 +284,18 @@ func (y *Cloud189TV) GetDetails(ctx context.Context) (*model.StorageDetails, err if err != nil { return nil, err } - var total, free uint64 + var total, used int64 if y.isFamily() { total = capacityInfo.FamilyCapacityInfo.TotalSize - free = capacityInfo.FamilyCapacityInfo.FreeSize + used = capacityInfo.FamilyCapacityInfo.UsedSize } else { total = capacityInfo.CloudCapacityInfo.TotalSize - free = capacityInfo.CloudCapacityInfo.FreeSize + used = capacityInfo.CloudCapacityInfo.UsedSize } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: used, }, }, nil } diff --git a/drivers/189_tv/types.go b/drivers/189_tv/types.go index 686a090e8..b8243974b 100644 --- a/drivers/189_tv/types.go +++ b/drivers/189_tv/types.go @@ -322,15 +322,15 @@ type CapacityResp struct { ResMessage string `json:"res_message"` Account string `json:"account"` CloudCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - MailUsedSize uint64 `json:"mail189UsedSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + MailUsedSize int64 `json:"mail189UsedSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"cloudCapacityInfo"` FamilyCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } diff --git a/drivers/189_tv/utils.go b/drivers/189_tv/utils.go index 8678f826e..cea9411ec 100644 --- a/drivers/189_tv/utils.go +++ b/drivers/189_tv/utils.go @@ -70,6 +70,9 @@ func (y *Cloud189TV) request(url, method string, callback base.ReqCallback, para } func (y *Cloud189TV) requestWithRetry(url, method string, callback base.ReqCallback, params map[string]string, resp interface{}, retryCount int, isFamily ...bool) ([]byte, error) { + if y.tokenInfo == nil { + return nil, fmt.Errorf("login failed") + } req := y.client.R().SetQueryParams(clientSuffix()) if params != nil { diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 6861f7c23..6aa06c068 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -416,18 +416,18 @@ func (y *Cloud189PC) GetDetails(ctx context.Context) (*model.StorageDetails, err if err != nil { return nil, err } - var total, free uint64 + var total, used int64 if y.isFamily() { total = capacityInfo.FamilyCapacityInfo.TotalSize - free = capacityInfo.FamilyCapacityInfo.FreeSize + used = capacityInfo.FamilyCapacityInfo.UsedSize } else { total = capacityInfo.CloudCapacityInfo.TotalSize - free = capacityInfo.CloudCapacityInfo.FreeSize + used = capacityInfo.CloudCapacityInfo.UsedSize } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: used, }, }, nil } diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index fe219d8c3..32996debd 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -415,15 +415,15 @@ type CapacityResp struct { ResMessage string `json:"res_message"` Account string `json:"account"` CloudCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - MailUsedSize uint64 `json:"mail189UsedSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + MailUsedSize int64 `json:"mail189UsedSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"cloudCapacityInfo"` FamilyCapacityInfo struct { - FreeSize uint64 `json:"freeSize"` - TotalSize uint64 `json:"totalSize"` - UsedSize uint64 `json:"usedSize"` + FreeSize int64 `json:"freeSize"` + TotalSize int64 `json:"totalSize"` + UsedSize int64 `json:"usedSize"` } `json:"familyCapacityInfo"` TotalSize uint64 `json:"totalSize"` } diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 346b8df2d..08ee658ca 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -90,6 +90,9 @@ func (y *Cloud189PC) EncryptParams(params Params, isFamily bool) string { } func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, params Params, resp interface{}, isFamily ...bool) ([]byte, error) { + if y.getTokenInfo() == nil { + return nil, fmt.Errorf("login failed") + } req := y.getClient().R().SetQueryParams(clientSuffix()) // 设置params @@ -753,30 +756,24 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo } partInfo := "" var reader io.ReadSeeker - var rateLimitedRd io.Reader threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { - if reader == nil { - var err error - reader, err = ss.GetSectionReader(offset, partSize) - if err != nil { - return err - } - silceMd5.Reset() - w, err := utils.CopyWithBuffer(writers, reader) - if w != partSize { - return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", partSize, w, err) - } - // 计算块md5并进行hex和base64编码 - md5Bytes := silceMd5.Sum(nil) - silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes))) - partInfo = fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) - - rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader) + Before: func(ctx context.Context) (err error) { + reader, err = ss.GetSectionReader(offset, partSize) + if err != nil { + return err } + silceMd5.Reset() + w, err := utils.CopyWithBuffer(writers, reader) + if w != partSize { + return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", partSize, w, err) + } + // 计算块md5并进行hex和base64编码 + md5Bytes := silceMd5.Sum(nil) + silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Bytes))) + partInfo = fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) return nil }, - Do: func(ctx context.Context) error { + Do: func(ctx context.Context) (err error) { reader.Seek(0, io.SeekStart) uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, initMultiUpload.Data.UploadFileID, partInfo) if err != nil { @@ -785,11 +782,11 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo // step.4 上传切片 uploadUrl := uploadUrls[0] - _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, rateLimitedRd, isFamily) + _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, driver.NewLimitedUploadStream(ctx, reader), isFamily) if err != nil { return err } - up(float64(threadG.Success()) * 100 / float64(count)) + up(float64(threadG.Success()+1) * 100 / float64(count+1)) return nil }, After: func(err error) { @@ -801,6 +798,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo if err = threadG.Wait(); err != nil { return nil, err } + defer up(100) if fileMd5 != nil { fileMd5Hex = strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) @@ -992,7 +990,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode return err } - up(float64(threadG.Success()) * 100 / float64(len(uploadUrls))) + up(float64(threadG.Success()+1) * 100 / float64(len(uploadUrls)+1)) uploadProgress.UploadParts[i] = "" return nil }) @@ -1004,6 +1002,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } return nil, err } + defer up(100) } // step.5 提交 diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 7f20637fd..e1ba41eb6 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "math/rand" "net/url" stdpath "path" "strings" @@ -16,6 +17,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ) @@ -23,10 +25,9 @@ import ( type Alias struct { model.Storage Addition - rootOrder []string - pathMap map[string][]string - autoFlatten bool - oneKey string + rootOrder []string + pathMap map[string][]string + root model.Obj } func (d *Alias) Config() driver.Config { @@ -38,9 +39,6 @@ func (d *Alias) GetAddition() driver.Additional { } func (d *Alias) Init(ctx context.Context) error { - if d.Paths == "" { - return errors.New("paths is required") - } paths := strings.Split(d.Paths, "\n") d.rootOrder = make([]string, 0, len(paths)) d.pathMap = make(map[string][]string) @@ -50,19 +48,50 @@ func (d *Alias) Init(ctx context.Context) error { continue } k, v := getPair(path) - if _, ok := d.pathMap[k]; !ok { + temp, ok := d.pathMap[k] + if !ok { d.rootOrder = append(d.rootOrder, k) } - d.pathMap[k] = append(d.pathMap[k], v) + d.pathMap[k] = append(temp, v) } - if len(d.pathMap) == 1 { - for k := range d.pathMap { - d.oneKey = k + + switch len(d.rootOrder) { + case 0: + return errors.New("paths is required") + case 1: + paths := d.pathMap[d.rootOrder[0]] + roots := make(BalancedObjs, 0, len(paths)) + roots = append(roots, &model.Object{ + Name: "root", + Path: paths[0], + IsFolder: true, + Modified: d.Modified, + Mask: model.Locked, + }) + for _, path := range paths[1:] { + roots = append(roots, &model.Object{ + Path: path, + }) + } + d.root = roots + default: + d.root = &model.Object{ + Name: "root", + Path: "/", + IsFolder: true, + Modified: d.Modified, + Mask: model.ReadOnly, } - d.autoFlatten = true - } else { - d.oneKey = "" - d.autoFlatten = false + } + + if !utils.SliceContains(ValidReadConflictPolicy, d.ReadConflictPolicy) { + d.ReadConflictPolicy = FirstRWP + } + if !utils.SliceContains(ValidWriteConflictPolicy, d.WriteConflictPolicy) { + d.WriteConflictPolicy = DisabledWP + } + if !utils.SliceContains(ValidPutConflictPolicy, d.PutConflictPolicy) { + d.PutConflictPolicy = DisabledWP } return nil } @@ -70,308 +99,318 @@ func (d *Alias) Init(ctx context.Context) error { func (d *Alias) Drop(ctx context.Context) error { d.rootOrder = nil d.pathMap = nil + d.root = nil return nil } -func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { - if utils.PathEqual(path, "/") { - return &model.Object{ - Name: "Root", - IsFolder: true, - Path: "/", - }, nil +func (d *Alias) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root == nil { + return nil, errs.StorageNotInit } - root, sub := d.getRootAndPath(path) - dsts, ok := d.pathMap[root] - if !ok { + return d.root, nil +} + +// 通过op.Get调用的话，path一定是子路径(/开头) +func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) { + roots, sub := d.getRootsAndPath(path) + if len(roots) == 0 { return nil, errs.ObjectNotFound } - var ret *model.Object - provider := "" - for _, dst := range dsts { - rawPath := stdpath.Join(dst, sub) + for idx, root := range roots { + rawPath := stdpath.Join(root, sub) obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true}) if err != nil { continue } - storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}) - if ret == nil { - ret = &model.Object{ - Path: path, - Name: obj.GetName(), - Size: obj.GetSize(), - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - HashInfo: obj.GetHash(), - } - if !d.ProviderPassThrough || err != nil { - break + mask := model.GetObjMask(obj) &^ model.Temp + if sub == "" { + // 根目录 + mask |= model.Locked | model.Virtual + } + ret := model.Object{ + Path: rawPath, + Name: obj.GetName(), + Size: obj.GetSize(), + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + Mask: mask, + } + obj = &ret + if d.ProviderPassThrough && !obj.IsDir() { + if storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{}); err == nil { + obj = &model.ObjectProvider{ + Object: ret, + Provider: model.Provider{ + Provider: storage.Config().Name, + }, + } } - provider = storage.Config().Name - } else if err != nil || provider != storage.GetStorage().Driver { - provider = "" - break } + + roots = roots[idx+1:] + var objs BalancedObjs + if idx > 0 { + objs = make(BalancedObjs, 0, len(roots)+2) + } else { + objs = make(BalancedObjs, 0, len(roots)+1) + } + objs = append(objs, obj) + if idx > 0 { + objs = append(objs, nil) + } + for _, d := range roots { + objs = append(objs, &tempObj{model.Object{ + Path: stdpath.Join(d, sub), + }}) + } + return objs, nil } - if ret == nil { - return nil, errs.ObjectNotFound - } - if provider != "" { - return &model.ObjectProvider{ - Object: *ret, - Provider: model.Provider{ - Provider: provider, - }, - }, nil - } - return ret, nil + return nil, errs.ObjectNotFound } func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - path := dir.GetPath() - if utils.PathEqual(path, "/") && !d.autoFlatten { - return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough), nil - } - root, sub := d.getRootAndPath(path) - dsts, ok := d.pathMap[root] + dirs, ok := dir.(BalancedObjs) if !ok { - return nil, errs.ObjectNotFound + return d.listRoot(ctx, args.WithStorageDetails && d.DetailsPassThrough, args.Refresh), nil } - var objs []model.Obj - for _, dst := range dsts { - tmp, err := fs.List(ctx, stdpath.Join(dst, sub), &fs.ListArgs{ + + // 因为alias是NoCache且Get方法不会返回NotSupport或NotImplement错误 + // 所以这里对象不会传回到alias，也就不需要返回BalancedObjs了 + objMap := make(map[string]model.Obj) + for _, dir := range dirs { + if dir == nil { + continue + } + dirPath := dir.GetPath() + tmp, err := fs.List(ctx, dirPath, &fs.ListArgs{ NoLog: true, Refresh: args.Refresh, WithStorageDetails: args.WithStorageDetails && d.DetailsPassThrough, }) - if err == nil { - tmp, err = utils.SliceConvert(tmp, func(obj model.Obj) (model.Obj, error) { - thumb, ok := model.GetThumb(obj) - objRes := model.Object{ - Name: obj.GetName(), - Size: obj.GetSize(), - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - } - if !ok { - return &objRes, nil - } - return &model.ObjThumb{ + if err != nil { + continue + } + for _, obj := range tmp { + name := obj.GetName() + if _, exists := objMap[name]; exists { + continue + } + mask := model.GetObjMask(obj) &^ model.Temp + objRes := model.Object{ + Name: name, + Path: stdpath.Join(dirPath, name), + Size: obj.GetSize(), + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + Mask: mask, + } + var objRet model.Obj + if thumb, ok := model.GetThumb(obj); ok { + objRet = &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ Thumbnail: thumb, }, - }, nil - }) + } + } else { + objRet = &objRes + } + if details, ok := model.GetStorageDetails(obj); ok { + objRet = &model.ObjStorageDetails{ + Obj: objRet, + StorageDetails: details, + } + } + objMap[name] = objRet } - if err == nil { - objs = append(objs, tmp...) + } + objs := make([]model.Obj, 0, len(objMap)) + for _, obj := range objMap { + objs = append(objs, obj) + } + if d.OrderBy == "" { + sort := getAllSort(dirs) + if sort.OrderBy != "" { + model.SortFiles(objs, sort.OrderBy, sort.OrderDirection) + } + if d.ExtractFolder == "" && sort.ExtractFolder != "" { + model.ExtractFolder(objs, sort.ExtractFolder) } } return objs, nil } func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - root, sub := d.getRootAndPath(file.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound - } - // proxy || ftp,s3 - if common.GetApiUrl(ctx) == "" { - args.Redirect = false - } - for _, dst := range dsts { - reqPath := stdpath.Join(dst, sub) - link, fi, err := d.link(ctx, reqPath, args) + if d.ReadConflictPolicy == AllRWP && !args.Redirect { + files, err := d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) if err != nil { - continue + return nil, err } - if link == nil { - // 重定向且需要通过代理 - return &model.Link{ - URL: fmt.Sprintf("%s/p%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)), - }, nil + linkClosers := make([]io.Closer, 0, len(files)) + rrf := make([]model.RangeReaderIF, 0, len(files)) + for _, f := range files { + link, fi, err := d.link(ctx, f.GetPath(), args) + if err != nil { + continue + } + if fi.GetSize() != files.GetSize() { + _ = link.Close() + continue + } + l := *link // 复制一份，避免修改到原始link + if l.ContentLength == 0 { + l.ContentLength = fi.GetSize() + } + if d.DownloadConcurrency > 0 { + l.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + l.PartSize = d.DownloadPartSize * utils.KB + } + rr, err := stream.GetRangeReaderFromLink(l.ContentLength, &l) + if err != nil { + _ = link.Close() + continue + } + linkClosers = append(linkClosers, link) + rrf = append(rrf, rr) } - - resultLink := *link - resultLink.SyncClosers = utils.NewSyncClosers(link) - if args.Redirect { - return &resultLink, nil + rr := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return rrf[rand.Intn(len(rrf))].RangeRead(ctx, httpRange) } + return &model.Link{ + RangeReader: stream.RangeReaderFunc(rr), + SyncClosers: utils.NewSyncClosers(linkClosers...), + }, nil + } - if resultLink.ContentLength == 0 { - resultLink.ContentLength = fi.GetSize() - } - if resultLink.MFile != nil { - return &resultLink, nil - } - if d.DownloadConcurrency > 0 { - resultLink.Concurrency = d.DownloadConcurrency + var link *model.Link + var fi model.Obj + var err error + files := file.(BalancedObjs) + if d.ReadConflictPolicy == RandomBalancedRP || d.ReadConflictPolicy == AllRWP { + rand.Shuffle(len(files), func(i, j int) { + files[i], files[j] = files[j], files[i] + }) + } + for _, f := range files { + if f == nil { + continue } - if d.DownloadPartSize > 0 { - resultLink.PartSize = d.DownloadPartSize * utils.KB + link, fi, err = d.link(ctx, f.GetPath(), args) + if err == nil { + if link == nil { + // 重定向且需要通过代理 + return &model.Link{ + URL: fmt.Sprintf("%s/p%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(f.GetPath(), true), + sign.Sign(f.GetPath())), + }, nil + } + break } + } + if err != nil { + return nil, err + } + resultLink := *link // 复制一份，避免修改到原始link + resultLink.Expiration = nil + resultLink.SyncClosers = utils.NewSyncClosers(link) + if args.Redirect { return &resultLink, nil } - return nil, errs.ObjectNotFound + if resultLink.ContentLength == 0 { + resultLink.ContentLength = fi.GetSize() + } + if d.DownloadConcurrency > 0 { + resultLink.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + resultLink.PartSize = d.DownloadPartSize * utils.KB + } + return &resultLink, nil } func (d *Alias) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { - root, sub := d.getRootAndPath(args.Obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound - } - for _, dst := range dsts { - rawPath := stdpath.Join(dst, sub) - storage, actualPath, err := op.GetStorageAndActualPath(rawPath) - if err != nil { - continue - } - other, ok := storage.(driver.Other) - if !ok { - continue - } - obj, err := op.GetUnwrap(ctx, storage, actualPath) - if err != nil { - continue - } - return other.Other(ctx, model.OtherArgs{ - Obj: obj, - Method: args.Method, - Data: args.Data, - }) + // Other 不应负载均衡，这是因为前端是否调用 /fs/other 的判断条件是返回的 provider 的值 + // 而 ProviderPassThrough 开启时，返回的 provider 固定为第一个 obj 的后端驱动 + storage, actualPath, err := op.GetStorageAndActualPath(args.Obj.GetPath()) + if err != nil { + return nil, err } - return nil, errs.NotImplement + return op.Other(ctx, storage, model.FsOtherArgs{ + Path: actualPath, + Method: args.Method, + Data: args.Data, + }) } func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, parentDir, true) + objs, err := d.getWriteObjs(ctx, parentDir) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(*path, dirName))) + for _, obj := range objs { + err = errors.Join(err, fs.MakeDir(ctx, stdpath.Join(obj.GetPath(), dirName))) } - return err - } - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot make sub-dir") } return err } func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - if !d.Writable { - return errs.PermissionDenied - } - srcPath, err := d.getReqPath(ctx, srcObj, false) - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be moved") - } - if err != nil { - return err - } - dstPath, err := d.getReqPath(ctx, dstDir, true) - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be moved to") - } - if err != nil { - return err - } - if len(srcPath) == len(dstPath) { - for i := range srcPath { - _, e := fs.Move(ctx, *srcPath[i], *dstPath[i]) + srcs, dsts, err := d.getMoveObjs(ctx, srcObj, dstDir) + if err == nil { + for i, dst := range dsts { + src := srcs[i] + _, e := fs.Move(ctx, src.GetPath(), dst.GetPath()) + err = errors.Join(err, e) + } + srcs = srcs[len(dsts):] + for _, src := range srcs { + e := fs.Remove(ctx, src.GetPath()) err = errors.Join(err, e) } - return err - } else { - return errors.New("parallel paths mismatch") } + return err } func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, srcObj, false) + objs, err := d.getWriteObjs(ctx, srcObj) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.Rename(ctx, *path, newName)) + for _, obj := range objs { + err = errors.Join(err, fs.Rename(ctx, obj.GetPath(), newName)) } - return err - } - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be Rename") } return err } func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - if !d.Writable { - return errs.PermissionDenied - } - srcPath, err := d.getReqPath(ctx, srcObj, false) - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be copied") - } - if err != nil { - return err - } - dstPath, err := d.getReqPath(ctx, dstDir, true) - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be copied to") - } - if err != nil { - return err - } - if len(srcPath) == len(dstPath) { - for i := range srcPath { - _, e := fs.Copy(ctx, *srcPath[i], *dstPath[i]) - err = errors.Join(err, e) - } - return err - } else if len(srcPath) == 1 || !d.ProtectSameName { - for _, path := range dstPath { - _, e := fs.Copy(ctx, *srcPath[0], *path) + srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) + if err == nil { + for i, src := range srcs { + dst := dsts[i] + _, e := fs.Copy(ctx, src.GetPath(), dst.GetPath()) err = errors.Join(err, e) } - return err - } else { - return errors.New("parallel paths mismatch") } + return err } func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, obj, false) + objs, err := d.getWriteObjs(ctx, obj) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.Remove(ctx, *path)) + for _, obj := range objs { + err = errors.Join(err, fs.Remove(ctx, obj.GetPath())) } - return err - } - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be Delete") } return err } func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, dstDir, true) + objs, err := d.getPutObjs(ctx, dstDir) if err == nil { - if len(reqPath) == 1 { - storage, reqActualPath, err := op.GetStorageAndActualPath(*reqPath[0]) + if len(objs) == 1 { + storage, reqActualPath, err := op.GetStorageAndActualPath(objs.GetPath()) if err != nil { return err } @@ -385,10 +424,10 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, if err != nil { return err } - count := float64(len(reqPath) + 1) + count := float64(len(objs) + 1) up(100 / count) - for i, path := range reqPath { - err = errors.Join(err, fs.PutDirectly(ctx, *path, &stream.FileStream{ + for i, obj := range objs { + err = errors.Join(err, fs.PutDirectly(ctx, obj.GetPath(), &stream.FileStream{ Obj: s, Mimetype: s.GetMimetype(), Reader: file, @@ -402,55 +441,40 @@ func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, return err } } - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be Put") - } return err } func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { - if !d.Writable { - return errs.PermissionDenied - } - reqPath, err := d.getReqPath(ctx, dstDir, true) + objs, err := d.getPutObjs(ctx, dstDir) if err == nil { - for _, path := range reqPath { - err = errors.Join(err, fs.PutURL(ctx, *path, name, url)) + for _, obj := range objs { + err = errors.Join(err, fs.PutURL(ctx, obj.GetPath(), name, url)) } return err } - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot offline download") - } return err } func (d *Alias) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { - root, sub := d.getRootAndPath(obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := d.getBalancedPath(ctx, obj) + if reqPath == "" { + return nil, errs.NotFile } - for _, dst := range dsts { - meta, err := d.getArchiveMeta(ctx, dst, sub, args) - if err == nil { - return meta, nil - } + meta, err := d.getArchiveMeta(ctx, reqPath, args) + if err == nil { + return meta, nil } return nil, errs.NotImplement } func (d *Alias) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { - root, sub := d.getRootAndPath(obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := d.getBalancedPath(ctx, obj) + if reqPath == "" { + return nil, errs.NotFile } - for _, dst := range dsts { - l, err := d.listArchive(ctx, dst, sub, args) - if err == nil { - return l, nil - } + l, err := d.listArchive(ctx, reqPath, args) + if err == nil { + return l, nil } return nil, errs.NotImplement } @@ -459,67 +483,88 @@ func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveIn // alias的两个驱动，一个支持驱动提取，一个不支持，如何兼容？ // 如果访问的是不支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回errs.NotImplement，提取URL前缀就会是/ae，Extract就不会被调用 // 如果访问的是支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回有效值，提取URL前缀就会是/ad，Extract就会被调用 - root, sub := d.getRootAndPath(obj.GetPath()) - dsts, ok := d.pathMap[root] - if !ok { - return nil, errs.ObjectNotFound + reqPath := d.getBalancedPath(ctx, obj) + if reqPath == "" { + return nil, errs.NotFile } - for _, dst := range dsts { - reqPath := stdpath.Join(dst, sub) - link, err := d.extract(ctx, reqPath, args) - if err != nil { - continue - } - if link == nil { - return &model.Link{ - URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - utils.EncodePath(args.InnerPath, true), - url.QueryEscape(args.Password), - sign.SignArchive(reqPath)), - }, nil - } - resultLink := *link - resultLink.SyncClosers = utils.NewSyncClosers(link) - return &resultLink, nil + link, err := d.extract(ctx, reqPath, args) + if err != nil { + return nil, errs.NotImplement + } + if link == nil { + return &model.Link{ + URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(reqPath, true), + utils.EncodePath(args.InnerPath, true), + url.QueryEscape(args.Password), + sign.SignArchive(reqPath)), + }, nil } - return nil, errs.NotImplement + resultLink := *link + resultLink.SyncClosers = utils.NewSyncClosers(link) + return &resultLink, nil } func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { - if !d.Writable { - return errs.PermissionDenied + srcs, dsts, err := d.getCopyObjs(ctx, srcObj, dstDir) + if err == nil { + for i, src := range srcs { + dst := dsts[i] + _, e := fs.ArchiveDecompress(ctx, src.GetPath(), dst.GetPath(), args) + err = errors.Join(err, e) + } } - srcPath, err := d.getReqPath(ctx, srcObj, false) - if errs.IsNotImplementError(err) { - return errors.New("same-name files cannot be decompressed") + return err +} + +func (d *Alias) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + if !d.DetailsPassThrough { + return nil, errs.NotImplement } - if err != nil { - return err + if len(d.rootOrder) != 1 { + return nil, errs.NotImplement } - dstPath, err := d.getReqPath(ctx, dstDir, true) - if errs.IsNotImplementError(err) { - return errors.New("same-name dirs cannot be decompressed to") + backends := d.pathMap[d.rootOrder[0]] + var storage driver.Driver + for _, backend := range backends { + s, err := fs.GetStorage(backend, &fs.GetStoragesArgs{}) + if err != nil { + return nil, errs.NotImplement + } + if storage == nil { + storage = s + } else if storage.GetStorage().MountPath != s.GetStorage().MountPath { + return nil, errs.NotImplement + } } - if err != nil { - return err + if storage == nil { // should never access + return nil, errs.NotImplement } - if len(srcPath) == len(dstPath) { - for i := range srcPath { - _, e := fs.ArchiveDecompress(ctx, *srcPath[i], *dstPath[i], args) - err = errors.Join(err, e) + return op.GetStorageDetails(ctx, storage) +} + +func (d *Alias) ResolveLinkCacheMode(path string) driver.LinkCacheMode { + roots, sub := d.getRootsAndPath(path) + if len(roots) == 0 { + return 0 + } + for _, root := range roots { + storage, actualPath, err := op.GetStorageAndActualPath(stdpath.Join(root, sub)) + if err != nil { + continue } - return err - } else if len(srcPath) == 1 || !d.ProtectSameName { - for _, path := range dstPath { - _, e := fs.ArchiveDecompress(ctx, *srcPath[0], *path, args) - err = errors.Join(err, e) + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + continue + } + mode := storage.Config().LinkCacheMode + if mode == -1 { + return storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(actualPath) + } else { + return mode } - return err - } else { - return errors.New("parallel paths mismatch") } + return 0 } var _ driver.Driver = (*Alias)(nil) diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index bdab53748..72eb3c877 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -6,17 +6,15 @@ import ( ) type Addition struct { - // Usually one of two - // driver.RootPath - // define other - Paths string `json:"paths" required:"true" type:"text"` - ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"` - ParallelWrite bool `json:"parallel_write" type:"bool" default:"false"` - DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` - DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` - Writable bool `json:"writable" type:"bool" default:"false"` - ProviderPassThrough bool `json:"provider_pass_through" type:"bool" default:"false"` - DetailsPassThrough bool `json:"details_pass_through" type:"bool" default:"false"` + Paths string `json:"paths" required:"true" type:"text"` + ReadConflictPolicy string `json:"read_conflict_policy" type:"select" options:"first,random,all" default:"first"` + WriteConflictPolicy string `json:"write_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict" default:"disabled" help:"How the driver handles identical backend paths when renaming, removing, or making directories."` + PutConflictPolicy string `json:"put_conflict_policy" type:"select" options:"disabled,first,deterministic,deterministic_or_all,all,all_strict,random,quota,quota_strict" default:"disabled" help:"How the driver handles identical backend paths when uploading, copying, moving, or decompressing."` + FileConsistencyCheck bool `json:"file_consistency_check" type:"bool" default:"false"` + DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` + DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` + ProviderPassThrough bool `json:"provider_pass_through" type:"bool" default:"false"` + DetailsPassThrough bool `json:"details_pass_through" type:"bool" default:"false"` } var config = driver.Config{ @@ -26,14 +24,11 @@ var config = driver.Config{ NoUpload: false, DefaultRoot: "/", ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheAuto, } func init() { op.RegisterDriver(func() driver.Driver { - return &Alias{ - Addition: Addition{ - ProtectSameName: true, - }, - } + return &Alias{} }) } diff --git a/drivers/alias/types.go b/drivers/alias/types.go index e560393da..9fade0c5b 100644 --- a/drivers/alias/types.go +++ b/drivers/alias/types.go @@ -1 +1,78 @@ package alias + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/pkg/errors" +) + +const ( + DisabledWP = "disabled" + FirstRWP = "first" + DeterministicWP = "deterministic" + DeterministicOrAllWP = "deterministic_or_all" + AllRWP = "all" + AllStrictWP = "all_strict" + RandomBalancedRP = "random" + BalancedByQuotaP = "quota" + BalancedByQuotaStrictP = "quota_strict" +) + +var ( + ValidReadConflictPolicy = []string{FirstRWP, RandomBalancedRP, AllRWP} + ValidWriteConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllRWP, + AllStrictWP} + ValidPutConflictPolicy = []string{DisabledWP, FirstRWP, DeterministicWP, DeterministicOrAllWP, AllRWP, + AllStrictWP, RandomBalancedRP, BalancedByQuotaP, BalancedByQuotaStrictP} +) + +var ( + ErrPathConflict = errors.New("path conflict") + ErrSamePathLeak = errors.New("leak some of same-name dirs") + ErrNoEnoughSpace = errors.New("none of same-name dirs has enough space") + ErrNotEnoughSrcObjs = errors.New("cannot move fewer objs to more paths, please try copying") +) + +type BalancedObjs []model.Obj + +func (b BalancedObjs) GetSize() int64 { + return b[0].GetSize() +} + +func (b BalancedObjs) ModTime() time.Time { + return b[0].ModTime() +} + +func (b BalancedObjs) CreateTime() time.Time { + return b[0].CreateTime() +} + +func (b BalancedObjs) IsDir() bool { + return b[0].IsDir() +} + +func (b BalancedObjs) GetHash() utils.HashInfo { + return b[0].GetHash() +} + +func (b BalancedObjs) GetName() string { + return b[0].GetName() +} + +func (b BalancedObjs) GetPath() string { + return b[0].GetPath() +} + +func (b BalancedObjs) GetID() string { + return b[0].GetID() +} + +func (b BalancedObjs) Unwrap() model.Obj { + return b[0] +} + +var _ model.Obj = (BalancedObjs)(nil) + +type tempObj struct{ model.Object } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 33960c944..8e5eb8a84 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -2,10 +2,10 @@ package alias import ( "context" - "errors" + "math/rand" stdpath "path" "strings" - "sync" + "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -13,20 +13,29 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) -func (d *Alias) listRoot(ctx context.Context, withDetails bool) []model.Obj { +type detailWithIndex struct { + idx int + val *model.StorageDetails +} + +func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model.Obj { var objs []model.Obj - var wg sync.WaitGroup + detailsChan := make(chan detailWithIndex, len(d.pathMap)) + workerCount := 0 for _, k := range d.rootOrder { - obj := model.Object{ + obj := &model.Object{ Name: k, + Path: "/" + k, IsFolder: true, Modified: d.Modified, + Mask: model.Locked | model.Virtual, } idx := len(objs) - objs = append(objs, &obj) + objs = append(objs, obj) v := d.pathMap[k] if !withDetails || len(v) != 1 { continue @@ -35,56 +44,56 @@ func (d *Alias) listRoot(ctx context.Context, withDetails bool) []model.Obj { if err != nil { continue } + obj.Modified = remoteDriver.GetStorage().Modified _, ok := remoteDriver.(driver.WithDetails) if !ok { continue } objs[idx] = &model.ObjStorageDetails{ - Obj: objs[idx], - StorageDetailsWithName: model.StorageDetailsWithName{ - StorageDetails: nil, - DriverName: remoteDriver.Config().Name, - }, - } - wg.Add(1) - go func() { - defer wg.Done() - details, e := op.GetStorageDetails(ctx, remoteDriver) + Obj: objs[idx], + StorageDetails: nil, + } + workerCount++ + go func(dri driver.Driver, i int) { + details, e := op.GetStorageDetails(ctx, dri, refresh) if e != nil { - if !errors.Is(e, errs.NotImplement) { - log.Errorf("failed get %s storage details: %+v", remoteDriver.GetStorage().MountPath, e) + if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { + log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e) } - return } - objs[idx].(*model.ObjStorageDetails).StorageDetails = details - }() + detailsChan <- detailWithIndex{idx: i, val: details} + }(remoteDriver, idx) + } + for workerCount > 0 { + select { + case r := <-detailsChan: + objs[r.idx].(*model.ObjStorageDetails).StorageDetails = r.val + workerCount-- + case <-time.After(time.Second): + workerCount = 0 + } } - wg.Wait() return objs } // do others that not defined in Driver interface func getPair(path string) (string, string) { - // path = strings.TrimSpace(path) - if strings.Contains(path, ":") { - pair := strings.SplitN(path, ":", 2) - if !strings.Contains(pair[0], "/") { - return pair[0], pair[1] - } + if name, path, ok := strings.Cut(path, ":"); ok && !strings.Contains(name, "/") { + return name, path } return stdpath.Base(path), path } -func (d *Alias) getRootAndPath(path string) (string, string) { - if d.autoFlatten { - return d.oneKey, path +func (d *Alias) getRootsAndPath(path string) (roots []string, sub string) { + if len(d.rootOrder) == 1 { + return d.pathMap[d.rootOrder[0]], path } path = strings.TrimPrefix(path, "/") - parts := strings.SplitN(path, "/", 2) - if len(parts) == 1 { - return parts[0], "" + before, after, ok := strings.Cut(path, "/") + if !ok { + return d.pathMap[path], "" } - return parts[0], parts[1] + return d.pathMap[before], after } func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) (*model.Link, model.Obj, error) { @@ -92,59 +101,350 @@ func (d *Alias) link(ctx context.Context, reqPath string, args model.LinkArgs) ( if err != nil { return nil, nil, err } - if !args.Redirect { - return op.Link(ctx, storage, reqActualPath, args) + if args.Redirect && common.ShouldProxy(storage, stdpath.Base(reqPath)) { + return nil, nil, nil } - obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) - if err != nil { - return nil, nil, err + return op.Link(ctx, storage, reqActualPath, args) +} + +func isConsistent(a, b model.Obj) bool { + if a.GetSize() != b.GetSize() { + return false } - if common.ShouldProxy(storage, stdpath.Base(reqPath)) { - return nil, obj, nil + for ht, v := range a.GetHash().All() { + ah := b.GetHash().GetHash(ht) + if ah != "" && ah != v { + return false + } } - return op.Link(ctx, storage, reqActualPath, args) + return true } -func (d *Alias) getReqPath(ctx context.Context, obj model.Obj, isParent bool) ([]*string, error) { - root, sub := d.getRootAndPath(obj.GetPath()) - if sub == "" && !isParent { - return nil, errs.NotSupport +func (d *Alias) getAllObjs(ctx context.Context, bObj model.Obj, ifContinue func(err error) (bool, error)) (BalancedObjs, error) { + objs := bObj.(BalancedObjs) + length := 0 + for _, o := range objs { + var err error + var obj model.Obj + temp, isTemp := o.(*tempObj) + if isTemp { + obj, err = fs.Get(ctx, o.GetPath(), &fs.GetArgs{NoLog: true}) + if err == nil { + if !bObj.IsDir() { + if obj.IsDir() { + err = errs.NotFile + } else if d.FileConsistencyCheck && !isConsistent(bObj, obj) { + err = errs.ObjectNotFound + } + } else if !obj.IsDir() { + err = errs.NotFolder + } + } + } else if o == nil { + err = errs.ObjectNotFound + } + + cont, err := ifContinue(err) + if err != nil { + if cont { + continue + } + return nil, err + } + if isTemp { + objRes := temp.Object + // objRes.Name = obj.GetName() + // objRes.Size = obj.GetSize() + // objRes.Modified = obj.ModTime() + // objRes.HashInfo = obj.GetHash() + objs[length] = &objRes + } else { + objs[length] = o + } + length++ + if !cont { + break + } } - dsts, ok := d.pathMap[root] - all := true - if !ok { + if length == 0 { return nil, errs.ObjectNotFound } - var reqPath []*string - for _, dst := range dsts { - path := stdpath.Join(dst, sub) - _, err := fs.Get(ctx, path, &fs.GetArgs{NoLog: true}) + return objs[:length], nil +} + +func (d *Alias) getBalancedPath(ctx context.Context, file model.Obj) string { + if d.ReadConflictPolicy == FirstRWP { + return file.GetPath() + } + files := file.(BalancedObjs) + if rand.Intn(len(files)) == 0 { + return file.GetPath() + } + files, _ = d.getAllObjs(ctx, file, getWriteAndPutFilterFunc(AllRWP)) + return files[rand.Intn(len(files))].GetPath() +} + +func getWriteAndPutFilterFunc(policy string) func(error) (bool, error) { + if policy == AllRWP { + return func(err error) (bool, error) { + return true, err + } + } + all := true + l := 0 + return func(err error) (bool, error) { if err != nil { + switch policy { + case AllStrictWP: + return false, ErrSamePathLeak + case DeterministicOrAllWP: + if l >= 2 { + return false, ErrSamePathLeak + } + } all = false - if d.ProtectSameName && d.ParallelWrite && len(reqPath) >= 2 { - return nil, errs.NotImplement + } else { + switch policy { + case FirstRWP: + return false, nil + case DeterministicWP: + if l > 0 { + return false, ErrPathConflict + } + case DeterministicOrAllWP: + if l > 0 && !all { + return false, ErrSamePathLeak + } } + l += 1 + } + return true, err + } +} + +func (d *Alias) getWriteObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { + if d.WriteConflictPolicy == DisabledWP { + return nil, errs.PermissionDenied + } + return d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.WriteConflictPolicy)) +} + +func (d *Alias) getPutObjs(ctx context.Context, obj model.Obj) (BalancedObjs, error) { + if d.PutConflictPolicy == DisabledWP { + return nil, errs.PermissionDenied + } + objs, err := d.getAllObjs(ctx, obj, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + if err != nil { + return nil, err + } + strict := false + switch d.PutConflictPolicy { + case RandomBalancedRP: + ri := rand.Intn(len(objs)) + return objs[ri : ri+1], nil + case BalancedByQuotaStrictP: + strict = true + fallthrough + case BalancedByQuotaP: + objs, ok := getRandomObjByQuotaBalanced(ctx, objs, strict, obj.GetSize()) + if !ok { + return nil, ErrNoEnoughSpace + } + return objs, nil + default: + return objs, nil + } +} + +func getRandomObjByQuotaBalanced(ctx context.Context, reqPath BalancedObjs, strict bool, objSize int64) (BalancedObjs, bool) { + // Get all space + details := make([]*model.StorageDetails, len(reqPath)) + detailsChan := make(chan detailWithIndex, len(reqPath)) + workerCount := 0 + for i, p := range reqPath { + s, err := fs.GetStorage(p.GetPath(), &fs.GetStoragesArgs{}) + if err != nil { continue } - if !d.ProtectSameName && !d.ParallelWrite { - return []*string{&path}, nil + if _, ok := s.(driver.WithDetails); !ok { + continue } - reqPath = append(reqPath, &path) - if d.ProtectSameName && !d.ParallelWrite && len(reqPath) >= 2 { - return nil, errs.NotImplement + workerCount++ + go func(dri driver.Driver, i int) { + d, e := op.GetStorageDetails(ctx, dri) + if e != nil { + if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { + log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, e) + } + } + detailsChan <- detailWithIndex{idx: i, val: d} + }(s, i) + } + for workerCount > 0 { + select { + case r := <-detailsChan: + details[r.idx] = r.val + workerCount-- + case <-time.After(time.Second): + workerCount = 0 } - if d.ProtectSameName && d.ParallelWrite && len(reqPath) >= 2 && !all { - return nil, errs.NotImplement + } + + // Try select one that has space info + selected, ok := selectRandom(details, func(d *model.StorageDetails) uint64 { + if d == nil || d.FreeSpace() < objSize { + return 0 + } + return uint64(d.FreeSpace()) + }) + if !ok { + if strict { + return nil, false + } else { + // No strict mode, return any of non-details ones + noDetails := make([]int, 0, len(details)) + for i, d := range details { + if d == nil { + noDetails = append(noDetails, i) + } + } + if len(noDetails) == 0 { + return nil, false + } + selected = noDetails[rand.Intn(len(noDetails))] } } - if len(reqPath) == 0 { - return nil, errs.ObjectNotFound + return reqPath[selected : selected+1], true +} + +func selectRandom[Item any](arr []Item, getWeight func(Item) uint64) (int, bool) { + var totalWeight uint64 = 0 + for _, i := range arr { + totalWeight += getWeight(i) + } + if totalWeight == 0 { + return 0, false + } + r := rand.Uint64() % totalWeight + for i, item := range arr { + w := getWeight(item) + if r < w { + return i, true + } + r -= w + } + return 0, false +} + +func (d *Alias) getCopyObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { + if d.PutConflictPolicy == DisabledWP { + return nil, nil, errs.PermissionDenied + } + dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + if err != nil { + return nil, nil, err + } + dstStorageMap := make(map[string][]model.Obj) + allocatingDst := make(map[model.Obj]struct{}) + for _, o := range dstObjs { + storage, e := fs.GetStorage(o.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + return nil, nil, errors.WithMessagef(e, "cannot copy to virtual path [%s]", o.GetPath()) + } + mp := storage.GetStorage().MountPath + dstStorageMap[mp] = append(dstStorageMap[mp], o) + allocatingDst[o] = struct{}{} + } + tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllRWP)) + if err != nil { + return nil, nil, err + } + srcObjs := make(BalancedObjs, 0, len(dstObjs)) + for _, src := range tmpSrcObjs { + storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + continue + } + mp := storage.GetStorage().MountPath + if tmp, ok := dstStorageMap[mp]; ok { + for _, dst := range tmp { + dstObjs[len(srcObjs)] = dst + srcObjs = append(srcObjs, src) + delete(allocatingDst, dst) + } + delete(dstStorageMap, mp) + } + } + dstObjs = dstObjs[:len(srcObjs)] + for dst := range allocatingDst { + src := tmpSrcObjs[0] + if d.ReadConflictPolicy == RandomBalancedRP || d.ReadConflictPolicy == AllRWP { + src = tmpSrcObjs[rand.Intn(len(tmpSrcObjs))] + } + srcObjs = append(srcObjs, src) + dstObjs = append(dstObjs, dst) + } + return srcObjs, dstObjs, nil +} + +func (d *Alias) getMoveObjs(ctx context.Context, srcObj, dstDir model.Obj) (BalancedObjs, BalancedObjs, error) { + if d.PutConflictPolicy == DisabledWP { + return nil, nil, errs.PermissionDenied + } + dstObjs, err := d.getAllObjs(ctx, dstDir, getWriteAndPutFilterFunc(d.PutConflictPolicy)) + if err != nil { + return nil, nil, err + } + tmpSrcObjs, err := d.getAllObjs(ctx, srcObj, getWriteAndPutFilterFunc(AllRWP)) + if err != nil { + return nil, nil, err + } + if len(tmpSrcObjs) < len(dstObjs) { + return nil, nil, ErrNotEnoughSrcObjs + } + dstStorageMap := make(map[string][]model.Obj) + allocatingDst := make(map[model.Obj]struct{}) + for _, o := range dstObjs { + storage, e := fs.GetStorage(o.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + return nil, nil, errors.WithMessagef(e, "cannot move to virtual path [%s]", o.GetPath()) + } + mp := storage.GetStorage().MountPath + dstStorageMap[mp] = append(dstStorageMap[mp], o) + allocatingDst[o] = struct{}{} + } + srcObjs := make(BalancedObjs, 0, len(tmpSrcObjs)) + restSrcObjs := make(BalancedObjs, 0, len(tmpSrcObjs)-len(dstObjs)) + for _, src := range tmpSrcObjs { + storage, e := fs.GetStorage(src.GetPath(), &fs.GetStoragesArgs{}) + if e != nil { + continue + } + mp := storage.GetStorage().MountPath + if tmp, ok := dstStorageMap[mp]; ok { + dst := tmp[0] + if len(tmp) == 1 { + delete(dstStorageMap, mp) + } else { + dstStorageMap[mp] = tmp[1:] + } + dstObjs[len(srcObjs)] = dst + srcObjs = append(srcObjs, src) + delete(allocatingDst, dst) + } else { + restSrcObjs = append(restSrcObjs, src) + } } - return reqPath, nil + dstObjs = dstObjs[:len(srcObjs)] + // len(restSrcObjs) >= len(allocatingDst) + srcObjs = append(srcObjs, restSrcObjs...) + for dst := range allocatingDst { + dstObjs = append(dstObjs, dst) + } + return srcObjs, dstObjs, nil } -func (d *Alias) getArchiveMeta(ctx context.Context, dst, sub string, args model.ArchiveArgs) (model.ArchiveMeta, error) { - reqPath := stdpath.Join(dst, sub) +func (d *Alias) getArchiveMeta(ctx context.Context, reqPath string, args model.ArchiveArgs) (model.ArchiveMeta, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err @@ -158,8 +458,7 @@ func (d *Alias) getArchiveMeta(ctx context.Context, dst, sub string, args model. return nil, errs.NotImplement } -func (d *Alias) listArchive(ctx context.Context, dst, sub string, args model.ArchiveInnerArgs) ([]model.Obj, error) { - reqPath := stdpath.Join(dst, sub) +func (d *Alias) listArchive(ctx context.Context, reqPath string, args model.ArchiveInnerArgs) ([]model.Obj, error) { storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err @@ -191,3 +490,43 @@ func (d *Alias) extract(ctx context.Context, reqPath string, args model.ArchiveI link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) return link, err } + +func getAllSort(dirs []model.Obj) model.Sort { + ret := model.Sort{} + noSort := false + noExtractFolder := false + for _, dir := range dirs { + if dir == nil { + continue + } + storage, err := fs.GetStorage(dir.GetPath(), &fs.GetStoragesArgs{}) + if err != nil { + continue + } + if !noSort && storage.GetStorage().OrderBy != "" { + if ret.OrderBy == "" { + ret.OrderBy = storage.GetStorage().OrderBy + ret.OrderDirection = storage.GetStorage().OrderDirection + if ret.OrderDirection == "" { + ret.OrderDirection = "asc" + } + } else if ret.OrderBy != storage.GetStorage().OrderBy || ret.OrderDirection != storage.GetStorage().OrderDirection { + ret.OrderBy = "" + ret.OrderDirection = "" + noSort = true + } + } + if !noExtractFolder && storage.GetStorage().ExtractFolder != "" { + if ret.ExtractFolder == "" { + ret.ExtractFolder = storage.GetStorage().ExtractFolder + } else if ret.ExtractFolder != storage.GetStorage().ExtractFolder { + ret.ExtractFolder = "" + noExtractFolder = true + } + } + if noSort && noExtractFolder { + break + } + } + return ret +} diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go new file mode 100644 index 000000000..a9fc44fcb --- /dev/null +++ b/drivers/alist_v3/driver.go @@ -0,0 +1,379 @@ +package alist_v3 + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +type AListV3 struct { + model.Storage + Addition +} + +func (d *AListV3) Config() driver.Config { + return config +} + +func (d *AListV3) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *AListV3) Init(ctx context.Context) error { + d.Addition.Address = strings.TrimSuffix(d.Addition.Address, "/") + var resp common.Resp[MeResp] + _, _, err := d.request("/me", http.MethodGet, func(req *resty.Request) { + req.SetResult(&resp) + }) + if err != nil { + return err + } + // if the username is not empty and the username is not the same as the current username, then login again + if d.Username != resp.Data.Username { + err = d.login() + if err != nil { + return err + } + } + // re-get the user info + _, _, err = d.request("/me", http.MethodGet, func(req *resty.Request) { + req.SetResult(&resp) + }) + if err != nil { + return err + } + if utils.SliceContains(resp.Data.Role, model.GUEST) { + u := d.Address + "/api/public/settings" + res, err := base.RestyClient.R().Get(u) + if err != nil { + return err + } + allowMounted := utils.Json.Get(res.Body(), "data", conf.AllowMounted).ToString() == "true" + if !allowMounted { + return fmt.Errorf("the site does not allow mounted") + } + } + return err +} + +func (d *AListV3) Drop(ctx context.Context) error { + return nil +} + +func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var resp common.Resp[FsListResp] + _, _, err := d.request("/fs/list", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ListReq{ + PageReq: model.PageReq{ + Page: 1, + PerPage: 0, + }, + Path: dir.GetPath(), + Password: d.MetaPassword, + Refresh: false, + }) + }) + if err != nil { + return nil, err + } + var files []model.Obj + for _, f := range resp.Data.Content { + file := model.ObjThumb{ + Object: model.Object{ + Name: f.Name, + Modified: f.Modified, + Ctime: f.Created, + Size: f.Size, + IsFolder: f.IsDir, + HashInfo: utils.FromString(f.HashInfo), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, + } + files = append(files, &file) + } + return files, nil +} + +func (d *AListV3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var resp common.Resp[FsGetResp] + headers := map[string]string{ + "User-Agent": base.UserAgent, + } + // if PassUAToUpsteam is true, then pass the user-agent to the upstream + if d.PassUAToUpsteam { + userAgent := args.Header.Get("user-agent") + if userAgent != "" { + headers["User-Agent"] = userAgent + } + } + // if PassIPToUpsteam is true, then pass the ip address to the upstream + if d.PassIPToUpsteam { + ip := args.IP + if ip != "" { + headers["X-Forwarded-For"] = ip + headers["X-Real-Ip"] = ip + } + } + _, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(FsGetReq{ + Path: file.GetPath(), + Password: d.MetaPassword, + }).SetHeaders(headers) + }) + if err != nil { + return nil, err + } + return &model.Link{ + URL: resp.Data.RawURL, + }, nil +} + +func (d *AListV3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + _, _, err := d.request("/fs/mkdir", http.MethodPost, func(req *resty.Request) { + req.SetBody(MkdirOrLinkReq{ + Path: path.Join(parentDir.GetPath(), dirName), + }) + }) + return err +} + +func (d *AListV3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + _, _, err := d.request("/fs/move", http.MethodPost, func(req *resty.Request) { + req.SetBody(MoveCopyReq{ + SrcDir: path.Dir(srcObj.GetPath()), + DstDir: dstDir.GetPath(), + Names: []string{srcObj.GetName()}, + }) + }) + return err +} + +func (d *AListV3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + _, _, err := d.request("/fs/rename", http.MethodPost, func(req *resty.Request) { + req.SetBody(RenameReq{ + Path: srcObj.GetPath(), + Name: newName, + }) + }) + return err +} + +func (d *AListV3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + _, _, err := d.request("/fs/copy", http.MethodPost, func(req *resty.Request) { + req.SetBody(MoveCopyReq{ + SrcDir: path.Dir(srcObj.GetPath()), + DstDir: dstDir.GetPath(), + Names: []string{srcObj.GetName()}, + }) + }) + return err +} + +func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { + _, _, err := d.request("/fs/remove", http.MethodPost, func(req *resty.Request) { + req.SetBody(RemoveReq{ + Dir: path.Dir(obj.GetPath()), + Names: []string{obj.GetName()}, + }) + }) + return err +} + +func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", reader) + if err != nil { + return err + } + req.Header.Set("Authorization", d.Token) + req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) + req.Header.Set("Password", d.MetaPassword) + if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { + req.Header.Set("X-File-Md5", md5) + } + if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { + req.Header.Set("X-File-Sha1", sha1) + } + if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { + req.Header.Set("X-File-Sha256", sha256) + } + + req.ContentLength = s.GetSize() + // client := base.NewHttpClient() + // client.Timeout = time.Hour * 6 + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + + bytes, err := io.ReadAll(res.Body) + if err != nil { + return err + } + log.Debugf("[openlist] response body: %s", string(bytes)) + if res.StatusCode >= 400 { + return fmt.Errorf("request failed, status: %s", res.Status) + } + code := utils.Json.Get(bytes, "code").ToInt() + if code != 200 { + if code == 401 || code == 403 { + err = d.login() + if err != nil { + return err + } + } + return fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(bytes, "message").ToString()) + } + return nil +} + +func (d *AListV3) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotImplement + } + var resp common.Resp[ArchiveMetaResp] + _, code, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }) + }) + if code == 202 { + return nil, errs.WrongArchivePassword + } + if err != nil { + return nil, err + } + var tree []model.ObjTree + if resp.Data.Content != nil { + tree = make([]model.ObjTree, 0, len(resp.Data.Content)) + for _, content := range resp.Data.Content { + tree = append(tree, &content) + } + } + return &model.ArchiveMetaInfo{ + Comment: resp.Data.Comment, + Encrypted: resp.Data.Encrypted, + Tree: tree, + }, nil +} + +func (d *AListV3) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotImplement + } + var resp common.Resp[ArchiveListResp] + _, code, err := d.request("/fs/archive/list", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveListReq{ + ArchiveMetaReq: ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }, + PageReq: model.PageReq{ + Page: 1, + PerPage: 0, + }, + InnerPath: args.InnerPath, + }) + }) + if code == 202 { + return nil, errs.WrongArchivePassword + } + if err != nil { + return nil, err + } + var files []model.Obj + for _, f := range resp.Data.Content { + file := model.ObjThumb{ + Object: model.Object{ + Name: f.Name, + Modified: f.Modified, + Ctime: f.Created, + Size: f.Size, + IsFolder: f.IsDir, + HashInfo: utils.FromString(f.HashInfo), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, + } + files = append(files, &file) + } + return files, nil +} + +func (d *AListV3) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotSupport + } + var resp common.Resp[ArchiveMetaResp] + _, _, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }) + }) + if err != nil { + return nil, err + } + return &model.Link{ + URL: fmt.Sprintf("%s?inner=%s&pass=%s&sign=%s", + resp.Data.RawURL, + utils.EncodePath(args.InnerPath, true), + url.QueryEscape(args.Password), + resp.Data.Sign), + }, nil +} + +func (d *AListV3) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { + if !d.ForwardArchiveReq { + return errs.NotImplement + } + dir, name := path.Split(srcObj.GetPath()) + _, _, err := d.request("/fs/archive/decompress", http.MethodPost, func(req *resty.Request) { + req.SetBody(DecompressReq{ + ArchivePass: args.Password, + CacheFull: args.CacheFull, + DstDir: dstDir.GetPath(), + InnerPath: args.InnerPath, + Name: []string{name}, + PutIntoNewDir: args.PutIntoNewDir, + SrcDir: dir, + }) + }) + return err +} + +func (d *AListV3) ResolveLinkCacheMode(_ string) driver.LinkCacheMode { + var mode driver.LinkCacheMode + if d.PassIPToUpsteam { + mode |= driver.LinkCacheIP + } + if d.PassUAToUpsteam { + mode |= driver.LinkCacheUA + } + return mode +} + +var _ driver.Driver = (*AListV3)(nil) diff --git a/drivers/alist_v3/meta.go b/drivers/alist_v3/meta.go new file mode 100644 index 000000000..2e5b83f8e --- /dev/null +++ b/drivers/alist_v3/meta.go @@ -0,0 +1,32 @@ +package alist_v3 + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + Address string `json:"url" required:"true"` + MetaPassword string `json:"meta_password"` + Username string `json:"username"` + Password string `json:"password"` + Token string `json:"token"` + PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` + PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` + ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` +} + +var config = driver.Config{ + Name: "AList V3", + LocalSort: true, + DefaultRoot: "/", + ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheAuto, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &AListV3{} + }) +} diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go new file mode 100644 index 000000000..527de7799 --- /dev/null +++ b/drivers/alist_v3/types.go @@ -0,0 +1,182 @@ +package alist_v3 + +import ( + "encoding/json" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type ListReq struct { + model.PageReq + Path string `json:"path" form:"path"` + Password string `json:"password" form:"password"` + Refresh bool `json:"refresh"` +} + +type ObjResp struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfo string `json:"hashinfo"` +} + +type FsListResp struct { + Content []ObjResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Write bool `json:"write"` + Provider string `json:"provider"` +} + +type FsGetReq struct { + Path string `json:"path" form:"path"` + Password string `json:"password" form:"password"` +} + +type FsGetResp struct { + ObjResp + RawURL string `json:"raw_url"` + Readme string `json:"readme"` + Provider string `json:"provider"` + Related []ObjResp `json:"related"` +} + +type MkdirOrLinkReq struct { + Path string `json:"path" form:"path"` +} + +type MoveCopyReq struct { + SrcDir string `json:"src_dir"` + DstDir string `json:"dst_dir"` + Names []string `json:"names"` +} + +type RenameReq struct { + Path string `json:"path"` + Name string `json:"name"` +} + +type RemoveReq struct { + Dir string `json:"dir"` + Names []string `json:"names"` +} + +type LoginResp struct { + Token string `json:"token"` +} + +type MeResp struct { + Id int `json:"id"` + Username string `json:"username"` + Password string `json:"password"` + BasePath string `json:"base_path"` + Role IntSlice `json:"role"` + Disabled bool `json:"disabled"` + Permission int `json:"permission"` + SsoId string `json:"sso_id"` + Otp bool `json:"otp"` +} + +type IntSlice []int + +func (s *IntSlice) UnmarshalJSON(b []byte) error { + var i int + if json.Unmarshal(b, &i) == nil { + *s = []int{i} + return nil + } + return json.Unmarshal(b, (*[]int)(s)) +} + +type ArchiveMetaReq struct { + ArchivePass string `json:"archive_pass"` + Password string `json:"password"` + Path string `json:"path"` + Refresh bool `json:"refresh"` +} + +type TreeResp struct { + ObjResp + Children []TreeResp `json:"children"` + hashCache *utils.HashInfo +} + +func (t *TreeResp) GetSize() int64 { + return t.Size +} + +func (t *TreeResp) GetName() string { + return t.Name +} + +func (t *TreeResp) ModTime() time.Time { + return t.Modified +} + +func (t *TreeResp) CreateTime() time.Time { + return t.Created +} + +func (t *TreeResp) IsDir() bool { + return t.ObjResp.IsDir +} + +func (t *TreeResp) GetHash() utils.HashInfo { + return utils.FromString(t.HashInfo) +} + +func (t *TreeResp) GetID() string { + return "" +} + +func (t *TreeResp) GetPath() string { + return "" +} + +func (t *TreeResp) GetChildren() []model.ObjTree { + ret := make([]model.ObjTree, 0, len(t.Children)) + for _, child := range t.Children { + ret = append(ret, &child) + } + return ret +} + +func (t *TreeResp) Thumb() string { + return t.ObjResp.Thumb +} + +type ArchiveMetaResp struct { + Comment string `json:"comment"` + Encrypted bool `json:"encrypted"` + Content []TreeResp `json:"content"` + RawURL string `json:"raw_url"` + Sign string `json:"sign"` +} + +type ArchiveListReq struct { + model.PageReq + ArchiveMetaReq + InnerPath string `json:"inner_path"` +} + +type ArchiveListResp struct { + Content []ObjResp `json:"content"` + Total int64 `json:"total"` +} + +type DecompressReq struct { + ArchivePass string `json:"archive_pass"` + CacheFull bool `json:"cache_full"` + DstDir string `json:"dst_dir"` + InnerPath string `json:"inner_path"` + Name []string `json:"name"` + PutIntoNewDir bool `json:"put_into_new_dir"` + SrcDir string `json:"src_dir"` +} diff --git a/drivers/alist_v3/util.go b/drivers/alist_v3/util.go new file mode 100644 index 000000000..821fe5465 --- /dev/null +++ b/drivers/alist_v3/util.go @@ -0,0 +1,65 @@ +package alist_v3 + +import ( + "fmt" + "net/http" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +func (d *AListV3) login() error { + if d.Username == "" { + return nil + } + var resp common.Resp[LoginResp] + _, _, err := d.request("/auth/login", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(base.Json{ + "username": d.Username, + "password": d.Password, + }) + }) + if err != nil { + return err + } + d.Token = resp.Data.Token + op.MustSaveDriverStorage(d) + return nil +} + +func (d *AListV3) request(api, method string, callback base.ReqCallback, retry ...bool) ([]byte, int, error) { + url := d.Address + "/api" + api + req := base.RestyClient.R() + req.SetHeader("Authorization", d.Token) + if callback != nil { + callback(req) + } + res, err := req.Execute(method, url) + if err != nil { + code := 0 + if res != nil { + code = res.StatusCode() + } + return nil, code, err + } + log.Debugf("[openlist] response body: %s", res.String()) + if res.StatusCode() >= 400 { + return nil, res.StatusCode(), fmt.Errorf("request failed, status: %s", res.Status()) + } + code := utils.Json.Get(res.Body(), "code").ToInt() + if code != 200 { + if (code == 401 || code == 403) && !utils.IsBool(retry...) { + err = d.login() + if err != nil { + return nil, code, err + } + return d.request(api, method, callback, true) + } + return nil, code, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) + } + return res.Body(), 200, nil +} diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index 6fd9bff26..3e71e8c6e 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -334,12 +334,12 @@ func (d *AliDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error if err != nil { return nil, err } - used := utils.Json.Get(res, "drive_used_size").ToUint64() - total := utils.Json.Get(res, "drive_total_size").ToUint64() + used := utils.Json.Get(res, "drive_used_size").ToInt64() + total := utils.Json.Get(res, "drive_total_size").ToInt64() return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index 20dd92dca..ee93b3303 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -77,7 +77,6 @@ func (d *AliyundriveOpen) GetRoot(ctx context.Context) (model.Obj, error) { ID: d.RootFolderID, Path: "/", Name: "root", - Size: 0, Modified: d.Modified, IsFolder: true, }, nil @@ -296,12 +295,12 @@ func (d *AliyundriveOpen) GetDetails(ctx context.Context) (*model.StorageDetails if err != nil { return nil, err } - total := utils.Json.Get(res, "personal_space_info", "total_size").ToUint64() - used := utils.Json.Get(res, "personal_space_info", "used_size").ToUint64() + total := utils.Json.Get(res, "personal_space_info", "total_size").ToInt64() + used := utils.Json.Get(res, "personal_space_info", "used_size").ToInt64() return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 9114997c2..a4a6c1de1 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -242,11 +242,11 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m if err != nil { return nil, err } - rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd) err = retry.Do(func() error { rd.Seek(0, io.SeekStart) - return d.uploadPart(ctx, rateLimitedRd, createResp.PartInfoList[i]) + return d.uploadPart(ctx, driver.NewLimitedUploadStream(ctx, rd), createResp.PartInfoList[i]) }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) diff --git a/drivers/aliyundrive_open/util.go b/drivers/aliyundrive_open/util.go index d171a7909..22112332a 100644 --- a/drivers/aliyundrive_open/util.go +++ b/drivers/aliyundrive_open/util.go @@ -38,7 +38,6 @@ func (d *AliyundriveOpen) _refreshToken(ctx context.Context) (string, string, er return "", "", err } _, err = base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, diff --git a/drivers/all.go b/drivers/all.go index 8184f9871..de5dfa41b 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -13,9 +13,11 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/189_tv" _ "github.com/OpenListTeam/OpenList/v4/drivers/189pc" _ "github.com/OpenListTeam/OpenList/v4/drivers/alias" + _ "github.com/OpenListTeam/OpenList/v4/drivers/alist_v3" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/aliyundrive_share" + _ "github.com/OpenListTeam/OpenList/v4/drivers/autoindex" _ "github.com/OpenListTeam/OpenList/v4/drivers/azure_blob" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_netdisk" _ "github.com/OpenListTeam/OpenList/v4/drivers/baidu_photo" @@ -36,6 +38,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/google_drive" _ "github.com/OpenListTeam/OpenList/v4/drivers/google_photo" _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud" + _ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/ilanzou" _ "github.com/OpenListTeam/OpenList/v4/drivers/ipfs_api" _ "github.com/OpenListTeam/OpenList/v4/drivers/kodbox" @@ -55,9 +58,11 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/openlist_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" _ "github.com/OpenListTeam/OpenList/v4/drivers/pikpak_share" + _ "github.com/OpenListTeam/OpenList/v4/drivers/proton_drive" _ "github.com/OpenListTeam/OpenList/v4/drivers/quark_open" _ "github.com/OpenListTeam/OpenList/v4/drivers/quark_uc" _ "github.com/OpenListTeam/OpenList/v4/drivers/quark_uc_tv" + _ "github.com/OpenListTeam/OpenList/v4/drivers/qihoo360" _ "github.com/OpenListTeam/OpenList/v4/drivers/s3" _ "github.com/OpenListTeam/OpenList/v4/drivers/seafile" _ "github.com/OpenListTeam/OpenList/v4/drivers/sftp" @@ -75,11 +80,11 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/webdav" _ "github.com/OpenListTeam/OpenList/v4/drivers/weiyun" _ "github.com/OpenListTeam/OpenList/v4/drivers/wopan" + _ "github.com/OpenListTeam/OpenList/v4/drivers/wps" _ "github.com/OpenListTeam/OpenList/v4/drivers/yandex_disk" ) // All do nothing,just for import // same as _ import func All() { - } diff --git a/drivers/autoindex/driver.go b/drivers/autoindex/driver.go new file mode 100644 index 000000000..5758ec6cd --- /dev/null +++ b/drivers/autoindex/driver.go @@ -0,0 +1,169 @@ +package autoindex + +import ( + "context" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/antchfx/htmlquery" + "github.com/antchfx/xpath" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type AutoIndex struct { + model.Storage + Addition + itemXPath *xpath.Expr + nameXPath *xpath.Expr + modifiedXPath *xpath.Expr + sizeXPath *xpath.Expr + ignores map[string]any +} + +func (d *AutoIndex) Config() driver.Config { + return config +} + +func (d *AutoIndex) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *AutoIndex) Init(ctx context.Context) error { + var err error + d.itemXPath, err = xpath.Compile(d.ItemXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Item XPath") + } + d.nameXPath, err = xpath.Compile(d.NameXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Name XPath") + } + if len(d.ModifiedXPath) > 0 { + d.modifiedXPath, err = xpath.Compile(d.ModifiedXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Modified XPath") + } + } + if len(d.SizeXPath) > 0 { + d.sizeXPath, err = xpath.Compile(d.SizeXPath) + if err != nil { + return errors.WithMessage(err, "failed to compile Size XPath") + } + } + ignores := strings.Split(d.IgnoreFileNames, "\n") + d.ignores = make(map[string]any, len(ignores)) + for _, i := range ignores { + i = strings.TrimSpace(i) + if len(i) == 0 { + continue + } + d.ignores[i] = struct{}{} + } + hasScheme := strings.Contains(d.URL, "://") + hasSuffix := strings.HasSuffix(d.URL, "/") + if !hasScheme || !hasSuffix { + if !hasSuffix { + d.URL = d.URL + "/" + } + if !hasScheme { + d.URL = "https://" + d.URL + } + op.MustSaveDriverStorage(d) + } + return nil +} + +func (d *AutoIndex) Drop(ctx context.Context) error { + return nil +} + +func (d *AutoIndex) GetRoot(ctx context.Context) (model.Obj, error) { + return &model.Object{ + Name: op.RootName, + Path: d.URL, + Modified: d.Modified, + Mask: model.Locked, + IsFolder: true, + }, nil +} + +func (d *AutoIndex) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + res, err := base.RestyClient.R(). + SetContext(ctx). + SetDoNotParseResponse(true). + Get(dir.GetPath()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to get url [%s]", dir.GetPath()) + } + defer res.RawResponse.Body.Close() + doc, err := htmlquery.Parse(res.RawBody()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to parse [%s]", dir.GetPath()) + } + itemsIter := d.itemXPath.Select(htmlquery.CreateXPathNavigator(doc)) + var objs []model.Obj + for itemsIter.MoveNext() { + nameFull, err := parseString(d.nameXPath.Evaluate(itemsIter.Current().Copy())) + if err != nil { + log.Warnf("skip invalid name evaluating result: %v", err) + continue + } + nameFull = strings.TrimSpace(nameFull) + name, isDir := strings.CutSuffix(nameFull, "/") + if _, ok := d.ignores[name]; ok { + continue + } + var size int64 = 0 + exact := false + modified := time.Now() + if d.sizeXPath != nil { + size, exact, err = parseSize(d.sizeXPath.Evaluate(itemsIter.Current().Copy())) + if err != nil { + log.Errorf("failed to parse size of %s: %v", name, err) + } + } + if d.modifiedXPath != nil { + modified, err = parseTime(d.modifiedXPath.Evaluate(itemsIter.Current().Copy()), d.ModifiedTimeFormat) + if err != nil { + log.Errorf("failed to parse modified time of %s: %v", name, err) + } + } + var o model.Obj = &model.Object{ + Name: name, + IsFolder: isDir, + Path: dir.GetPath() + nameFull, + Modified: modified, + Size: size, + } + if exact { + o = &exactSizeObj{Obj: o} + } + objs = append(objs, o) + } + return objs, nil +} + +func (d *AutoIndex) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if _, ok := file.(*exactSizeObj); ok || args.Redirect { + return &model.Link{URL: file.GetPath()}, nil + } + res, err := base.RestyClient.R(). + SetContext(ctx). + SetDoNotParseResponse(true). + Head(file.GetPath()) + if err != nil { + return nil, errors.WithMessagef(err, "failed to head [%s]", file.GetPath()) + } + _ = res.RawResponse.Body.Close() + return &model.Link{ + URL: file.GetPath(), + ContentLength: res.RawResponse.ContentLength, + }, nil +} + +var _ driver.Driver = (*AutoIndex)(nil) diff --git a/drivers/autoindex/meta.go b/drivers/autoindex/meta.go new file mode 100644 index 000000000..8ebae5924 --- /dev/null +++ b/drivers/autoindex/meta.go @@ -0,0 +1,29 @@ +package autoindex + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + URL string `json:"url" required:"true"` + ItemXPath string `json:"item_xpath" required:"true"` + NameXPath string `json:"name_xpath" required:"true"` + ModifiedXPath string `json:"modified_xpath"` + SizeXPath string `json:"size_xpath"` + IgnoreFileNames string `json:"ignore_file_names" type:"text" default:".\n..\nParent Directory\nUp"` + ModifiedTimeFormat string `json:"modified_time_format" default:"02-Jan-2006 15:04" help:"Must be based on the time point Mon Jan 2 15:04:05 -0700 MST 2006"` +} + +var config = driver.Config{ + Name: "AutoIndex", + LocalSort: true, + CheckStatus: true, + NoUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &AutoIndex{} + }) +} diff --git a/drivers/autoindex/types.go b/drivers/autoindex/types.go new file mode 100644 index 000000000..48ec396f5 --- /dev/null +++ b/drivers/autoindex/types.go @@ -0,0 +1,13 @@ +package autoindex + +import ( + "fmt" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +var ( + errEmptyEvaluateResult = fmt.Errorf("empty result") +) + +type exactSizeObj struct{ model.Obj } diff --git a/drivers/autoindex/util.go b/drivers/autoindex/util.go new file mode 100644 index 000000000..f5f36a736 --- /dev/null +++ b/drivers/autoindex/util.go @@ -0,0 +1,116 @@ +package autoindex + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/antchfx/xpath" + "github.com/pkg/errors" +) + +var units = map[string]int64{ + "": 1, + "b": 1, + "byte": 1, + "bytes": 1, + "k": 1 << 10, + "kb": 1 << 10, + "kib": 1 << 10, + "m": 1 << 20, + "mb": 1 << 20, + "mib": 1 << 20, + "g": 1 << 30, + "gb": 1 << 30, + "gib": 1 << 30, + "t": 1 << 40, + "tb": 1 << 40, + "tib": 1 << 40, + "p": 1 << 50, + "pb": 1 << 50, + "pib": 1 << 50, +} + +func splitUnit(s string) (string, string) { + for i := len(s) - 1; i >= 0; i-- { + if s[i] >= '0' && s[i] <= '9' { + return strings.TrimSpace(s[:i+1]), strings.TrimSpace(s[i+1:]) + } + } + return "", s +} + +func parseSize(a any) (int64, bool, error) { + // 第二个返回值exact表示大小是否精确 + if f, ok := a.(float64); ok { + return int64(f), false, nil + } + s, err := parseString(a) + if errors.Is(err, errEmptyEvaluateResult) { + // 可能是错误，也可能确实大小为0 + // 如果确实大小为0，大概率不会下载，exact返回false也不会有什么性能损失 + // 如果是错误，exact返回true会导致本地代理出错，综合来看返回false更好 + return 0, false, nil + } + if err != nil { + return 0, false, err + } + s = strings.TrimSpace(s) + if s == "-" { + return 0, false, nil + } + nbs, unit := splitUnit(s) + mul, ok := units[strings.ToLower(unit)] + exact := mul == 1 + if !ok { + mul = 1 + // 推测无单位，exact应为false + } + nb, err := strconv.ParseInt(nbs, 10, 64) + if err != nil { + fnb, err := strconv.ParseFloat(nbs, 64) + if err != nil { + return 0, false, fmt.Errorf("failed to convert %s to number", nbs) + } + nb = int64(fnb * float64(mul)) + exact = false + } else { + nb = nb * mul + } + return nb, exact, nil +} + +func parseString(res any) (string, error) { + if r, ok := res.(string); ok { + if len(r) == 0 { + return "", errEmptyEvaluateResult + } + return r, nil + } + n, ok := res.(*xpath.NodeIterator) + if !ok { + return "", fmt.Errorf("unsupported evaluating result") + } + if !n.MoveNext() { + return "", fmt.Errorf("no matched nodes") + } + ns := n.Current().Value() + if len(ns) == 0 { + return "", errEmptyEvaluateResult + } + return ns, nil +} + +func parseTime(res any, format string) (time.Time, error) { + s, err := parseString(res) + if err != nil { + return time.Now(), err + } + s = strings.TrimSpace(s) + t, err := time.Parse(format, s) + if err != nil { + return time.Now(), errors.WithMessagef(err, "failed to convert %s to time", s) + } + return t, nil +} diff --git a/drivers/autoindex/util_test.go b/drivers/autoindex/util_test.go new file mode 100644 index 000000000..ba743b943 --- /dev/null +++ b/drivers/autoindex/util_test.go @@ -0,0 +1,49 @@ +package autoindex + +import ( + "testing" +) + +type wantType struct { + v int64 + exact bool + error bool +} + +func TestParseSize(t *testing.T) { + tests := []struct { + input string + want wantType + }{ + {"100", wantType{100, true, false}}, + {"1k", wantType{1024, false, false}}, + {"1kb", wantType{1024, false, false}}, + {"1K", wantType{1024, false, false}}, // case insensitive + {"1.5m", wantType{1572864, false, false}}, // 1.5 * 1024^2 + {"500 bytes", wantType{500, true, false}}, + {"-", wantType{0, false, false}}, + {"", wantType{0, false, false}}, + {"abc", wantType{0, false, true}}, + {"1.5GB", wantType{1610612736, false, false}}, // 1.5 * 1024^3 + {"2t", wantType{2199023255552, false, false}}, // 2 * 1024^4 + {"1p", wantType{1125899906842624, false, false}}, // 1 * 1024^5 + {"0", wantType{0, true, false}}, + {" 100 ", wantType{100, true, false}}, // trimmed + {"100b", wantType{100, true, false}}, + {"1gib", wantType{1073741824, false, false}}, // 1024^3 + {"1z", wantType{1, false, false}}, // invalid unit, mul=1 + {"1.5", wantType{1, false, false}}, // float without unit, truncated + {"2.7k", wantType{2764, false, false}}, // 2.7 * 1024 truncated + {"1.0g", wantType{1073741824, false, false}}, // 1.0 * 1024^3 + {"invalid", wantType{0, false, true}}, + {"123xyz", wantType{123, false, false}}, // unit not found, mul=1 + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + got, exact, err := parseSize(tt.input) + if got != tt.want.v || exact != tt.want.exact || (err != nil) != tt.want.error { + t.Errorf("ParseSize(%q) = (%d, %t, %t), want (%d, %t, %t)", tt.input, got, exact, err != nil, tt.want.v, tt.want.exact, tt.want.error) + } + }) + } +} diff --git a/drivers/azure_blob/driver.go b/drivers/azure_blob/driver.go index 888637dc9..ddfe3ff68 100644 --- a/drivers/azure_blob/driver.go +++ b/drivers/azure_blob/driver.go @@ -25,12 +25,11 @@ type AzureBlob struct { Addition client *azblob.Client containerClient *container.Client - config driver.Config } // Config returns the driver configuration. func (d *AzureBlob) Config() driver.Config { - return d.config + return config } // GetAddition returns additional settings specific to Azure Blob Storage. diff --git a/drivers/azure_blob/meta.go b/drivers/azure_blob/meta.go index 3aae5044f..d675d4083 100644 --- a/drivers/azure_blob/meta.go +++ b/drivers/azure_blob/meta.go @@ -6,17 +6,13 @@ import ( ) type Addition struct { + driver.RootPath Endpoint string `json:"endpoint" required:"true" default:"https://.blob.core.windows.net/" help:"e.g. https://accountname.blob.core.windows.net/. The full endpoint URL for Azure Storage, including the unique storage account name (3 ~ 24 numbers and lowercase letters only)."` AccessKey string `json:"access_key" required:"true" help:"The access key for Azure Storage, used for authentication. https://learn.microsoft.com/azure/storage/common/storage-account-keys-manage"` ContainerName string `json:"container_name" required:"true" help:"The name of the container in Azure Storage (created in the Azure portal). https://learn.microsoft.com/azure/storage/blobs/blob-containers-portal"` SignURLExpire int `json:"sign_url_expire" type:"number" default:"4" help:"The expiration time for SAS URLs, in hours."` } -// implement GetRootId interface -func (r Addition) GetRootId() string { - return r.ContainerName -} - var config = driver.Config{ Name: "Azure Blob Storage", LocalSort: true, @@ -25,8 +21,6 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { - return &AzureBlob{ - config: config, - } + return &AzureBlob{} }) } diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 4f8e3446f..fe77aca38 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -1,15 +1,19 @@ package baidu_netdisk import ( + "bytes" "context" "crypto/md5" "encoding/hex" "errors" "io" + "mime/multipart" + "net/http" "net/url" "os" stdpath "path" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -17,6 +21,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" @@ -31,6 +36,8 @@ type BaiduNetdisk struct { vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) } +var ErrUploadIDExpired = errors.New("uploadid expired") + func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -41,18 +48,20 @@ func (d *BaiduNetdisk) GetAddition() driver.Additional { func (d *BaiduNetdisk) Init(ctx context.Context) error { d.uploadThread, _ = strconv.Atoi(d.UploadThread) - if d.uploadThread < 1 || d.uploadThread > 32 { - d.uploadThread, d.UploadThread = 3, "3" + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" } if _, err := url.Parse(d.UploadAPI); d.UploadAPI == "" || err != nil { - d.UploadAPI = "https://d.pcs.baidu.com" + d.UploadAPI = UPLOAD_FALLBACK_API } res, err := d.get("/xpan/nas", map[string]string{ "method": "uinfo", }, nil) - log.Debugf("[baidu] get uinfo: %s", string(res)) + log.Debugf("[baidu_netdisk] get uinfo: %s", string(res)) if err != nil { return err } @@ -75,9 +84,10 @@ func (d *BaiduNetdisk) List(ctx context.Context, dir model.Obj, args model.ListA } func (d *BaiduNetdisk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - if d.DownloadAPI == "crack" { + switch d.DownloadAPI { + case "crack": return d.linkCrack(file, args) - } else if d.DownloadAPI == "crack_video" { + case "crack_video": return d.linkCrackVideo(file, args) } return d.linkOfficial(file, args) @@ -179,6 +189,11 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo // **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间，而不是文件时间。 // 而实际上云盘存储的时间是文件时间，所以此处需要覆盖时间，保证缓存与云盘的数据一致 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // 百度网盘不允许上传空文件 + if stream.GetSize() < 1 { + return nil, ErrBaiduEmptyFilesNotAllowed + } + // rapid upload if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { return newObj, nil @@ -189,7 +204,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F tmpF *os.File err error ) - if _, ok := cache.(io.ReaderAt); !ok { + if cache == nil { tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err @@ -214,7 +229,6 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // cal md5 for first 256k data const SliceSize int64 = 256 * utils.KB - // cal md5 blockList := make([]string, 0, count) byteSize := sliceSize fileMd5H := md5.New() @@ -244,7 +258,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } if tmpF != nil { if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) + return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) } _, err = tmpF.Seek(0, io.SeekStart) if err != nil { @@ -258,31 +272,14 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 预上传 - // 尝试获取之前的进度 + // step.1 尝试读取已保存进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - params := map[string]string{ - "method": "precreate", - } - form := map[string]string{ - "path": path, - "size": strconv.FormatInt(streamSize, 10), - "isdir": "0", - "autoinit": "1", - "rtype": "3", - "block_list": blockListStr, - "content-md5": contentMd5, - "slice-md5": sliceMd5, - } - joinTime(form, ctime, mtime) - - log.Debugf("[baidu_netdisk] precreate data: %s", form) - _, err = d.postForm("/xpan/file", params, form, &precreateResp) + // 没有进度，走预上传 + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) if err != nil { return nil, err } - log.Debugf("%+v", precreateResp) if precreateResp.ReturnType == 2 { // rapid upload, since got md5 match from baidu server // 修复时间，具体原因见 Put 方法注释的 **注意** @@ -291,48 +288,95 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return fileToObj(precreateResp.File), nil } } + ensureUploadURL := func() { + if precreateResp.UploadURL != "" { + return + } + precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) + } + // step.2 上传分片 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(1), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) - - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break +uploadLoop: + for range 2 { + // 获取上传域名 + ensureUploadURL() + // 并发上传 + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(UPLOAD_RETRY_COUNT), + retry.Delay(UPLOAD_RETRY_WAIT_TIME), + retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), + retry.DelayType(retry.BackOffDelay), + retry.RetryIf(func(err error) bool { + return !errors.Is(err, ErrUploadIDExpired) + }), + retry.LastErrorOnly(true)) + + totalParts := len(precreateResp.BlockList) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) { + break + } + if partseq < 0 { + continue + } + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize + } + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": precreateResp.Uploadid, + "partseq": strconv.Itoa(partseq), + } + section := io.NewSectionReader(cache, offset, size) + err := d.uploadSlice(ctx, precreateResp.UploadURL, params, stream.GetName(), section) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + progress := float64(threadG.Success()+1) * 100 / float64(totalParts+1) + up(progress) + return nil + }) } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - byteSize = lastBlockSize + err = threadG.Wait() + if err == nil { + break uploadLoop } - threadG.Go(func(ctx context.Context) error { - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), + + // 保存进度（所有错误都会保存） + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + + if errors.Is(err, context.Canceled) { + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch") + // 重新 precreate（所有分片都要重传） + newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime) + if err2 != nil { + return nil, err2 } - err := d.uploadSlice(ctx, params, stream.GetName(), - driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) - if err != nil { - return err + if newPre.ReturnType == 2 { + return fileToObj(newPre.File), nil } - up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) - precreateResp.BlockList[i] = -1 - return nil - }) - } - if err = threadG.Wait(); err != nil { - // 如果属于用户主动取消，则保存上传进度 - if errors.Is(err, context.Canceled) { - precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + precreateResp = newPre + precreateResp.UploadURL = "" + // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + continue uploadLoop } return nil, err } + defer up(100) // step.3 创建文件 var newFile File @@ -343,23 +387,104 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = ctime newFile.Mtime = mtime + // 上传成功清理进度 + base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5) return fileToObj(newFile), nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { - res, err := base.RestyClient.R(). - SetContext(ctx). - SetQueryParams(params). - SetFileReader("file", fileName, file). - Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") +// precreate 执行预上传操作，支持首次上传和 uploadid 过期重试 +func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + params := map[string]string{"method": "precreate"} + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + } + + // 只有在首次上传时才包含 content-md5 和 slice-md5 + if contentMd5 != "" && sliceMd5 != "" { + form["content-md5"] = contentMd5 + form["slice-md5"] = sliceMd5 + } + + joinTime(form, ctime, mtime) + + var precreateResp PrecreateResp + _, err := d.postForm("/xpan/file", params, form, &precreateResp) + if err != nil { + return nil, err + } + + // 修复时间，具体原因见 Put 方法注释的 **注意** + if precreateResp.ReturnType == 2 { + precreateResp.File.Ctime = ctime + precreateResp.File.Mtime = mtime + } + + return &precreateResp, nil +} + +func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file *io.SectionReader) error { + b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) + mw := multipart.NewWriter(b) + _, err := mw.CreateFormFile("file", fileName) + if err != nil { + return err + } + headSize := b.Len() + err = mw.Close() if err != nil { return err } - log.Debugln(res.RawResponse.Status + res.String()) - errCode := utils.Json.Get(res.Body(), "error_code").ToInt() - errNo := utils.Json.Get(res.Body(), "errno").ToInt() + head := bytes.NewReader(b.Bytes()[:headSize]) + tail := bytes.NewReader(b.Bytes()[headSize:]) + rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) + if err != nil { + return err + } + query := req.URL.Query() + for k, v := range params { + query.Set(k, v) + } + req.URL.RawQuery = query.Encode() + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(b.Len()) + file.Size() + + client := net.NewHttpClient() + if d.UploadSliceTimeout > 0 { + client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) + } else { + client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT + } + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + b.Reset() + _, err = b.ReadFrom(resp.Body) + if err != nil { + return err + } + body := b.Bytes() + respStr := string(body) + log.Debugln(respStr) + lower := strings.ToLower(respStr) + // 合并 uploadid 过期检测逻辑 + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error in uploading to baidu, will retry. response=%s", res.String()) + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) } return nil } @@ -369,7 +494,7 @@ func (d *BaiduNetdisk) GetDetails(ctx context.Context) (*model.StorageDetails, e if err != nil { return nil, err } - return &model.StorageDetails{DiskUsage: *du}, nil + return &model.StorageDetails{DiskUsage: du}, nil } var _ driver.Driver = (*BaiduNetdisk)(nil) diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 642025911..3f3bed022 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -1,6 +1,8 @@ package baidu_netdisk import ( + "time" + "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) @@ -18,15 +20,27 @@ type Addition struct { AccessToken string RefreshToken string `json:"refresh_token" required:"true"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + UploadSliceTimeout int `json:"upload_timeout" type:"number" default:"60" help:"per-slice upload timeout in seconds"` UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + UseDynamicUploadAPI bool `json:"use_dynamic_upload_api" default:"true" help:"dynamically get upload api domain, when enabled, the 'Upload API' setting will be used as a fallback if failed to get"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"` OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` } +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 + UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) + DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 60 // 上传分片请求默认超时时间 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second * 1 + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 +) + var config = driver.Config{ Name: "BaiduNetdisk", DefaultRoot: "/", + PreferProxy: true, } func init() { diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index 0e9ee4432..2ea037b3f 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -1,12 +1,16 @@ package baidu_netdisk import ( + "errors" "path" "strconv" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +var ( + ErrBaiduEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu netdisk") ) type TokenErrResp struct { @@ -71,9 +75,7 @@ func fileToObj(f File) *model.ObjThumb { Modified: time.Unix(f.ServerMtime, 0), Ctime: time.Unix(f.ServerCtime, 0), IsFolder: f.Isdir == 1, - - // 直接获取的MD5是错误的 - HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), + // 百度API返回的MD5不可信，不使用HashInfo }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } @@ -188,13 +190,39 @@ type PrecreateResp struct { // return_type=2 File File `json:"info"` + + UploadURL string `json:"-"` // 保存断点续传对应的上传域名 +} + +type UploadServerResp struct { + BakServer []any `json:"bak_server"` + BakServers []struct { + Server string `json:"server"` + } `json:"bak_servers"` + ClientIP string `json:"client_ip"` + ErrorCode int `json:"error_code"` + ErrorMsg string `json:"error_msg"` + Expire int `json:"expire"` + Host string `json:"host"` + Newno string `json:"newno"` + QuicServer []any `json:"quic_server"` + QuicServers []struct { + Server string `json:"server"` + } `json:"quic_servers"` + RequestID int64 `json:"request_id"` + Server []any `json:"server"` + ServerTime int `json:"server_time"` + Servers []struct { + Server string `json:"server"` + } `json:"servers"` + Sl int `json:"sl"` } type QuotaResp struct { - Errno int `json:"errno"` - RequestId int64 `json:"request_id"` - Total uint64 `json:"total"` - Used uint64 `json:"used"` - //Free uint64 `json:"free"` + Errno int `json:"errno"` + RequestId int64 `json:"request_id"` + Total int64 `json:"total"` + Used int64 `json:"used"` + //FreeSpace uint64 `json:"free"` //Expire bool `json:"expire"` } diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 6ae911e95..0e27fb305 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -41,7 +41,6 @@ func (d *BaiduNetdisk) _refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, @@ -114,14 +113,14 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall errno := utils.Json.Get(res.Body(), "errno").ToInt() if errno != 0 { if utils.SliceContains([]int{111, -6}, errno) { - log.Info("refreshing baidu_netdisk token.") + log.Info("[baidu_netdisk] refreshing baidu_netdisk token.") err2 := d.refreshToken() if err2 != nil { return retry.Unrecoverable(err2) } } - if 31023 == errno && d.DownloadAPI == "crack_video" { + if errno == 31023 && d.DownloadAPI == "crack_video" { result = res.Body() return nil } @@ -153,7 +152,7 @@ func (d *BaiduNetdisk) postForm(pathname string, params map[string]string, form func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { start := 0 - limit := 200 + limit := 1000 params := map[string]string{ "method": "list", "dir": dir, @@ -169,7 +168,6 @@ func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { for { params["start"] = strconv.Itoa(start) params["limit"] = strconv.Itoa(limit) - start += limit var resp ListResp _, err := d.get("/xpan/file", params, &resp) if err != nil { @@ -188,6 +186,11 @@ func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { } else { res = append(res, resp.List...) } + + if len(resp.List) < limit { + break + } + start += limit } return res, nil } @@ -246,7 +249,7 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, _ model.LinkArgs) (*model.Link, func (d *BaiduNetdisk) linkCrackVideo(file model.Obj, _ model.LinkArgs) (*model.Link, error) { param := map[string]string{ "type": "VideoURL", - "path": fmt.Sprintf("%s", file.GetPath()), + "path": file.GetPath(), "fs_id": file.GetID(), "devuid": "0%1", "clienttype": "1", @@ -325,10 +328,10 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { // 非会员固定为 4MB if d.vipType == 0 { if d.CustomUploadPartSize != 0 { - log.Warnf("CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") + log.Warnf("[baidu_netdisk] CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") } if filesize > MaxSliceNum*DefaultSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return DefaultSliceSize @@ -336,17 +339,17 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { if d.CustomUploadPartSize != 0 { if d.CustomUploadPartSize < DefaultSliceSize { - log.Warnf("CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) return DefaultSliceSize } if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) return VipSliceSize } if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) return SVipSliceSize } @@ -376,24 +379,67 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { } if filesize > MaxSliceNum*maxSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return maxSliceSize } -func (d *BaiduNetdisk) quota(ctx context.Context) (*model.DiskUsage, error) { +func (d *BaiduNetdisk) quota(ctx context.Context) (model.DiskUsage, error) { var resp QuotaResp _, err := d.request("https://pan.baidu.com/api/quota", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) }, &resp) if err != nil { - return nil, err + return model.DiskUsage{}, err } - return &model.DiskUsage{ - TotalSpace: resp.Total, - FreeSpace: resp.Total - resp.Used, - }, nil + return model.DiskUsage{TotalSpace: resp.Total, UsedSpace: resp.Used}, nil +} + +// getUploadUrl 从开放平台获取上传域名/地址，并发请求会被合并，结果会在 uploadid 生命周期内复用。 +// 如果获取失败，则返回 Upload API设置项。 +func (d *BaiduNetdisk) getUploadUrl(path, uploadId string) string { + if !d.UseDynamicUploadAPI || uploadId == "" { + return d.UploadAPI + } + + uploadUrl, err := d.requestForUploadUrl(path, uploadId) + if err != nil { + return d.UploadAPI + } + return uploadUrl +} + +// requestForUploadUrl 请求获取上传地址。 +// 实测此接口不需要认证，传method和upload_version就行，不过还是按文档规范调用。 +// https://pan.baidu.com/union/doc/Mlvw5hfnr +func (d *BaiduNetdisk) requestForUploadUrl(path, uploadId string) (string, error) { + params := map[string]string{ + "method": "locateupload", + "appid": "250528", + "path": path, + "uploadid": uploadId, + "upload_version": "2.0", + } + apiUrl := "https://d.pcs.baidu.com/rest/2.0/pcs/file" + var resp UploadServerResp + _, err := d.request(apiUrl, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(params) + }, &resp) + if err != nil { + return "", err + } + // 应该是https开头的一个地址 + var uploadUrl string + if len(resp.Servers) > 0 { + uploadUrl = resp.Servers[0].Server + } else if len(resp.BakServers) > 0 { + uploadUrl = resp.BakServers[0].Server + } + if uploadUrl == "" { + return "", errors.New("upload URL is empty") + } + return uploadUrl, nil } // func encodeURIComponent(str string) string { diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index 00e36ee65..1d5d5105d 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -371,7 +371,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if err != nil { return err } - up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) + up(float64(threadG.Success()+1) * 100 / float64(len(precreateResp.BlockList)+1)) precreateResp.BlockList[i] = -1 return nil }) @@ -383,6 +383,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil } return nil, err } + defer up(100) fallthrough case 2: //step.4 创建文件 params["uploadid"] = precreateResp.UploadID diff --git a/drivers/baidu_photo/meta.go b/drivers/baidu_photo/meta.go index 91997c9d7..6c6ca5c3a 100644 --- a/drivers/baidu_photo/meta.go +++ b/drivers/baidu_photo/meta.go @@ -18,8 +18,9 @@ type Addition struct { } var config = driver.Config{ - Name: "BaiduPhoto", - LocalSort: true, + Name: "BaiduPhoto", + LocalSort: true, + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/base/client.go b/drivers/base/client.go index a91d62541..196b078da 100644 --- a/drivers/base/client.go +++ b/drivers/base/client.go @@ -15,9 +15,12 @@ var ( RestyClient *resty.Client HttpClient *http.Client ) -var UserAgent = "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0" + var DefaultTimeout = time.Second * 30 +const UserAgent = "Mozilla/5.0 (Macintosh; Apple macOS 26_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/142.0.0.0 OpenList/425.6.30" +const UserAgentNT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/142.0.0.0 OpenList/425.6.30" + func InitClient() { NoRedirectClient = resty.New().SetRedirectPolicy( resty.RedirectPolicyFunc(func(req *http.Request, via []*http.Request) error { @@ -25,6 +28,7 @@ func InitClient() { }), ).SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) NoRedirectClient.SetHeader("user-agent", UserAgent) + net.SetRestyProxyIfConfigured(NoRedirectClient) RestyClient = NewRestyClient() HttpClient = net.NewHttpClient() @@ -37,5 +41,7 @@ func NewRestyClient() *resty.Client { SetRetryResetReaders(true). SetTimeout(DefaultTimeout). SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) + + net.SetRestyProxyIfConfigured(client) return client } diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index cb12b29f1..dfd25d195 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -10,6 +10,7 @@ import ( "mime/multipart" "net/http" "net/url" + "strconv" "strings" "time" @@ -225,21 +226,18 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStr if resp.Result != 1 { return errors.New("get upload data error") } - body := &bytes.Buffer{} + body := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) writer := multipart.NewWriter(body) - filePart, err := writer.CreateFormFile("file", file.GetName()) - if err != nil { - return err - } - _, err = utils.CopyWithBuffer(filePart, file) + _, err = writer.CreateFormFile("file", file.GetName()) if err != nil { return err } + headSize := body.Len() err = writer.WriteField("_token", resp.Msg.Token) if err != nil { return err } - err = writer.WriteField("puid", fmt.Sprintf("%d", resp.Msg.Puid)) + err = writer.WriteField("puid", strconv.Itoa(resp.Msg.Puid)) if err != nil { fmt.Println("Error writing param2 to request body:", err) return err @@ -248,30 +246,34 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStr if err != nil { return err } + head := bytes.NewReader(body.Bytes()[:headSize]) + tail := bytes.NewReader(body.Bytes()[headSize:]) r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: &driver.SimpleReaderWithSize{ - Reader: body, - Size: int64(body.Len()), + Reader: io.MultiReader(head, file, tail), + Size: int64(body.Len()) + file.GetSize(), }, UpdateProgress: up, }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://pan-yz.chaoxing.com/upload", r) if err != nil { return err } req.Header.Set("Content-Type", writer.FormDataContentType()) - req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len())) + req.ContentLength = int64(body.Len()) + file.GetSize() resps, err := http.DefaultClient.Do(req) if err != nil { return err } defer resps.Body.Close() - bodys, err := io.ReadAll(resps.Body) + body.Reset() + _, err = body.ReadFrom(resps.Body) if err != nil { return err } var fileRsp UploadFileDataRsp - err = json.Unmarshal(bodys, &fileRsp) + err = json.Unmarshal(body.Bytes(), &fileRsp) if err != nil { return err } diff --git a/drivers/chaoxing/types.go b/drivers/chaoxing/types.go index 9e6e270d2..ca171c4d5 100644 --- a/drivers/chaoxing/types.go +++ b/drivers/chaoxing/types.go @@ -258,7 +258,7 @@ type UploadDoneParam struct { func fileToObj(f File) *model.Object { if len(f.Content.FolderName) > 0 { return &model.Object{ - ID: fmt.Sprintf("%d", f.ID), + ID: strconv.Itoa(f.ID), Name: f.Content.FolderName, Size: 0, Modified: time.UnixMilli(f.Inserttime), diff --git a/drivers/chaoxing/util.go b/drivers/chaoxing/util.go index 715c248a8..7e3067ce9 100644 --- a/drivers/chaoxing/util.go +++ b/drivers/chaoxing/util.go @@ -9,6 +9,7 @@ import ( "fmt" "mime/multipart" "net/http" + "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -172,7 +173,7 @@ func (d *ChaoXing) Login() (string, error) { return "", err } req.Header.Set("Content-Type", writer.FormDataContentType()) - req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len())) + req.Header.Set("Content-Length", strconv.Itoa(body.Len())) resp, err := http.DefaultClient.Do(req) if err != nil { return "", err diff --git a/drivers/chunk/driver.go b/drivers/chunk/driver.go index 572f9936b..b544391dd 100644 --- a/drivers/chunk/driver.go +++ b/drivers/chunk/driver.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" @@ -52,14 +53,11 @@ func (d *Chunk) Drop(ctx context.Context) error { return nil } +func (Addition) GetRootPath() string { + return "/" +} + func (d *Chunk) Get(ctx context.Context, path string) (model.Obj, error) { - if utils.PathEqual(path, "/") { - return &model.Object{ - Name: "Root", - IsFolder: true, - Path: "/", - }, nil - } remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(d.RemotePath) if err != nil { return nil, err @@ -272,17 +270,13 @@ func (d *Chunk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( // 检查0号块不等于-1 以支持空文件 // 如果块数量大于1 最后一块不可能为0 // 只检查中间块是否有0 - for i, l := 0, len(chunkFile.chunkSizes)-2; ; i++ { - if i == 0 { - if chunkFile.chunkSizes[i] == -1 { - return nil, fmt.Errorf("chunk part[%d] are missing", i) - } - } else if chunkFile.chunkSizes[i] == 0 { + if chunkFile.chunkSizes[0] == -1 { + return nil, fmt.Errorf("chunk part[%d] are missing", 0) + } + for i, l := 1, len(chunkFile.chunkSizes)-1; i < l; i++ { + if chunkFile.chunkSizes[i] == 0 { return nil, fmt.Errorf("chunk part[%d] are missing", i) } - if i >= l { - break - } } fileSize := chunkFile.GetSize() mergedRrf := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { @@ -429,9 +423,10 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream UpdateProgress: up, } dst := stdpath.Join(remoteActualPath, dstDir.GetPath(), d.ChunkPrefix+file.GetName()) + skipHookCtx := context.WithValue(ctx, conf.SkipHookKey, struct{}{}) if d.StoreHash { for ht, value := range file.GetHash().All() { - _ = op.Put(ctx, remoteStorage, dst, &stream.FileStream{ + _ = op.Put(skipHookCtx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: fmt.Sprintf("hash_%s_%s%s", ht.Name, value, d.CustomExt), Size: 1, @@ -439,7 +434,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, Mimetype: "application/octet-stream", Reader: bytes.NewReader([]byte{0}), // 兼容不支持空文件的驱动 - }, nil, true) + }, nil) } } fullPartCount := int(file.GetSize() / d.PartSize) @@ -450,7 +445,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream } partIndex := 0 for partIndex < fullPartCount { - err = op.Put(ctx, remoteStorage, dst, &stream.FileStream{ + err = op.Put(skipHookCtx, remoteStorage, dst, &stream.FileStream{ Obj: &model.Object{ Name: d.getPartName(partIndex), Size: d.PartSize, @@ -458,7 +453,7 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream }, Mimetype: file.GetMimetype(), Reader: io.LimitReader(upReader, d.PartSize), - }, nil, true) + }, nil) if err != nil { _ = op.Remove(ctx, remoteStorage, dst) return err diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 41444c023..55462be7e 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -73,6 +73,7 @@ func (d *Cloudreve) List(ctx context.Context, dir model.Obj, args model.ListArgs } src.Size = dprop.Size } + src.Path = path.Join(dir.GetPath(), src.Name) return objectToObj(src, thumb), nil }) } @@ -203,6 +204,17 @@ func (d *Cloudreve) create(ctx context.Context, dir model.Obj, file model.Obj) e }, nil) } +func (d *Cloudreve) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var r StorageDetails + d.request(http.MethodGet, "/user/storage", nil, &r) + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: r.Total, + UsedSpace: r.Used, + }, + }, nil +} + //func (d *Cloudreve) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index c9f89b2b3..db545c7a9 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -59,6 +59,7 @@ func objectToObj(f Object, t model.Thumbnail) *model.ObjThumb { Size: int64(f.Size), Modified: f.Date, IsFolder: f.Type == "dir", + Path: f.Path, }, Thumbnail: t, } @@ -68,3 +69,9 @@ type Config struct { LoginCaptcha bool `json:"loginCaptcha"` CaptchaType string `json:"captcha_type"` } + +type StorageDetails struct { + Used int64 `json:"used"` + Free int64 `json:"free"` + Total int64 `json:"total"` +} diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index c9894b3a1..6d5e5aae1 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -291,6 +291,7 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U } return nil }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), @@ -351,7 +352,9 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u default: return nil } - }, retry.Attempts(3), + }, + retry.Context(ctx), + retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) @@ -414,7 +417,9 @@ func (d *Cloudreve) upS3(ctx context.Context, stream model.FileStreamer, u Uploa etags = append(etags, etag) return nil } - }, retry.Attempts(3), + }, + retry.Context(ctx), + retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) diff --git a/drivers/cloudreve_v4/driver.go b/drivers/cloudreve_v4/driver.go index 180ca1bdd..cd5cf1b3b 100644 --- a/drivers/cloudreve_v4/driver.go +++ b/drivers/cloudreve_v4/driver.go @@ -129,20 +129,23 @@ func (d *CloudreveV4) List(ctx context.Context, dir model.Obj, args model.ListAr } } return &model.ObjThumb{ - Object: model.Object{ - ID: src.ID, - Path: src.Path, - Name: src.Name, - Size: src.Size, - Modified: src.UpdatedAt, - Ctime: src.CreatedAt, - IsFolder: src.Type == 1, - }, + Object: *fileToObject(&src), Thumbnail: thumb, }, nil }) } +func (d *CloudreveV4) Get(ctx context.Context, path string) (model.Obj, error) { + var info File + err := d.request(http.MethodGet, "/file/info", func(req *resty.Request) { + req.SetQueryParam("uri", d.RootFolderPath+path) + }, &info) + if err != nil { + return nil, err + } + return fileToObject(&info), nil +} + func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var url FileUrlResp err := d.request(http.MethodPost, "/file/url", func(req *resty.Request) { @@ -299,7 +302,9 @@ func (d *CloudreveV4) Put(ctx context.Context, dstDir model.Obj, file model.File case "onedrive": err = d.upOneDrive(ctx, file, u, up) case "s3": - err = d.upS3(ctx, file, u, up) + err = d.upS3(ctx, file, u, up, "s3") + case "ks3": + err = d.upS3(ctx, file, u, up, "ks3") default: return errs.NotImplement } @@ -351,7 +356,7 @@ func (d *CloudreveV4) GetDetails(ctx context.Context) (*model.StorageDetails, er return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: r.Total, - FreeSpace: r.Total - r.Used, + UsedSpace: r.Used, }, }, nil } diff --git a/drivers/cloudreve_v4/types.go b/drivers/cloudreve_v4/types.go index a10f9fe1c..b67cfc862 100644 --- a/drivers/cloudreve_v4/types.go +++ b/drivers/cloudreve_v4/types.go @@ -122,6 +122,18 @@ type File struct { PrimaryEntity string `json:"primary_entity"` } +func fileToObject(f *File) *model.Object { + return &model.Object{ + ID: f.ID, + Path: f.Path, + Name: f.Name, + Size: f.Size, + Modified: f.UpdatedAt, + Ctime: f.CreatedAt, + IsFolder: f.Type == 1, + } +} + type StoragePolicy struct { ID string `json:"id"` Name string `json:"name"` @@ -206,7 +218,7 @@ type FolderSummaryResp struct { } type CapacityResp struct { - Total uint64 `json:"total"` - Used uint64 `json:"used"` + Total int64 `json:"total"` + Used int64 `json:"used"` // StoragePackTotal uint64 `json:"storage_pack_total"` } diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go index 200dfbb69..f8fe5f269 100644 --- a/drivers/cloudreve_v4/util.go +++ b/drivers/cloudreve_v4/util.go @@ -16,6 +16,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -30,6 +31,7 @@ import ( const ( CodeLoginRequired = http.StatusUnauthorized + CodePathNotExist = 40016 // Path not exist CodeCredentialInvalid = 40020 // Failed to issue token ) @@ -101,6 +103,9 @@ func (d *CloudreveV4) _request(method string, path string, callback base.ReqCall if r.Code == CodeCredentialInvalid { return ErrorIssueToken } + if r.Code == CodePathNotExist { + return errs.ObjectNotFound + } return fmt.Errorf("%d: %s", r.Code, r.Msg) } @@ -447,7 +452,9 @@ func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u F return errors.New(up.Msg) } return nil - }, retry.Attempts(3), + }, + retry.Context(ctx), + retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) @@ -508,7 +515,9 @@ func (d *CloudreveV4) upOneDrive(ctx context.Context, file model.FileStreamer, u default: return nil } - }, retry.Attempts(3), + }, + retry.Context(ctx), + retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), ) @@ -525,7 +534,7 @@ func (d *CloudreveV4) upOneDrive(ctx context.Context, file model.FileStreamer, u }, nil) } -func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { +func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress, s3Type string) error { DEFAULT := int64(u.ChunkSize) ss, err := stream.NewStreamSectionReader(file, int(DEFAULT), &up) if err != nil { @@ -556,6 +565,9 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU } req.ContentLength = byteSize req.Header.Set("User-Agent", d.getUA()) + if s3Type == "ks3" { + req.Header.Set("Content-Type", "application/octet-stream") + } res, err := base.HttpClient.Do(req) if err != nil { return err @@ -572,6 +584,7 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU return nil } }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), @@ -604,7 +617,11 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU if err != nil { return err } - req.Header.Set("Content-Type", "application/xml") + if s3Type == "ks3" { + req.Header.Set("Content-Type", "application/octet-stream") + } else { + req.Header.Set("Content-Type", "application/xml") + } req.Header.Set("User-Agent", d.getUA()) res, err := base.HttpClient.Do(req) if err != nil { @@ -617,7 +634,5 @@ func (d *CloudreveV4) upS3(ctx context.Context, file model.FileStreamer, u FileU } // 上传成功发送回调请求 - return d.request(http.MethodGet, "/callback/s3/"+u.SessionID+"/"+u.CallbackSecret, func(req *resty.Request) { - req.SetBody("{}") - }, nil) + return d.request(http.MethodGet, "/callback/"+s3Type+"/"+u.SessionID+"/"+u.CallbackSecret, nil, nil) } diff --git a/drivers/cnb_releases/driver.go b/drivers/cnb_releases/driver.go index d80e69584..09ec153c8 100644 --- a/drivers/cnb_releases/driver.go +++ b/drivers/cnb_releases/driver.go @@ -50,7 +50,8 @@ func (d *CnbReleases) Drop(ctx context.Context) error { } func (d *CnbReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - if dir.GetPath() == "/" { + dirID := dir.GetID() + if dirID == "" { // get all releases for root dir var resp ReleaseList @@ -75,36 +76,32 @@ func (d *CnbReleases) List(ctx context.Context, dir model.Obj, args model.ListAr IsFolder: true, }, nil }) - } else { - // get release info by release id - releaseID := dir.GetID() - if releaseID == "" { - return nil, errs.ObjectNotFound - } - var resp Release - err := d.Request(http.MethodGet, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { - req.SetPathParam("repo", d.Repo) - req.SetPathParam("release_id", releaseID) - }, &resp) - if err != nil { - return nil, err - } + } - return utils.SliceConvert(resp.Assets, func(src ReleaseAsset) (model.Obj, error) { - return &Object{ - Object: model.Object{ - ID: src.ID, - Path: src.Path, - Name: src.Name, - Size: src.Size, - Ctime: src.CreatedAt, - Modified: src.UpdatedAt, - IsFolder: false, - }, - ParentID: dir.GetID(), - }, nil - }) + var resp Release + err := d.Request(http.MethodGet, "/{repo}/-/releases/{release_id}", func(req *resty.Request) { + req.SetPathParam("repo", d.Repo) + req.SetPathParam("release_id", dirID) + }, &resp) + if err != nil { + return nil, err } + + return utils.SliceConvert(resp.Assets, func(src ReleaseAsset) (model.Obj, error) { + return &Object{ + Object: model.Object{ + ID: src.ID, + Path: src.Path, + Name: src.Name, + Size: src.Size, + Ctime: src.CreatedAt, + Modified: src.UpdatedAt, + IsFolder: false, + }, + ParentID: dirID, + }, nil + }) + } func (d *CnbReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -200,15 +197,20 @@ func (d *CnbReleases) Put(ctx context.Context, dstDir model.Obj, file model.File if err != nil { return err } - head := bytes.NewReader(b.Bytes()[:headSize]) tail := bytes.NewReader(b.Bytes()[headSize:]) - rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: io.MultiReader(head, file, tail), + Size: int64(b.Len()) + file.GetSize(), + }, + UpdateProgress: up, + }) // use net/http to upload file ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(resp.ExpiresInSec+1)*time.Second) defer cancel() - req, err := http.NewRequestWithContext(ctxWithTimeout, http.MethodPost, resp.UploadURL, rateLimitedRd) + req, err := http.NewRequestWithContext(ctxWithTimeout, http.MethodPost, resp.UploadURL, r) if err != nil { return err } diff --git a/drivers/cnb_releases/meta.go b/drivers/cnb_releases/meta.go index 2894d8a2b..7301861d3 100644 --- a/drivers/cnb_releases/meta.go +++ b/drivers/cnb_releases/meta.go @@ -6,7 +6,7 @@ import ( ) type Addition struct { - driver.RootPath + driver.RootID Repo string `json:"repo" type:"string" required:"true"` Token string `json:"token" type:"string" required:"true"` UseTagName bool `json:"use_tag_name" type:"bool" default:"false" help:"Use tag name instead of release name"` @@ -14,9 +14,8 @@ type Addition struct { } var config = driver.Config{ - Name: "CNB Releases", - LocalSort: true, - DefaultRoot: "/", + Name: "CNB Releases", + LocalSort: true, } func init() { diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index b00a2ea05..9e8b5c5c7 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -3,6 +3,7 @@ package crypt import ( "bytes" "context" + "errors" "fmt" "io" stdpath "path" @@ -29,8 +30,7 @@ import ( type Crypt struct { model.Storage Addition - cipher *rcCrypt.Cipher - remoteStorage driver.Driver + cipher *rcCrypt.Cipher } const obfuscatedPrefix = "___Obfuscated___" @@ -60,15 +60,7 @@ func (d *Crypt) Init(ctx context.Context) error { } d.FileNameEncoding = utils.GetNoneEmpty(d.FileNameEncoding, "base64") d.EncryptedSuffix = utils.GetNoneEmpty(d.EncryptedSuffix, ".bin") - - op.MustSaveDriverStorage(d) - - // need remote storage exist - storage, err := fs.GetStorage(d.RemotePath, &fs.GetStoragesArgs{}) - if err != nil { - return fmt.Errorf("can't find remote storage: %w", err) - } - d.remoteStorage = storage + d.RemotePath = utils.FixAndCleanPath(d.RemotePath) p, _ := strings.CutPrefix(d.Password, obfuscatedPrefix) p2, _ := strings.CutPrefix(d.Salt, obfuscatedPrefix) @@ -108,150 +100,146 @@ func (d *Crypt) Drop(ctx context.Context) error { } func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - path := dir.GetPath() - // return d.list(ctx, d.RemotePath, path) - // remoteFull - - objs, err := fs.List(ctx, d.getPathForRemote(path, true), &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) + remoteFullPath := dir.GetPath() + objs, err := fs.List(ctx, remoteFullPath, &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) // the obj must implement the model.SetPath interface // return objs, err if err != nil { return nil, err } - var result []model.Obj + result := make([]model.Obj, 0, len(objs)) for _, obj := range objs { - if obj.IsDir() { - name, err := d.cipher.DecryptDirName(obj.GetName()) - if err != nil { - // filter illegal files - continue - } - if !d.ShowHidden && strings.HasPrefix(name, ".") { - continue - } - objRes := model.Object{ - Name: name, - Size: 0, - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - Ctime: obj.CreateTime(), - // discarding hash as it's encrypted - } - result = append(result, &objRes) - } else { - thumb, ok := model.GetThumb(obj) - size, err := d.cipher.DecryptedSize(obj.GetSize()) - if err != nil { - // filter illegal files - continue - } - name, err := d.cipher.DecryptFileName(obj.GetName()) - if err != nil { - // filter illegal files - continue - } - if !d.ShowHidden && strings.HasPrefix(name, ".") { - continue - } - objRes := model.Object{ - Name: name, - Size: size, - Modified: obj.ModTime(), - IsFolder: obj.IsDir(), - Ctime: obj.CreateTime(), - // discarding hash as it's encrypted - } - if d.Thumbnail && thumb == "" { - thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") - thumb = fmt.Sprintf("%s/d%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(thumbPath, true), - sign.Sign(thumbPath)) - } - if !ok && !d.Thumbnail { - result = append(result, &objRes) + size := obj.GetSize() + mask := model.GetObjMask(obj) + name := obj.GetName() + if mask&model.Virtual == 0 { + if obj.IsDir() { + name, err = d.cipher.DecryptDirName(model.UnwrapObjName(obj).GetName()) + if err != nil { + // filter illegal files + continue + } } else { - objWithThumb := model.ObjThumb{ - Object: objRes, - Thumbnail: model.Thumbnail{ - Thumbnail: thumb, - }, + size, err = d.cipher.DecryptedSize(size) + if err != nil { + // filter illegal files + continue + } + name, err = d.cipher.DecryptFileName(model.UnwrapObjName(obj).GetName()) + if err != nil { + // filter illegal files + continue } - result = append(result, &objWithThumb) } } + if !d.ShowHidden && strings.HasPrefix(name, ".") { + continue + } + objRes := &model.Object{ + Path: stdpath.Join(remoteFullPath, obj.GetName()), + Name: name, + Size: size, + Modified: obj.ModTime(), + IsFolder: obj.IsDir(), + Ctime: obj.CreateTime(), + Mask: mask &^ model.Temp, + // discarding hash as it's encrypted + } + if !d.Thumbnail || !strings.HasPrefix(args.ReqPath, "/") { + result = append(result, objRes) + continue + } + thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") + thumb := fmt.Sprintf("%s/d%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(thumbPath, true), + sign.Sign(thumbPath)) + result = append(result, &model.ObjThumb{ + Object: *objRes, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + }) } return result, nil } +func (a Addition) GetRootPath() string { + return a.RemotePath +} + func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) { - if utils.PathEqual(path, "/") { - return &model.Object{ - Name: "Root", - IsFolder: true, - Path: "/", - }, nil - } - remoteFullPath := "" - var remoteObj model.Obj - var err, err2 error firstTryIsFolder, secondTry := guessPath(path) - remoteFullPath = d.getPathForRemote(path, firstTryIsFolder) - remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) + remoteFullPath := stdpath.Join(d.RemotePath, d.encryptPath(path, firstTryIsFolder)) + remoteObj, err := fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) if err != nil { - if errs.IsObjectNotFound(err) && secondTry { + if errors.Is(err, errs.StorageNotFound) { + remoteFullPath = stdpath.Join(d.RemotePath, path) + remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) + if err != nil { + // 可能是 虚拟路径+开启文件夹加密：返回NotSupport让op.Get去尝试op.List查找 + return nil, errs.NotSupport + } + } else if secondTry && errs.IsObjectNotFound(err) { // try the opposite - remoteFullPath = d.getPathForRemote(path, !firstTryIsFolder) - remoteObj, err2 = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) - if err2 != nil { - return nil, err2 + remoteFullPath = stdpath.Join(d.RemotePath, d.encryptPath(path, !firstTryIsFolder)) + remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, err } } else { return nil, err } } - var size int64 = 0 - name := "" - if !remoteObj.IsDir() { - size, err = d.cipher.DecryptedSize(remoteObj.GetSize()) - if err != nil { - log.Warnf("DecryptedSize failed for %s ,will use original size, err:%s", path, err) - size = remoteObj.GetSize() - } - name, err = d.cipher.DecryptFileName(remoteObj.GetName()) - if err != nil { - log.Warnf("DecryptFileName failed for %s ,will use original name, err:%s", path, err) - name = remoteObj.GetName() - } - } else { - name, err = d.cipher.DecryptDirName(remoteObj.GetName()) - if err != nil { - log.Warnf("DecryptDirName failed for %s ,will use original name, err:%s", path, err) - name = remoteObj.GetName() + + size := remoteObj.GetSize() + name := remoteObj.GetName() + mask := model.GetObjMask(remoteObj) &^ model.Temp + if mask&model.Virtual == 0 { + if !remoteObj.IsDir() { + decryptedSize, err := d.cipher.DecryptedSize(size) + if err != nil { + log.Warnf("DecryptedSize failed for %s ,will use original size, err:%s", path, err) + } else { + size = decryptedSize + } + decryptedName, err := d.cipher.DecryptFileName(model.UnwrapObjName(remoteObj).GetName()) + if err != nil { + log.Warnf("DecryptFileName failed for %s ,will use original name, err:%s", path, err) + } else { + name = decryptedName + } + } else { + decryptedName, err := d.cipher.DecryptDirName(model.UnwrapObjName(remoteObj).GetName()) + if err != nil { + log.Warnf("DecryptDirName failed for %s ,will use original name, err:%s", path, err) + } else { + name = decryptedName + } } } - obj := &model.Object{ - Path: path, + return &model.Object{ + Path: remoteFullPath, Name: name, Size: size, Modified: remoteObj.ModTime(), IsFolder: remoteObj.IsDir(), - } - return obj, nil - // return nil, errs.ObjectNotFound + Ctime: remoteObj.CreateTime(), + Mask: mask, + }, nil } // https://github.com/rclone/rclone/blob/v1.67.0/backend/crypt/cipher.go#L37 const fileHeaderSize = 32 -func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - dstDirActualPath, err := d.getActualPathForRemote(file.GetPath(), false) +func (d *Crypt) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(file.GetPath()) if err != nil { - return nil, fmt.Errorf("failed to convert path to remote path: %w", err) + return nil, err } - remoteLink, remoteFile, err := op.Link(ctx, d.remoteStorage, dstDirActualPath, args) + remoteLink, remoteFile, err := op.Link(ctx, remoteStorage, remoteActualPath, model.LinkArgs{}) if err != nil { return nil, err } @@ -317,35 +305,29 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } return readSeeker, nil }), - SyncClosers: utils.NewSyncClosers(remoteLink), + SyncClosers: utils.NewSyncClosers(remoteLink), + RequireReference: remoteLink.RequireReference, }, nil } func (d *Crypt) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - dstDirActualPath, err := d.getActualPathForRemote(parentDir.GetPath(), true) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(parentDir.GetPath()) if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) + return err } - dir := d.cipher.EncryptDirName(dirName) - return op.MakeDir(ctx, d.remoteStorage, stdpath.Join(dstDirActualPath, dir)) + encryptedName := d.cipher.EncryptDirName(dirName) + return op.MakeDir(ctx, remoteStorage, stdpath.Join(remoteActualPath, encryptedName)) } func (d *Crypt) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath(), srcObj.IsDir()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath(), dstDir.IsDir()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - return op.Move(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + _, err := fs.Move(ctx, srcObj.GetPath(), dstDir.GetPath()) + return err } func (d *Crypt) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - remoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath(), srcObj.IsDir()) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(srcObj.GetPath()) if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) + return err } var newEncryptedName string if srcObj.IsDir() { @@ -353,33 +335,26 @@ func (d *Crypt) Rename(ctx context.Context, srcObj model.Obj, newName string) er } else { newEncryptedName = d.cipher.EncryptFileName(newName) } - return op.Rename(ctx, d.remoteStorage, remoteActualPath, newEncryptedName) + return op.Rename(ctx, remoteStorage, remoteActualPath, newEncryptedName) } func (d *Crypt) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath(), srcObj.IsDir()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath(), dstDir.IsDir()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - return op.Copy(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + _, err := fs.Copy(ctx, srcObj.GetPath(), dstDir.GetPath()) + return err } func (d *Crypt) Remove(ctx context.Context, obj model.Obj) error { - remoteActualPath, err := d.getActualPathForRemote(obj.GetPath(), obj.IsDir()) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(obj.GetPath()) if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) + return err } - return op.Remove(ctx, d.remoteStorage, remoteActualPath) + return op.Remove(ctx, remoteStorage, remoteActualPath) } func (d *Crypt) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { - dstDirActualPath, err := d.getActualPathForRemote(dstDir.GetPath(), true) + remoteStorage, remoteActualPath, err := op.GetStorageAndActualPath(dstDir.GetPath()) if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) + return err } // Encrypt the data into wrappedIn @@ -403,15 +378,15 @@ func (d *Crypt) Put(ctx context.Context, dstDir model.Obj, streamer model.FileSt ForceStreamUpload: true, Exist: streamer.GetExist(), } - err = op.Put(ctx, d.remoteStorage, dstDirActualPath, streamOut, up, false) - if err != nil { - return err - } - return nil + return op.Put(ctx, remoteStorage, remoteActualPath, streamOut, up) } func (d *Crypt) GetDetails(ctx context.Context) (*model.StorageDetails, error) { - remoteDetails, err := op.GetStorageDetails(ctx, d.remoteStorage) + remoteStorage, _, err := op.GetStorageAndActualPath(d.RemotePath) + if err != nil { + return nil, errs.NotImplement + } + remoteDetails, err := op.GetStorageDetails(ctx, remoteStorage) if err != nil { return nil, err } @@ -420,8 +395,4 @@ func (d *Crypt) GetDetails(ctx context.Context) (*model.StorageDetails, error) { }, nil } -//func (d *Safe) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { -// return nil, errs.NotSupport -//} - var _ driver.Driver = (*Crypt)(nil) diff --git a/drivers/crypt/meta.go b/drivers/crypt/meta.go index b2d009ba3..6a4659910 100644 --- a/drivers/crypt/meta.go +++ b/drivers/crypt/meta.go @@ -6,11 +6,6 @@ import ( ) type Addition struct { - // Usually one of two - //driver.RootPath - //driver.RootID - // define other - FileNameEnc string `json:"filename_encryption" type:"select" required:"true" options:"off,standard,obfuscate" default:"off"` DirNameEnc string `json:"directory_name_encryption" type:"select" required:"true" options:"false,true" default:"false"` RemotePath string `json:"remote_path" required:"true" help:"This is where the encrypted data stores"` @@ -32,6 +27,7 @@ var config = driver.Config{ NoCache: true, DefaultRoot: "/", NoLinkURL: true, + CheckStatus: true, } func init() { diff --git a/drivers/crypt/util.go b/drivers/crypt/util.go index 417059d38..e0f2bef74 100644 --- a/drivers/crypt/util.go +++ b/drivers/crypt/util.go @@ -4,8 +4,6 @@ import ( stdpath "path" "path/filepath" "strings" - - "github.com/OpenListTeam/OpenList/v4/internal/op" ) // will give the best guessing based on the path @@ -15,30 +13,17 @@ func guessPath(path string) (isFolder, secondTry bool) { return true, false } lastSlash := strings.LastIndex(path, "/") - if strings.Index(path[lastSlash:], ".") < 0 { + if !strings.Contains(path[lastSlash:], ".") { //no dot, try folder then try file return true, true } return false, true } -func (d *Crypt) getPathForRemote(path string, isFolder bool) (remoteFullPath string) { - if isFolder && !strings.HasSuffix(path, "/") { - path = path + "/" +func (d *Crypt) encryptPath(path string, isFolder bool) string { + if isFolder { + return d.cipher.EncryptDirName(path) } dir, fileName := filepath.Split(path) - - remoteDir := d.cipher.EncryptDirName(dir) - remoteFileName := "" - if len(strings.TrimSpace(fileName)) > 0 { - remoteFileName = d.cipher.EncryptFileName(fileName) - } - return stdpath.Join(d.RemotePath, remoteDir, remoteFileName) - -} - -// actual path is used for internal only. any link for user should come from remoteFullPath -func (d *Crypt) getActualPathForRemote(path string, isFolder bool) (string, error) { - _, remoteActualPath, err := op.GetStorageAndActualPath(d.getPathForRemote(path, isFolder)) - return remoteActualPath, err + return stdpath.Join(d.cipher.EncryptDirName(dir), d.cipher.EncryptFileName(fileName)) } diff --git a/drivers/degoo/driver.go b/drivers/degoo/driver.go index 648a13679..4cbc8be63 100644 --- a/drivers/degoo/driver.go +++ b/drivers/degoo/driver.go @@ -201,3 +201,24 @@ func (d *Degoo) Put(ctx context.Context, dstDir model.Obj, file model.FileStream } return nil } + +func (d *Degoo) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.getUserInfo(ctx) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(quota.GetUserInfo3.UsedQuota, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse used quota: %v", err) + } + total, err := strconv.ParseInt(quota.GetUserInfo3.TotalQuota, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse total quota: %v", err) + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} diff --git a/drivers/degoo/types.go b/drivers/degoo/types.go index 9d793b764..b0420c953 100644 --- a/drivers/degoo/types.go +++ b/drivers/degoo/types.go @@ -108,3 +108,26 @@ type DegooGetBucketWriteAuth4Data struct { type DegooSetUploadFile3Data struct { SetUploadFile3 bool `json:"setUploadFile3"` } + +type DegooGetUserInfo3Data struct { + GetUserInfo3 struct { + // ID string + // FirstName string + // LastName string + // Email string + // AvatarURL string + // CountryCode string = CN + // LanguageCode string = zh-cn + // Phone string + // AccountType int + UsedQuota string `json:"UsedQuota"` + TotalQuota string `json:"TotalQuota"` + // OAuth2Provider + // GPMigrationStatus int + // FeatureNoAds bool + // FeatureTopSecret bool + // FeatureDownsampling bool + // FeatureAutomaticVideoUploads bool + // FileSizeLimit string + } `json:"getUserInfo3"` +} diff --git a/drivers/degoo/util.go b/drivers/degoo/util.go index a146e3d47..26901269c 100644 --- a/drivers/degoo/util.go +++ b/drivers/degoo/util.go @@ -460,3 +460,19 @@ func (d *Degoo) getOverlay4(ctx context.Context, id string) (DegooFileItem, erro } return resp.GetOverlay4, nil } + +func (d *Degoo) getUserInfo(ctx context.Context) (DegooGetUserInfo3Data, error) { + const query = "query GetUserInfo3($Token: String!) { getUserInfo3(Token: $Token) { UsedQuota TotalQuota } }" + variables := map[string]interface{}{ + "Token": d.AccessToken, + } + data, err := d.apiCall(ctx, "GetUserInfo3", query, variables) + var resp DegooGetUserInfo3Data + if err != nil { + return resp, err + } + if err = json.Unmarshal(data, &resp); err != nil { + return resp, fmt.Errorf("failed to parse user info: %w", err) + } + return resp, nil +} diff --git a/drivers/doubao/driver.go b/drivers/doubao/driver.go index d2ba04ea3..6d4a4fc6e 100644 --- a/drivers/doubao/driver.go +++ b/drivers/doubao/driver.go @@ -15,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" + "golang.org/x/time/rate" ) type Doubao struct { @@ -23,6 +24,7 @@ type Doubao struct { *UploadToken UserId string uploadThread int + limiter *rate.Limiter } func (d *Doubao) Config() driver.Config { @@ -61,6 +63,17 @@ func (d *Doubao) Init(ctx context.Context) error { d.UploadToken = uploadToken } + if d.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) + } + + return nil +} + +func (d *Doubao) WaitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } return nil } @@ -69,6 +82,10 @@ func (d *Doubao) Drop(ctx context.Context) error { } func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + var files []model.Obj fileList, err := d.getFiles(dir.GetID(), "") if err != nil { @@ -95,6 +112,10 @@ func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + var downloadUrl string if u, ok := file.(*Object); ok { @@ -160,6 +181,10 @@ func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var r UploadNodeResp _, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ @@ -177,6 +202,10 @@ func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var r UploadNodeResp _, err := d.request("/samantha/aispace/move_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ @@ -191,6 +220,10 @@ func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var r BaseResp _, err := d.request("/samantha/aispace/rename_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ @@ -207,6 +240,10 @@ func (d *Doubao) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, } func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + var r BaseResp _, err := d.request("/samantha/aispace/delete_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}}) @@ -215,6 +252,10 @@ func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error { } func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + // 根据MIME类型确定数据类型 mimetype := file.GetMimetype() dataType := FileDataType diff --git a/drivers/doubao/meta.go b/drivers/doubao/meta.go index d2e350e3c..83dade919 100644 --- a/drivers/doubao/meta.go +++ b/drivers/doubao/meta.go @@ -10,9 +10,10 @@ type Addition struct { // driver.RootPath driver.RootID // define other - Cookie string `json:"cookie" type:"text"` - UploadThread string `json:"upload_thread" default:"3"` - DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"` + Cookie string `json:"cookie" type:"text"` + UploadThread string `json:"upload_thread" default:"3"` + DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"` + LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"` } var config = driver.Config{ @@ -23,6 +24,10 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { - return &Doubao{} + return &Doubao{ + Addition: Addition{ + LimitRate: 2, + }, + } }) } diff --git a/drivers/doubao/util.go b/drivers/doubao/util.go index f45c276f6..b1b38ee92 100644 --- a/drivers/doubao/util.go +++ b/drivers/doubao/util.go @@ -10,7 +10,6 @@ import ( "fmt" "hash/crc32" "io" - "math" "math/rand" "net/http" "net/url" @@ -18,7 +17,6 @@ import ( "sort" "strconv" "strings" - "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -62,7 +60,7 @@ const ( VideoDataType = "video" DefaultChunkSize = int64(5 * 1024 * 1024) // 5MB MaxRetryAttempts = 3 // 最大重试次数 - UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" + UserAgent = base.UserAgentNT Region = "cn-north-1" UploadTimeout = 3 * time.Minute ) @@ -486,7 +484,7 @@ func (d *Doubao) Upload(ctx context.Context, config *UploadConfig, dstDir model. "Authorization": {storeInfo.Auth}, "Content-Type": {"application/octet-stream"}, "Content-Crc32": {crc32Value}, - "Content-Length": {fmt.Sprintf("%d", file.GetSize())}, + "Content-Length": {strconv.FormatInt(file.GetSize(), 10)}, "Content-Disposition": {fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI))}, } res, err := base.HttpClient.Do(req) @@ -562,9 +560,7 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi retry.MaxJitter(200*time.Millisecond), ) - var partsMutex sync.Mutex // 并行上传所有分片 - hash := crc32.NewIEEE() for partIndex := range totalParts { if utils.IsCanceled(uploadCtx) { break @@ -578,32 +574,38 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi size = fileSize - offset } var reader io.ReadSeeker - var rateLimitedRd io.Reader crc32Value := "" threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { - if reader == nil { - var err error - reader, err = ss.GetSectionReader(offset, size) - if err != nil { - return err - } - hash.Reset() - w, err := utils.CopyWithBuffer(hash, reader) + Before: func(ctx context.Context) (err error) { + reader, err = ss.GetSectionReader(offset, size) + return + }, + Do: func(ctx context.Context) (err error) { + reader.Seek(0, io.SeekStart) + if crc32Value == "" { + // 把耗时的计算放在这里，避免阻塞其他协程 + crc32Hash := crc32.NewIEEE() + w, err := utils.CopyWithBuffer(crc32Hash, reader) if w != size { return fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", size, w, err) } - crc32Value = hex.EncodeToString(hash.Sum(nil)) - rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader) + crc32Value = hex.EncodeToString(crc32Hash.Sum(nil)) + reader.Seek(0, io.SeekStart) } - return nil - }, - Do: func(ctx context.Context) error { - reader.Seek(0, io.SeekStart) - req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("%s?uploadid=%s&part_number=%d&phase=transfer", uploadUrl, uploadID, partNumber), rateLimitedRd) + req, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + uploadUrl, + driver.NewLimitedUploadStream(ctx, reader), + ) if err != nil { return err } + query := req.URL.Query() + query.Add("uploadid", uploadID) + query.Add("part_number", strconv.FormatInt(partNumber, 10)) + query.Add("phase", "transfer") + req.URL.RawQuery = query.Encode() req.Header = map[string][]string{ "Referer": {BaseURL + "/"}, "Origin": {BaseURL}, @@ -612,7 +614,7 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi "Authorization": {storeInfo.Auth}, "Content-Type": {"application/octet-stream"}, "Content-Crc32": {crc32Value}, - "Content-Length": {fmt.Sprintf("%d", size)}, + "Content-Length": {strconv.FormatInt(size, 10)}, "Content-Disposition": {fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI))}, } res, err := base.HttpClient.Do(req) @@ -629,16 +631,14 @@ func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fi return fmt.Errorf("upload part failed: crc32 mismatch, expected %s, got %s", crc32Value, uploadResp.Data.Crc32) } // 记录成功上传的分片 - partsMutex.Lock() parts[partIndex] = UploadPart{ PartNumber: strconv.FormatInt(partNumber, 10), Etag: uploadResp.Data.Etag, Crc32: crc32Value, } - partsMutex.Unlock() // 更新进度 - progress := 10.0 + 90.0*float64(threadG.Success()+1)/float64(totalParts) - up(math.Min(progress, 95.0)) + progress := 95 * float64(threadG.Success()+1) / float64(totalParts) + up(progress) return nil }, After: func(err error) { diff --git a/drivers/doubao_share/driver.go b/drivers/doubao_share/driver.go index 89cda2129..4d23d7022 100644 --- a/drivers/doubao_share/driver.go +++ b/drivers/doubao_share/driver.go @@ -40,6 +40,7 @@ func (d *DoubaoShare) Drop(ctx context.Context) error { return nil } +// 潜在bug：配置二级目录时，可能会出问题 func (d *DoubaoShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { // 检查是否为根目录 if dir.GetID() == "" && dir.GetPath() == "/" { @@ -91,18 +92,17 @@ func (d *DoubaoShare) Link(ctx context.Context, file model.Obj, args model.LinkA downloadUrl = r.Data.OriginalMediaInfo.MainURL default: - var r GetFileUrlResp - _, err := d.request("/alice/message/get_file_url", http.MethodPost, func(req *resty.Request) { + var r GetDownloadInfoResp + _, err := d.request("/samantha/aispace/get_download_info", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ - "uris": []string{u.Key}, - "type": FileNodeType[u.NodeType], + "requests": []base.Json{{"node_id": file.GetID()}}, }) }, &r) if err != nil { return nil, err } - downloadUrl = r.Data.FileUrls[0].MainURL + downloadUrl = r.Data.DownloadInfos[0].MainURL } // 生成标准的Content-Disposition diff --git a/drivers/doubao_share/types.go b/drivers/doubao_share/types.go index 4ef12cedb..15e107656 100644 --- a/drivers/doubao_share/types.go +++ b/drivers/doubao_share/types.go @@ -115,14 +115,14 @@ type FilePath []struct { UpdateTime int64 `json:"update_time"` } -type GetFileUrlResp struct { +type GetDownloadInfoResp struct { BaseResp Data struct { - FileUrls []struct { - URI string `json:"uri"` - MainURL string `json:"main_url"` - BackURL string `json:"back_url"` - } `json:"file_urls"` + DownloadInfos []struct { + NodeID string `json:"node_id"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"download_infos"` } `json:"data"` } diff --git a/drivers/doubao_share/util.go b/drivers/doubao_share/util.go index 8e9887152..a52130305 100644 --- a/drivers/doubao_share/util.go +++ b/drivers/doubao_share/util.go @@ -43,7 +43,7 @@ const ( FileDataType = "file" ImgDataType = "image" VideoDataType = "video" - UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" + UserAgent = base.UserAgentNT ) func (d *DoubaoShare) request(path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { diff --git a/drivers/dropbox/util.go b/drivers/dropbox/util.go index d7404a429..ac0bcceeb 100644 --- a/drivers/dropbox/util.go +++ b/drivers/dropbox/util.go @@ -24,7 +24,6 @@ func (d *Dropbox) refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, @@ -176,12 +175,12 @@ func (d *Dropbox) finishUploadSession(ctx context.Context, toPath string, offset req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Authorization", "Bearer "+d.AccessToken) if d.RootNamespaceId != "" { - apiPathRootJson, err := d.buildPathRootHeader() - if err != nil { - return err + apiPathRootJson, err := d.buildPathRootHeader() + if err != nil { + return err + } + req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) } - req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) -} uploadFinishArgs := UploadFinishArgs{ Commit: struct { @@ -227,12 +226,12 @@ func (d *Dropbox) startUploadSession(ctx context.Context) (string, error) { req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Authorization", "Bearer "+d.AccessToken) if d.RootNamespaceId != "" { - apiPathRootJson, err := d.buildPathRootHeader() - if err != nil { - return "", err + apiPathRootJson, err := d.buildPathRootHeader() + if err != nil { + return "", err + } + req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) } - req.Header.Set("Dropbox-API-Path-Root", apiPathRootJson) -} req.Header.Set("Dropbox-API-Arg", "{\"close\":false}") res, err := base.HttpClient.Do(req) @@ -249,9 +248,8 @@ func (d *Dropbox) startUploadSession(ctx context.Context) (string, error) { } func (d *Dropbox) buildPathRootHeader() (string, error) { - return utils.Json.MarshalToString(map[string]interface{}{ - ".tag": "root", - "root": d.RootNamespaceId, - }) + return utils.Json.MarshalToString(map[string]interface{}{ + ".tag": "root", + "root": d.RootNamespaceId, + }) } - diff --git a/drivers/febbox/meta.go b/drivers/febbox/meta.go index e449ad002..cb6ff8fb1 100644 --- a/drivers/febbox/meta.go +++ b/drivers/febbox/meta.go @@ -16,9 +16,10 @@ type Addition struct { } var config = driver.Config{ - Name: "FebBox", - NoUpload: true, - DefaultRoot: "0", + Name: "FebBox", + NoUpload: true, + DefaultRoot: "0", + LinkCacheMode: driver.LinkCacheIP, } func init() { diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index f8d0e6ef5..484533786 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -60,11 +60,13 @@ func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m if entry.Name == "." || entry.Name == ".." { continue } + name := decode(entry.Name, d.Encoding) f := model.Object{ - Name: decode(entry.Name, d.Encoding), + Name: name, Size: int64(entry.Size), Modified: entry.Time, IsFolder: entry.Type == ftp.EntryTypeFolder, + Path: stdpath.Join(dir.GetPath(), name), } res = append(res, &f) } @@ -113,9 +115,7 @@ func (d *FTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*m } return &model.Link{ - RangeReader: &model.FileRangeReader{ - RangeReaderIF: stream.RateLimitRangeReaderFunc(resultRangeReader), - }, + RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader), SyncClosers: utils.NewSyncClosers(utils.CloseFunc(conn.Quit)), }, nil } diff --git a/drivers/ftp/meta.go b/drivers/ftp/meta.go index 8f30776c0..0ec0e735e 100644 --- a/drivers/ftp/meta.go +++ b/drivers/ftp/meta.go @@ -31,11 +31,11 @@ type Addition struct { } var config = driver.Config{ - Name: "FTP", - LocalSort: true, - OnlyLinkMFile: false, - DefaultRoot: "/", - NoLinkURL: true, + Name: "FTP", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: "/", + NoLinkURL: true, } func init() { diff --git a/drivers/github/types.go b/drivers/github/types.go index 6f9029d5b..d0d24839c 100644 --- a/drivers/github/types.go +++ b/drivers/github/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) type Links struct { @@ -36,6 +37,7 @@ func (o *Object) toModelObj() *model.Object { Size: o.Size, Modified: time.Unix(0, 0), IsFolder: o.Type == "dir", + Path: utils.FixAndCleanPath(o.Path), } } @@ -69,6 +71,7 @@ func (o *TreeObjResp) toModelObj() *model.Object { Size: o.Size, Modified: time.Unix(0, 0), IsFolder: o.Type == "tree", + Path: utils.FixAndCleanPath(o.Path), } } diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index c11c9cac2..8a8025c5a 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + stdpath "path" "strings" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -51,6 +52,9 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis if d.Addition.ShowReadme { files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) } + if d.Addition.ShowSourceCode { + files = append(files, point.GetSourceCode()...) + } } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 nextDir := GetNextDir(point.Point, path) if nextDir == "" { @@ -67,7 +71,7 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis } if !hasSameDir { files = append(files, File{ - Path: path + "/" + nextDir, + Path: stdpath.Join(path, nextDir), FileName: nextDir, Size: point.GetLatestSize(), UpdateAt: point.Release.PublishedAt, @@ -102,7 +106,7 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis if !hasSameDir { files = append(files, File{ FileName: nextDir, - Path: path + "/" + nextDir, + Path: stdpath.Join(path, nextDir), Size: point.GetAllVersionSize(), UpdateAt: (*point.Releases)[0].PublishedAt, CreateAt: (*point.Releases)[0].CreatedAt, @@ -117,6 +121,10 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis } files = append(files, point.GetReleaseByTagName(tagName)...) + + if d.Addition.ShowSourceCode { + files = append(files, point.GetSourceCodeByTagName(tagName)...) + } } } } diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go index 4f3aae7fc..fc5fad695 100644 --- a/drivers/github_releases/meta.go +++ b/drivers/github_releases/meta.go @@ -6,10 +6,11 @@ import ( ) type Addition struct { - driver.RootID + driver.RootPath RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"OpenListTeam/OpenList" help:"structure:[path:]org/repo"` ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` + ShowSourceCode bool `json:"show_source_code" type:"bool" default:"false" help:"show Source code (zip/tar.gz)"` ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` } diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index 3bbf1e766..663aec77e 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -2,6 +2,7 @@ package github_releases import ( "encoding/json" + "path" "strings" "time" @@ -45,10 +46,10 @@ func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, erro // 获取最新版本 func (m *MountPoint) GetLatestRelease() []File { - files := make([]File, 0) + files := make([]File, 0, len(m.Release.Assets)) for _, asset := range m.Release.Assets { files = append(files, File{ - Path: m.Point + "/" + asset.Name, + Path: path.Join(m.Point, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", @@ -74,7 +75,7 @@ func (m *MountPoint) GetAllVersion() []File { files := make([]File, 0) for _, release := range *m.Releases { file := File{ - Path: m.Point + "/" + release.TagName, + Path: path.Join(m.Point, release.TagName), FileName: release.TagName, Size: m.GetSizeByTagName(release.TagName), Type: "dir", @@ -97,7 +98,7 @@ func (m *MountPoint) GetReleaseByTagName(tagName string) []File { files := make([]File, 0) for _, asset := range item.Assets { files = append(files, File{ - Path: m.Point + "/" + tagName + "/" + asset.Name, + Path: path.Join(m.Point, tagName, asset.Name), FileName: asset.Name, Size: asset.Size, Type: "file", @@ -143,6 +144,60 @@ func (m *MountPoint) GetAllVersionSize() int64 { return size } +func (m *MountPoint) GetSourceCode() []File { + files := make([]File, 0) + + // 无法获取文件大小，此处设为 1 + files = append(files, File{ + Path: path.Join(m.Point, "Source code (zip)"), + FileName: "Source code (zip)", + Size: 1, + Type: "file", + UpdateAt: m.Release.CreatedAt, + CreateAt: m.Release.CreatedAt, + Url: m.Release.ZipballUrl, + }) + files = append(files, File{ + Path: path.Join(m.Point, "Source code (tar.gz)"), + FileName: "Source code (tar.gz)", + Size: 1, + Type: "file", + UpdateAt: m.Release.CreatedAt, + CreateAt: m.Release.CreatedAt, + Url: m.Release.TarballUrl, + }) + + return files +} + +func (m *MountPoint) GetSourceCodeByTagName(tagName string) []File { + for _, item := range *m.Releases { + if item.TagName == tagName { + files := make([]File, 0) + files = append(files, File{ + Path: path.Join(m.Point, "Source code (zip)"), + FileName: "Source code (zip)", + Size: 1, + Type: "file", + UpdateAt: item.CreatedAt, + CreateAt: item.CreatedAt, + Url: item.ZipballUrl, + }) + files = append(files, File{ + Path: path.Join(m.Point, "Source code (tar.gz)"), + FileName: "Source code (tar.gz)", + Size: 1, + Type: "file", + UpdateAt: item.CreatedAt, + CreateAt: item.CreatedAt, + Url: item.TarballUrl, + }) + return files + } + } + return nil +} + func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) []File { if m.OtherFile == nil || refresh { resp, _ := get("https://api.github.com/repos/" + m.Repo + "/contents") @@ -155,7 +210,7 @@ func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), for _, file := range *m.OtherFile { if strings.HasSuffix(file.Name, ".md") || strings.HasPrefix(file.Name, "LICENSE") { files = append(files, File{ - Path: m.Point + "/" + file.Name, + Path: path.Join(m.Point, file.Name), FileName: file.Name, Size: file.Size, Type: "file", diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 36b9ff01f..94ef854f2 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -175,23 +175,23 @@ func (d *GoogleDrive) GetDetails(ctx context.Context) (*model.StorageDetails, er if err != nil { return nil, err } - var total, used uint64 + var total, used int64 if about.StorageQuota.Limit == nil { total = 0 } else { - total, err = strconv.ParseUint(*about.StorageQuota.Limit, 10, 64) + total, err = strconv.ParseInt(*about.StorageQuota.Limit, 10, 64) if err != nil { return nil, err } } - used, err = strconv.ParseUint(about.StorageQuota.Usage, 10, 64) + used, err = strconv.ParseInt(about.StorageQuota.Usage, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 372b62540..042abafa4 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -27,6 +27,14 @@ import ( // do others that not defined in Driver interface +// Google Drive API field constants +const ( + // File list query fields + FilesListFields = "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken" + // Single file query fields + FileInfoFields = "id,name,mimeType,size,md5Checksum,sha1Checksum,sha256Checksum" +) + type googleDriveServiceAccount struct { // Type string `json:"type"` // ProjectID string `json:"project_id"` @@ -50,7 +58,6 @@ func (d *GoogleDrive) refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, @@ -235,7 +242,7 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { } query := map[string]string{ "orderBy": orderBy, - "fields": "files(id,name,mimeType,size,modifiedTime,createdTime,thumbnailLink,shortcutDetails,md5Checksum,sha1Checksum,sha256Checksum),nextPageToken", + "fields": FilesListFields, "pageSize": "1000", "q": fmt.Sprintf("'%s' in parents and trashed = false", id), //"includeItemsFromAllDrives": "true", @@ -249,11 +256,82 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { return nil, err } pageToken = resp.NextPageToken + + // Batch process shortcuts, API calls only for file shortcuts + shortcutTargetIds := make([]string, 0) + shortcutIndices := make([]int, 0) + + // Collect target IDs of all file shortcuts (skip folder shortcuts) + for i := range resp.Files { + if resp.Files[i].MimeType == "application/vnd.google-apps.shortcut" && + resp.Files[i].ShortcutDetails.TargetId != "" && + resp.Files[i].ShortcutDetails.TargetMimeType != "application/vnd.google-apps.folder" { + shortcutTargetIds = append(shortcutTargetIds, resp.Files[i].ShortcutDetails.TargetId) + shortcutIndices = append(shortcutIndices, i) + } + } + + // Batch get target file info (only for file shortcuts) + if len(shortcutTargetIds) > 0 { + targetFiles := d.batchGetTargetFilesInfo(shortcutTargetIds) + // Update shortcut file info + for j, targetId := range shortcutTargetIds { + if targetFile, exists := targetFiles[targetId]; exists { + fileIndex := shortcutIndices[j] + if targetFile.Size != "" { + resp.Files[fileIndex].Size = targetFile.Size + } + if targetFile.MD5Checksum != "" { + resp.Files[fileIndex].MD5Checksum = targetFile.MD5Checksum + } + if targetFile.SHA1Checksum != "" { + resp.Files[fileIndex].SHA1Checksum = targetFile.SHA1Checksum + } + if targetFile.SHA256Checksum != "" { + resp.Files[fileIndex].SHA256Checksum = targetFile.SHA256Checksum + } + } + } + } + res = append(res, resp.Files...) } return res, nil } +// getTargetFileInfo gets target file details for shortcuts +func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) { + var targetFile File + url := fmt.Sprintf("https://www.googleapis.com/drive/v3/files/%s", targetId) + query := map[string]string{ + "fields": FileInfoFields, + } + _, err := d.request(url, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &targetFile) + if err != nil { + return File{}, err + } + return targetFile, nil +} + +// batchGetTargetFilesInfo batch gets target file info, sequential processing to avoid concurrency complexity +func (d *GoogleDrive) batchGetTargetFilesInfo(targetIds []string) map[string]File { + if len(targetIds) == 0 { + return make(map[string]File) + } + + result := make(map[string]File) + // Sequential processing to avoid concurrency complexity + for _, targetId := range targetIds { + file, err := d.getTargetFileInfo(targetId) + if err == nil { + result[targetId] = file + } + } + return result +} + func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer, url string, up driver.UpdateProgress) error { defaultChunkSize := d.ChunkSize * 1024 * 1024 ss, err := stream.NewStreamSectionReader(file, int(defaultChunkSize), &up) @@ -304,6 +382,7 @@ func (d *GoogleDrive) chunkUpload(ctx context.Context, file model.FileStreamer, up(float64(offset+chunkSize) / float64(file.GetSize()) * 100) return nil }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) diff --git a/drivers/halalcloud_open/common.go b/drivers/halalcloud_open/common.go new file mode 100644 index 000000000..575ccff3a --- /dev/null +++ b/drivers/halalcloud_open/common.go @@ -0,0 +1,111 @@ +package halalcloudopen + +import ( + "sync" + "time" + + sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user" +) + +var ( + slicePostErrorRetryInterval = time.Second * 120 + retryTimes = 5 +) + +type halalCommon struct { + // *AuthService // 登录信息 + UserInfo *sdkUser.User // 用户信息 + refreshTokenFunc func(token string) error + // serv *AuthService + configs sync.Map +} + +func (m *halalCommon) GetAccessToken() (string, error) { + value, exists := m.configs.Load("access_token") + if !exists { + return "", nil // 如果不存在，返回空字符串 + } + return value.(string), nil // 返回配置项的值 +} + +// GetRefreshToken implements ConfigStore. +func (m *halalCommon) GetRefreshToken() (string, error) { + value, exists := m.configs.Load("refresh_token") + if !exists { + return "", nil // 如果不存在，返回空字符串 + } + return value.(string), nil // 返回配置项的值 +} + +// SetAccessToken implements ConfigStore. +func (m *halalCommon) SetAccessToken(token string) error { + m.configs.Store("access_token", token) + return nil +} + +// SetRefreshToken implements ConfigStore. +func (m *halalCommon) SetRefreshToken(token string) error { + m.configs.Store("refresh_token", token) + if m.refreshTokenFunc != nil { + return m.refreshTokenFunc(token) + } + return nil +} + +// SetToken implements ConfigStore. +func (m *halalCommon) SetToken(accessToken string, refreshToken string, expiresIn int64) error { + m.configs.Store("access_token", accessToken) + m.configs.Store("refresh_token", refreshToken) + m.configs.Store("expires_in", expiresIn) + if m.refreshTokenFunc != nil { + return m.refreshTokenFunc(refreshToken) + } + return nil +} + +// ClearConfigs implements ConfigStore. +func (m *halalCommon) ClearConfigs() error { + m.configs = sync.Map{} // 清空map + return nil +} + +// DeleteConfig implements ConfigStore. +func (m *halalCommon) DeleteConfig(key string) error { + _, exists := m.configs.Load(key) + if !exists { + return nil // 如果不存在，直接返回 + } + m.configs.Delete(key) // 删除指定的配置项 + return nil +} + +// GetConfig implements ConfigStore. +func (m *halalCommon) GetConfig(key string) (string, error) { + value, exists := m.configs.Load(key) + if !exists { + return "", nil // 如果不存在，返回空字符串 + } + return value.(string), nil // 返回配置项的值 +} + +// ListConfigs implements ConfigStore. +func (m *halalCommon) ListConfigs() (map[string]string, error) { + configs := make(map[string]string) + m.configs.Range(func(key, value interface{}) bool { + configs[key.(string)] = value.(string) // 将每个配置项添加到map中 + return true // 继续遍历 + }) + return configs, nil // 返回所有配置项 +} + +// SetConfig implements ConfigStore. +func (m *halalCommon) SetConfig(key string, value string) error { + m.configs.Store(key, value) // 使用Store方法设置或更新配置项 + return nil // 成功设置配置项后返回nil +} + +func NewHalalCommon() *halalCommon { + return &halalCommon{ + configs: sync.Map{}, + } +} diff --git a/drivers/halalcloud_open/driver.go b/drivers/halalcloud_open/driver.go new file mode 100644 index 000000000..6a74538d4 --- /dev/null +++ b/drivers/halalcloud_open/driver.go @@ -0,0 +1,29 @@ +package halalcloudopen + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + sdkClient "github.com/halalcloud/golang-sdk-lite/halalcloud/apiclient" + sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" +) + +type HalalCloudOpen struct { + *halalCommon + model.Storage + Addition + sdkClient *sdkClient.Client + sdkUserFileService *sdkUserFile.UserFileService + sdkUserService *sdkUser.UserService + uploadThread int +} + +func (d *HalalCloudOpen) Config() driver.Config { + return config +} + +func (d *HalalCloudOpen) GetAddition() driver.Additional { + return &d.Addition +} + +var _ driver.Driver = (*HalalCloudOpen)(nil) diff --git a/drivers/halalcloud_open/driver_curd_impl.go b/drivers/halalcloud_open/driver_curd_impl.go new file mode 100644 index 000000000..2bd32cf56 --- /dev/null +++ b/drivers/halalcloud_open/driver_curd_impl.go @@ -0,0 +1,127 @@ +package halalcloudopen + +import ( + "context" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + sdkModel "github.com/halalcloud/golang-sdk-lite/halalcloud/model" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" +) + +func (d *HalalCloudOpen) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj, error) { + + files := make([]model.Obj, 0) + limit := int64(100) + token := "" + + for { + result, err := d.sdkUserFileService.List(ctx, &sdkUserFile.FileListRequest{ + Parent: &sdkUserFile.File{Path: dir.GetPath()}, + ListInfo: &sdkModel.ScanListRequest{ + Limit: limit, + Token: token, + }, + }) + if err != nil { + return nil, err + } + + for i := 0; len(result.Files) > i; i++ { + files = append(files, NewObjFile(result.Files[i])) + } + + if result.ListInfo == nil || result.ListInfo.Token == "" { + break + } + token = result.ListInfo.Token + + } + return files, nil +} + +func (d *HalalCloudOpen) makeDir(ctx context.Context, dir model.Obj, name string) (model.Obj, error) { + _, err := d.sdkUserFileService.Create(ctx, &sdkUserFile.File{ + Path: dir.GetPath(), + Name: name, + }) + return nil, err +} + +func (d *HalalCloudOpen) move(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { + oldDir := obj.GetPath() + newDir := dir.GetPath() + _, err := d.sdkUserFileService.Move(ctx, &sdkUserFile.BatchOperationRequest{ + Source: []*sdkUserFile.File{ + { + Path: oldDir, + }, + }, + Dest: &sdkUserFile.File{ + Path: newDir, + }, + }) + return nil, err +} + +func (d *HalalCloudOpen) rename(ctx context.Context, obj model.Obj, name string) (model.Obj, error) { + + _, err := d.sdkUserFileService.Rename(ctx, &sdkUserFile.File{ + Path: obj.GetPath(), + Name: name, + }) + return nil, err +} + +func (d *HalalCloudOpen) copy(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { + id := obj.GetID() + sourcePath := obj.GetPath() + if len(id) > 0 { + sourcePath = "" + } + + destID := dir.GetID() + destPath := dir.GetPath() + if len(destID) > 0 { + destPath = "" + } + dest := &sdkUserFile.File{ + Path: destPath, + Identity: destID, + } + _, err := d.sdkUserFileService.Copy(ctx, &sdkUserFile.BatchOperationRequest{ + Source: []*sdkUserFile.File{ + { + Path: sourcePath, + Identity: id, + }, + }, + Dest: dest, + }) + return nil, err +} + +func (d *HalalCloudOpen) remove(ctx context.Context, obj model.Obj) error { + id := obj.GetID() + _, err := d.sdkUserFileService.Delete(ctx, &sdkUserFile.BatchOperationRequest{ + Source: []*sdkUserFile.File{ + { + Identity: id, + Path: obj.GetPath(), + }, + }, + }) + return err +} + +func (d *HalalCloudOpen) details(ctx context.Context) (*model.StorageDetails, error) { + ret, err := d.sdkUserService.GetStatisticsAndQuota(ctx) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: ret.DiskStatisticsQuota.BytesQuota, + UsedSpace: ret.DiskStatisticsQuota.BytesUsed, + }, + }, nil +} diff --git a/drivers/halalcloud_open/driver_get_link.go b/drivers/halalcloud_open/driver_get_link.go new file mode 100644 index 000000000..cecc46b38 --- /dev/null +++ b/drivers/halalcloud_open/driver_get_link.go @@ -0,0 +1,108 @@ +package halalcloudopen + +import ( + "context" + "crypto/sha1" + "io" + "strconv" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" + "github.com/rclone/rclone/lib/readers" +) + +func (d *HalalCloudOpen) getLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if args.Redirect { + // return nil, model.ErrUnsupported + fid := file.GetID() + fpath := file.GetPath() + if fid != "" { + fpath = "" + } + fi, err := d.sdkUserFileService.GetDirectDownloadAddress(ctx, &sdkUserFile.DirectDownloadRequest{ + Identity: fid, + Path: fpath, + }) + if err != nil { + return nil, err + } + expireAt := fi.ExpireAt + duration := time.Until(time.UnixMilli(expireAt)) + return &model.Link{ + URL: fi.DownloadAddress, + Expiration: &duration, + }, nil + } + result, err := d.sdkUserFileService.ParseFileSlice(ctx, &sdkUserFile.File{ + Identity: file.GetID(), + Path: file.GetPath(), + }) + if err != nil { + return nil, err + } + fileAddrs := []*sdkUserFile.SliceDownloadInfo{} + var addressDuration int64 + + nodesNumber := len(result.RawNodes) + nodesIndex := nodesNumber - 1 + startIndex, endIndex := 0, nodesIndex + for nodesIndex >= 0 { + if nodesIndex >= 200 { + endIndex = 200 + } else { + endIndex = nodesNumber + } + for ; endIndex <= nodesNumber; endIndex += 200 { + if endIndex == 0 { + endIndex = 1 + } + sliceAddress, err := d.sdkUserFileService.GetSliceDownloadAddress(ctx, &sdkUserFile.SliceDownloadAddressRequest{ + Identity: result.RawNodes[startIndex:endIndex], + Version: 1, + }) + if err != nil { + return nil, err + } + addressDuration, _ = strconv.ParseInt(sliceAddress.ExpireAt, 10, 64) + fileAddrs = append(fileAddrs, sliceAddress.Addresses...) + startIndex = endIndex + nodesIndex -= 200 + } + + } + + size, _ := strconv.ParseInt(result.FileSize, 10, 64) + chunks := getChunkSizes(result.Sizes) + resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + length := httpRange.Length + if httpRange.Length < 0 || httpRange.Start+httpRange.Length >= size { + length = size - httpRange.Start + } + oo := &openObject{ + ctx: ctx, + d: fileAddrs, + chunk: []byte{}, + chunks: chunks, + skip: httpRange.Start, + sha: result.Sha1, + shaTemp: sha1.New(), + } + + return readers.NewLimitedReadCloser(oo, length), nil + } + + var duration time.Duration + if addressDuration != 0 { + duration = time.Until(time.UnixMilli(addressDuration)) + } else { + duration = time.Until(time.Now().Add(time.Hour)) + } + + return &model.Link{ + RangeReader: stream.RateLimitRangeReaderFunc(resultRangeReader), + Expiration: &duration, + }, nil +} diff --git a/drivers/halalcloud_open/driver_init.go b/drivers/halalcloud_open/driver_init.go new file mode 100644 index 000000000..9f7026380 --- /dev/null +++ b/drivers/halalcloud_open/driver_init.go @@ -0,0 +1,50 @@ +package halalcloudopen + +import ( + "context" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/halalcloud/golang-sdk-lite/halalcloud/apiclient" + sdkUser "github.com/halalcloud/golang-sdk-lite/halalcloud/services/user" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" +) + +func (d *HalalCloudOpen) Init(ctx context.Context) error { + if d.uploadThread < 1 || d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 3, 3 + } + if d.halalCommon == nil { + d.halalCommon = &halalCommon{ + UserInfo: &sdkUser.User{}, + refreshTokenFunc: func(token string) error { + d.Addition.RefreshToken = token + op.MustSaveDriverStorage(d) + return nil + }, + } + } + if d.Addition.RefreshToken != "" { + d.halalCommon.SetRefreshToken(d.Addition.RefreshToken) + } + timeout := d.Addition.TimeOut + if timeout <= 0 { + timeout = 60 + } + host := d.Addition.Host + if host == "" { + host = "openapi.2dland.cn" + } + + client := apiclient.NewClient(nil, host, d.Addition.ClientID, d.Addition.ClientSecret, d.halalCommon, apiclient.WithTimeout(time.Second*time.Duration(timeout))) + d.sdkClient = client + d.sdkUserFileService = sdkUserFile.NewUserFileService(client) + d.sdkUserService = sdkUser.NewUserService(client) + userInfo, err := d.sdkUserService.Get(ctx, &sdkUser.User{}) + if err != nil { + return err + } + d.halalCommon.UserInfo = userInfo + // 能够获取到用户信息，已经检查了 RefreshToken 的有效性，无需再次检查 + return nil +} diff --git a/drivers/halalcloud_open/driver_interface.go b/drivers/halalcloud_open/driver_interface.go new file mode 100644 index 000000000..0ddf248bb --- /dev/null +++ b/drivers/halalcloud_open/driver_interface.go @@ -0,0 +1,48 @@ +package halalcloudopen + +import ( + "context" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func (d *HalalCloudOpen) Drop(ctx context.Context) error { + return nil +} + +func (d *HalalCloudOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.getFiles(ctx, dir) +} + +func (d *HalalCloudOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return d.getLink(ctx, file, args) +} + +func (d *HalalCloudOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return d.makeDir(ctx, parentDir, dirName) +} + +func (d *HalalCloudOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.move(ctx, srcObj, dstDir) +} + +func (d *HalalCloudOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return d.rename(ctx, srcObj, newName) +} + +func (d *HalalCloudOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.copy(ctx, srcObj, dstDir) +} + +func (d *HalalCloudOpen) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(ctx, obj) +} + +func (d *HalalCloudOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return d.put(ctx, dstDir, stream, up) +} + +func (d *HalalCloudOpen) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + return d.details(ctx) +} diff --git a/drivers/halalcloud_open/halalcloud_upload.go b/drivers/halalcloud_open/halalcloud_upload.go new file mode 100644 index 000000000..16a342fd9 --- /dev/null +++ b/drivers/halalcloud_open/halalcloud_upload.go @@ -0,0 +1,258 @@ +package halalcloudopen + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" + "github.com/ipfs/go-cid" + log "github.com/sirupsen/logrus" +) + +func (d *HalalCloudOpen) put(ctx context.Context, dstDir model.Obj, fileStream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + + newPath := path.Join(dstDir.GetPath(), fileStream.GetName()) + + uploadTask, err := d.sdkUserFileService.CreateUploadTask(ctx, &sdkUserFile.File{ + Path: newPath, + Size: fileStream.GetSize(), + }) + if err != nil { + return nil, err + } + + if uploadTask.Created { + return nil, nil + } + + slicesList := make([]string, 0) + codec := uint64(0x55) + if uploadTask.BlockCodec > 0 { + codec = uint64(uploadTask.BlockCodec) + } + blockHashType := uploadTask.BlockHashType + mhType := uint64(0x12) + if blockHashType > 0 { + mhType = uint64(blockHashType) + } + prefix := cid.Prefix{ + Codec: codec, + MhLength: -1, + MhType: mhType, + Version: 1, + } + blockSize := uploadTask.BlockSize + // + // Not sure whether FileStream supports concurrent read and write operations, so currently using single-threaded upload to ensure safety. + // read file + bufferSize := int(blockSize) + buffer := make([]byte, bufferSize) + offset := 0 + teeReader := io.TeeReader(fileStream, driver.NewProgress(fileStream.GetSize(), up)) + for { + n, err := teeReader.Read(buffer[offset:]) // 这里 len(buf[offset:]) <= 4MB + if n > 0 { + offset += n + if offset == int(blockSize) { + uploadCid, err := postFileSlice(ctx, buffer, uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes) + if err != nil { + return nil, err + } + slicesList = append(slicesList, uploadCid.String()) + offset = 0 + } + } + + if err != nil { + if err == io.EOF { + if offset > 0 { + uploadCid, err := postFileSlice(ctx, buffer[:offset], uploadTask.Task, uploadTask.UploadAddress, prefix, retryTimes) + if err != nil { + return nil, err + } + slicesList = append(slicesList, uploadCid.String()) + } + break + } + return nil, err + } + } + newFile, err := makeFile(ctx, slicesList, uploadTask.Task, uploadTask.UploadAddress, retryTimes) + if err != nil { + return nil, err + } + + return NewObjFile(newFile), nil + +} + +func makeFile(ctx context.Context, fileSlice []string, taskID string, uploadAddress string, retry int) (*sdkUserFile.File, error) { + var lastError error = nil + for range retry { + newFile, err := doMakeFile(fileSlice, taskID, uploadAddress) + if err == nil { + return newFile, nil + } + if ctx.Err() != nil { + return nil, err + } + log.Errorf("make file slice failed, retrying... error: %s", err.Error()) + if strings.Contains(err.Error(), "not found") { + return nil, err + } + lastError = err + time.Sleep(slicePostErrorRetryInterval) + } + return nil, fmt.Errorf("mk file slice failed after %d times, error: %s", retry, lastError.Error()) +} + +func doMakeFile(fileSlice []string, taskID string, uploadAddress string) (*sdkUserFile.File, error) { + accessUrl := uploadAddress + "/" + taskID + getTimeOut := time.Minute * 2 + u, err := url.Parse(accessUrl) + if err != nil { + return nil, err + } + n, _ := json.Marshal(fileSlice) + httpRequest := http.Request{ + Method: http.MethodPost, + URL: u, + Header: map[string][]string{ + "Accept": {"application/json"}, + "Content-Type": {"application/json"}, + //"Content-Length": {strconv.Itoa(len(n))}, + }, + Body: io.NopCloser(bytes.NewReader(n)), + } + httpClient := http.Client{ + Timeout: getTimeOut, + } + httpResponse, err := httpClient.Do(&httpRequest) + if err != nil { + return nil, err + } + defer httpResponse.Body.Close() + if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated { + b, _ := io.ReadAll(httpResponse.Body) + message := string(b) + log.Errorf("make file failed, status code: %d, message: %s", httpResponse.StatusCode, message) + + return nil, fmt.Errorf("mk file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message) + } + b, _ := io.ReadAll(httpResponse.Body) + var result *UploadedFile + err = json.Unmarshal(b, &result) + if err != nil { + log.Errorf("make file failed from response, status code: %d, message: %s", httpResponse.StatusCode, string(b)) + return nil, err + } + return &sdkUserFile.File{ + Identity: result.Identity, + Path: result.Path, + Size: result.Size, + ContentIdentity: result.ContentIdentity, + }, nil +} +func postFileSlice(ctx context.Context, fileSlice []byte, taskID string, uploadAddress string, preix cid.Prefix, retry int) (cid.Cid, error) { + var lastError error = nil + for range retry { + newCid, err := doPostFileSlice(fileSlice, taskID, uploadAddress, preix) + if err == nil { + return newCid, nil + } + if ctx.Err() != nil { + return cid.Undef, err + } + time.Sleep(slicePostErrorRetryInterval) + lastError = err + } + return cid.Undef, fmt.Errorf("upload file slice failed after %d times, error: %s", retry, lastError.Error()) +} +func doPostFileSlice(fileSlice []byte, taskID string, uploadAddress string, preix cid.Prefix) (cid.Cid, error) { + // 1. sum file slice + newCid, err := preix.Sum(fileSlice) + if err != nil { + return cid.Undef, err + } + // 2. post file slice + sliceCidString := newCid.String() + // /{taskID}/{sliceID} + accessUrl := uploadAddress + "/" + taskID + "/" + sliceCidString + getTimeOut := time.Second * 30 + // get {accessUrl} in {getTimeOut} + u, err := url.Parse(accessUrl) + if err != nil { + return cid.Undef, err + } + // header: accept: application/json + // header: content-type: application/octet-stream + // header: content-length: {fileSlice.length} + // header: x-content-cid: {sliceCidString} + // header: x-task-id: {taskID} + httpRequest := http.Request{ + Method: http.MethodGet, + URL: u, + Header: map[string][]string{ + "Accept": {"application/json"}, + }, + } + httpClient := http.Client{ + Timeout: getTimeOut, + } + httpResponse, err := httpClient.Do(&httpRequest) + if err != nil { + log.Errorf("access %s failed, method: %s", accessUrl, http.MethodGet) + return cid.Undef, err + } + if httpResponse.StatusCode != http.StatusOK { + log.Errorf("access %s failed, method: %s, status code: %d", accessUrl, http.MethodGet, httpResponse.StatusCode) + return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d", httpResponse.StatusCode) + } + var result bool + b, err := io.ReadAll(httpResponse.Body) + if err != nil { + return cid.Undef, err + } + err = json.Unmarshal(b, &result) + if err != nil { + return cid.Undef, err + } + if result { + return newCid, nil + } + + httpRequest = http.Request{ + Method: http.MethodPost, + URL: u, + Header: map[string][]string{ + "Accept": {"application/json"}, + "Content-Type": {"application/octet-stream"}, + // "Content-Length": {strconv.Itoa(len(fileSlice))}, + }, + Body: io.NopCloser(bytes.NewReader(fileSlice)), + } + httpResponse, err = httpClient.Do(&httpRequest) + if err != nil { + return cid.Undef, err + } + defer httpResponse.Body.Close() + if httpResponse.StatusCode != http.StatusOK && httpResponse.StatusCode != http.StatusCreated { + b, _ := io.ReadAll(httpResponse.Body) + message := string(b) + log.Errorf("upload file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message) + return cid.Undef, fmt.Errorf("upload file slice failed, status code: %d, message: %s", httpResponse.StatusCode, message) + } + // + + return newCid, nil +} diff --git a/drivers/halalcloud_open/meta.go b/drivers/halalcloud_open/meta.go new file mode 100644 index 000000000..62f280b4e --- /dev/null +++ b/drivers/halalcloud_open/meta.go @@ -0,0 +1,40 @@ +package halalcloudopen + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // define other + RefreshToken string `json:"refresh_token" required:"false" help:"If using a personal API approach, the RefreshToken is not required."` + UploadThread int `json:"upload_thread" type:"number" default:"3" help:"1 <= thread <= 32"` + + ClientID string `json:"client_id" required:"true" default:""` + ClientSecret string `json:"client_secret" required:"true" default:""` + Host string `json:"host" required:"false" default:"openapi.2dland.cn"` + TimeOut int `json:"timeout" type:"number" default:"60" help:"timeout in seconds"` +} + +var config = driver.Config{ + Name: "HalalCloudOpen", + OnlyProxy: false, + DefaultRoot: "/", + NoLinkURL: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &HalalCloudOpen{} + }) +} + +type UploadedFile struct { + Identity string `json:"identity"` + UserIdentity string `json:"user_identity"` + Path string `json:"path"` + Size int64 `json:"size"` + ContentIdentity string `json:"content_identity"` +} diff --git a/drivers/halalcloud_open/obj_file.go b/drivers/halalcloud_open/obj_file.go new file mode 100644 index 000000000..839e9f4c9 --- /dev/null +++ b/drivers/halalcloud_open/obj_file.go @@ -0,0 +1,60 @@ +package halalcloudopen + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" +) + +type ObjFile struct { + sdkFile *sdkUserFile.File + fileSize int64 + modTime time.Time + createTime time.Time +} + +func NewObjFile(f *sdkUserFile.File) model.Obj { + ofile := &ObjFile{sdkFile: f} + ofile.fileSize = f.Size + modTimeTs := f.UpdateTs + ofile.modTime = time.UnixMilli(modTimeTs) + createTimeTs := f.CreateTs + ofile.createTime = time.UnixMilli(createTimeTs) + return ofile +} + +func (f *ObjFile) GetSize() int64 { + return f.fileSize +} + +func (f *ObjFile) GetName() string { + return f.sdkFile.Name +} + +func (f *ObjFile) ModTime() time.Time { + return f.modTime +} + +func (f *ObjFile) IsDir() bool { + return f.sdkFile.Dir +} + +func (f *ObjFile) GetHash() utils.HashInfo { + return utils.HashInfo{ + // TODO: support more hash types + } +} + +func (f *ObjFile) GetID() string { + return f.sdkFile.Identity +} + +func (f *ObjFile) GetPath() string { + return f.sdkFile.Path +} + +func (f *ObjFile) CreateTime() time.Time { + return f.createTime +} diff --git a/drivers/halalcloud_open/utils.go b/drivers/halalcloud_open/utils.go new file mode 100644 index 000000000..2cb6706a6 --- /dev/null +++ b/drivers/halalcloud_open/utils.go @@ -0,0 +1,185 @@ +package halalcloudopen + +import ( + "context" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + sdkUserFile "github.com/halalcloud/golang-sdk-lite/halalcloud/services/userfile" + "github.com/ipfs/go-cid" +) + +// get the next chunk +func (oo *openObject) getChunk(_ context.Context) (err error) { + if oo.id >= len(oo.chunks) { + return io.EOF + } + var chunk []byte + err = utils.Retry(3, time.Second, func() (err error) { + chunk, err = getRawFiles(oo.d[oo.id]) + return err + }) + if err != nil { + return err + } + oo.id++ + oo.chunk = chunk + return nil +} + +// Read reads up to len(p) bytes into p. +func (oo *openObject) Read(p []byte) (n int, err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return 0, fmt.Errorf("read on closed file") + } + // Skip data at the start if requested + for oo.skip > 0 { + //size := 1024 * 1024 + _, size, err := oo.ChunkLocation(oo.id) + if err != nil { + return 0, err + } + if oo.skip < int64(size) { + break + } + oo.id++ + oo.skip -= int64(size) + } + if len(oo.chunk) == 0 { + err = oo.getChunk(oo.ctx) + if err != nil { + return 0, err + } + if oo.skip > 0 { + oo.chunk = (oo.chunk)[oo.skip:] + oo.skip = 0 + } + } + n = copy(p, oo.chunk) + oo.shaTemp.Write(p[:n]) + oo.chunk = (oo.chunk)[n:] + return n, nil +} + +// Close closed the file - MAC errors are reported here +func (oo *openObject) Close() (err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return nil + } + // 校验Sha1 + if string(oo.shaTemp.Sum(nil)) != oo.sha { + return fmt.Errorf("failed to finish download: SHA mismatch") + } + + oo.closed = true + return nil +} + +func GetMD5Hash(text string) string { + tHash := md5.Sum([]byte(text)) + return hex.EncodeToString(tHash[:]) +} + +type chunkSize struct { + position int64 + size int +} + +type openObject struct { + ctx context.Context + mu sync.Mutex + d []*sdkUserFile.SliceDownloadInfo + id int + skip int64 + chunk []byte + chunks []chunkSize + closed bool + sha string + shaTemp hash.Hash +} + +func getChunkSizes(sliceSize []*sdkUserFile.SliceSize) (chunks []chunkSize) { + chunks = make([]chunkSize, 0) + for _, s := range sliceSize { + // 对最后一个做特殊处理 + endIndex := s.EndIndex + startIndex := s.StartIndex + if endIndex == 0 { + endIndex = startIndex + } + for j := startIndex; j <= endIndex; j++ { + size := s.Size + chunks = append(chunks, chunkSize{position: j, size: int(size)}) + } + } + return chunks +} + +func (oo *openObject) ChunkLocation(id int) (position int64, size int, err error) { + if id < 0 || id >= len(oo.chunks) { + return 0, 0, errors.New("invalid arguments") + } + + return (oo.chunks)[id].position, (oo.chunks)[id].size, nil +} + +func getRawFiles(addr *sdkUserFile.SliceDownloadInfo) ([]byte, error) { + + if addr == nil { + return nil, errors.New("addr is nil") + } + + client := http.Client{ + Timeout: time.Duration(60 * time.Second), // Set timeout to 60 seconds + } + resp, err := client.Get(addr.DownloadAddress) + if err != nil { + + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bad status: %s, body: %s", resp.Status, body) + } + + if addr.Encrypt > 0 { + cd := uint8(addr.Encrypt) + for idx := 0; idx < len(body); idx++ { + body[idx] = body[idx] ^ cd + } + } + storeType := addr.StoreType + if storeType != 10 { + + sourceCid, err := cid.Decode(addr.Identity) + if err != nil { + return nil, err + } + checkCid, err := sourceCid.Prefix().Sum(body) + if err != nil { + return nil, err + } + if !checkCid.Equals(sourceCid) { + return nil, fmt.Errorf("bad cid: %s, body: %s", checkCid.String(), body) + } + } + + return body, nil + +} diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 1c1473602..e58e01bc2 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -152,8 +152,7 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) req := base.NoRedirectClient.R() req.SetHeaders(map[string]string{ - "Referer": d.conf.site + "/", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", + "Referer": d.conf.site + "/", }) if d.Addition.Ip != "" { req.SetHeader("X-Forwarded-For", d.Addition.Ip) @@ -409,12 +408,15 @@ func (d *ILanZou) GetDetails(ctx context.Context) (*model.StorageDetails, error) if err != nil { return nil, err } - total := utils.Json.Get(res, "map", "totalSize").ToUint64() * 1024 - used := utils.Json.Get(res, "map", "usedSize").ToUint64() * 1024 + vipSize := utils.Json.Get(res, "map", "vipSize").ToInt64() * 1024 + totalSize := utils.Json.Get(res, "map", "totalSize").ToInt64() * 1024 + rewardSize := utils.Json.Get(res, "map", "rewardSize").ToInt64() * 1024 + total := totalSize + rewardSize + vipSize + used := utils.Json.Get(res, "map", "usedSize").ToInt64() * 1024 return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index 3d26ebde2..ea03cfd02 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -71,9 +71,8 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr req.SetHeaders(map[string]string{ "Origin": d.conf.site, "Referer": d.conf.site + "/", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", - "Accept-Encoding": "gzip, deflate, br, zstd", - "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,mt;q=0.5", + "Accept-Encoding": "gzip", + "Accept-Language": "zh-CN,zh;q=0.9,en-US,en;q=0.8", }) if d.Addition.Ip != "" { diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index 6fa269930..01d7c1ece 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -31,7 +31,7 @@ func (d *LanZou) GetAddition() driver.Additional { func (d *LanZou) Init(ctx context.Context) (err error) { if d.UserAgent == "" { - d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/89.0.4389.111 Safari/537.39" + d.UserAgent = base.UserAgentNT } switch d.Type { case "account": diff --git a/drivers/lanzou/meta.go b/drivers/lanzou/meta.go index 50d88082e..fca9b88f5 100644 --- a/drivers/lanzou/meta.go +++ b/drivers/lanzou/meta.go @@ -17,7 +17,7 @@ type Addition struct { SharePassword string `json:"share_password"` BaseUrl string `json:"baseUrl" required:"true" default:"https://pc.woozooo.com" help:"basic URL for file operation"` ShareUrl string `json:"shareUrl" required:"true" default:"https://pan.lanzoui.com" help:"used to get the sharing page"` - UserAgent string `json:"user_agent" required:"true" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/89.0.4389.111 Safari/537.39"` + UserAgent string `json:"user_agent" required:"true" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.39"` RepairFileInfo bool `json:"repair_file_info" help:"To use webdav, you need to enable it"` } diff --git a/drivers/lenovonas_share/driver.go b/drivers/lenovonas_share/driver.go index 0e5045340..012e2e63a 100644 --- a/drivers/lenovonas_share/driver.go +++ b/drivers/lenovonas_share/driver.go @@ -2,6 +2,7 @@ package LenovoNasShare import ( "context" + "fmt" "net/http" "net/url" "strings" @@ -47,12 +48,7 @@ func (d *LenovoNasShare) Drop(ctx context.Context) error { func (d *LenovoNasShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { d.checkStoken() // 检查stoken是否过期 - files := make([]File, 0) - - path := dir.GetPath() - if path == "" && !d.ShowRootFolder && d.RootFolderPath != "" { - path = d.RootFolderPath - } + path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) var resp Files query := map[string]string{ @@ -69,15 +65,14 @@ func (d *LenovoNasShare) List(ctx context.Context, dir model.Obj, args model.Lis return nil, err } - files = append(files, resp.Data.List...) - - return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return utils.SliceConvert(resp.Data.List, func(src File) (model.Obj, error) { if src.IsDir() { return src, nil } return &model.ObjThumb{ Object: model.Object{ Name: src.GetName(), + Path: src.GetPath(), Size: src.GetSize(), Modified: src.ModTime(), IsFolder: src.IsDir(), diff --git a/drivers/local/copy_namedpipes.go b/drivers/local/copy_namedpipes.go new file mode 100644 index 000000000..8217e3e02 --- /dev/null +++ b/drivers/local/copy_namedpipes.go @@ -0,0 +1,16 @@ +//go:build !windows && !plan9 && !netbsd && !aix && !illumos && !solaris && !js + +package local + +import ( + "os" + "path/filepath" + "syscall" +) + +func copyNamedPipe(dstPath string, mode os.FileMode, dirMode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(dstPath), dirMode); err != nil { + return err + } + return syscall.Mkfifo(dstPath, uint32(mode)) +} diff --git a/drivers/local/copy_namedpipes_x.go b/drivers/local/copy_namedpipes_x.go new file mode 100644 index 000000000..81f676f1e --- /dev/null +++ b/drivers/local/copy_namedpipes_x.go @@ -0,0 +1,9 @@ +//go:build windows || plan9 || netbsd || aix || illumos || solaris || js + +package local + +import "os" + +func copyNamedPipe(_ string, _, _ os.FileMode) error { + return nil +} diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 45a9a104a..7189648a7 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -23,7 +23,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/times" - cp "github.com/otiai10/copy" log "github.com/sirupsen/logrus" _ "golang.org/x/image/webp" ) @@ -235,6 +234,7 @@ func (d *Local) Get(ctx context.Context, path string) (model.Obj, error) { func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { fullPath := file.GetPath() link := &model.Link{} + var MFile model.File if args.Type == "thumb" && utils.Ext(file.GetName()) != "svg" { var buf *bytes.Buffer var thumbPath *string @@ -261,9 +261,9 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, err } link.ContentLength = int64(stat.Size()) - link.MFile = open + MFile = open } else { - link.MFile = bytes.NewReader(buf.Bytes()) + MFile = bytes.NewReader(buf.Bytes()) link.ContentLength = int64(buf.Len()) } } else { @@ -272,13 +272,11 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, err } link.ContentLength = file.GetSize() - link.MFile = open - } - link.AddIfCloser(link.MFile) - if !d.Config().OnlyLinkMFile { - link.RangeReader = stream.GetRangeReaderFromMFile(link.ContentLength, link.MFile) - link.MFile = nil + MFile = open } + link.SyncClosers.AddIfCloser(MFile) + link.RangeReader = stream.GetRangeReaderFromMFile(link.ContentLength, MFile) + link.RequireReference = link.SyncClosers.Length() > 0 return link, nil } @@ -298,16 +296,9 @@ func (d *Local) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return fmt.Errorf("the destination folder is a subfolder of the source folder") } err := os.Rename(srcPath, dstPath) - if err != nil && strings.Contains(err.Error(), "invalid cross-device link") { - // 跨设备移动，先复制再删除 - if err := d.Copy(ctx, srcObj, dstDir); err != nil { - return err - } - // 复制成功后直接删除源文件/文件夹 - if srcObj.IsDir() { - return os.RemoveAll(srcObj.GetPath()) - } - return os.Remove(srcObj.GetPath()) + if isCrossDeviceError(err) { + // 跨设备移动，变更为移动任务 + return errs.NotImplement } if err == nil { srcParent := filepath.Dir(srcPath) @@ -348,15 +339,14 @@ func (d *Local) Copy(_ context.Context, srcObj, dstDir model.Obj) error { if utils.IsSubPath(srcPath, dstPath) { return fmt.Errorf("the destination folder is a subfolder of the source folder") } - // Copy using otiai10/copy to perform more secure & efficient copy - err := cp.Copy(srcPath, dstPath, cp.Options{ - Sync: true, // Sync file to disk after copy, may have performance penalty in filesystem such as ZFS - PreserveTimes: true, - PreserveOwner: true, - }) + info, err := os.Lstat(srcPath) if err != nil { return err } + // 复制regular文件会返回errs.NotImplement, 转为复制任务 + if err = d.tryCopy(srcPath, dstPath, info); err != nil { + return err + } if d.directoryMap.Has(filepath.Dir(dstPath)) { d.directoryMap.UpdateDirSize(filepath.Dir(dstPath)) @@ -375,18 +365,26 @@ func (d *Local) Remove(ctx context.Context, obj model.Obj) error { err = os.Remove(obj.GetPath()) } } else { - if !utils.Exists(d.RecycleBinPath) { - err = os.MkdirAll(d.RecycleBinPath, 0o755) + objPath := obj.GetPath() + objName := obj.GetName() + var relPath string + relPath, err = filepath.Rel(d.GetRootPath(), filepath.Dir(objPath)) + if err != nil { + return err + } + recycleBinPath := filepath.Join(d.RecycleBinPath, relPath) + if !utils.Exists(recycleBinPath) { + err = os.MkdirAll(recycleBinPath, 0o755) if err != nil { return err } } - dstPath := filepath.Join(d.RecycleBinPath, obj.GetName()) + dstPath := filepath.Join(recycleBinPath, objName) if utils.Exists(dstPath) { - dstPath = filepath.Join(d.RecycleBinPath, obj.GetName()+"_"+time.Now().Format("20060102150405")) + dstPath = filepath.Join(recycleBinPath, objName+"_"+time.Now().Format("20060102150405")) } - err = os.Rename(obj.GetPath(), dstPath) + err = os.Rename(objPath, dstPath) } if err != nil { return err diff --git a/drivers/local/meta.go b/drivers/local/meta.go index b16f3ca5f..a27e9eecd 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -18,12 +18,12 @@ type Addition struct { } var config = driver.Config{ - Name: "Local", - OnlyLinkMFile: false, - LocalSort: true, - NoCache: true, - DefaultRoot: "/", - NoLinkURL: true, + Name: "Local", + LocalSort: true, + OnlyProxy: true, + NoCache: true, + DefaultRoot: "/", + NoLinkURL: true, } func init() { diff --git a/drivers/local/util.go b/drivers/local/util.go index cbf73ad52..be86d9c9b 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -3,6 +3,7 @@ package local import ( "bytes" "encoding/json" + "errors" "fmt" "io/fs" "os" @@ -14,7 +15,9 @@ import ( "strings" "sync" + "github.com/KarpelesLab/reflink" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/disintegration/imaging" @@ -148,7 +151,7 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { return nil, nil, err } if d.ThumbCacheFolder != "" { - err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0666) + err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0o666) if err != nil { return nil, nil, err } @@ -405,3 +408,79 @@ func (m *DirectoryMap) DeleteDirNode(dirname string) error { return nil } + +func (d *Local) tryCopy(srcPath, dstPath string, info os.FileInfo) error { + if info.Mode()&os.ModeDevice != 0 { + return errors.New("cannot copy a device") + } else if info.Mode()&os.ModeSymlink != 0 { + return d.copySymlink(srcPath, dstPath) + } else if info.Mode()&os.ModeNamedPipe != 0 { + return copyNamedPipe(dstPath, info.Mode(), os.FileMode(d.mkdirPerm)) + } else if info.IsDir() { + return d.recurAndTryCopy(srcPath, dstPath) + } else { + return tryReflinkCopy(srcPath, dstPath) + } +} + +func (d *Local) copySymlink(srcPath, dstPath string) error { + linkOrig, err := os.Readlink(srcPath) + if err != nil { + return err + } + dstDir := filepath.Dir(dstPath) + if !filepath.IsAbs(linkOrig) { + srcDir := filepath.Dir(srcPath) + rel, err := filepath.Rel(dstDir, srcDir) + if err != nil { + rel, err = filepath.Abs(srcDir) + } + if err != nil { + return err + } + linkOrig = filepath.Clean(filepath.Join(rel, linkOrig)) + } + err = os.MkdirAll(dstDir, os.FileMode(d.mkdirPerm)) + if err != nil { + return err + } + return os.Symlink(linkOrig, dstPath) +} + +func (d *Local) recurAndTryCopy(srcPath, dstPath string) error { + err := os.MkdirAll(dstPath, os.FileMode(d.mkdirPerm)) + if err != nil { + return err + } + files, err := readDir(srcPath) + if err != nil { + return err + } + for _, f := range files { + if !f.IsDir() { + sp := filepath.Join(srcPath, f.Name()) + dp := filepath.Join(dstPath, f.Name()) + if err = d.tryCopy(sp, dp, f); err != nil { + return err + } + } + } + for _, f := range files { + if f.IsDir() { + sp := filepath.Join(srcPath, f.Name()) + dp := filepath.Join(dstPath, f.Name()) + if err = d.recurAndTryCopy(sp, dp); err != nil { + return err + } + } + } + return nil +} + +func tryReflinkCopy(srcPath, dstPath string) error { + err := reflink.Always(srcPath, dstPath) + if errors.Is(err, reflink.ErrReflinkUnsupported) || errors.Is(err, reflink.ErrReflinkFailed) || isCrossDeviceError(err) { + return errs.NotImplement + } + return err +} diff --git a/drivers/local/util_unix.go b/drivers/local/util_unix.go index 3362df341..6f16a7c99 100644 --- a/drivers/local/util_unix.go +++ b/drivers/local/util_unix.go @@ -3,11 +3,13 @@ package local import ( + "errors" "io/fs" "strings" "syscall" "github.com/OpenListTeam/OpenList/v4/internal/model" + "golang.org/x/sys/unix" ) func isHidden(f fs.FileInfo, _ string) bool { @@ -20,10 +22,14 @@ func getDiskUsage(path string) (model.DiskUsage, error) { if err != nil { return model.DiskUsage{}, err } - total := stat.Blocks * uint64(stat.Bsize) - free := stat.Bfree * uint64(stat.Bsize) + total := int64(stat.Blocks) * int64(stat.Bsize) + free := int64(stat.Bfree) * int64(stat.Bsize) return model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: total - free, }, nil } + +func isCrossDeviceError(err error) bool { + return errors.Is(err, unix.EXDEV) +} diff --git a/drivers/local/util_windows.go b/drivers/local/util_windows.go index 370640098..9c7b4c070 100644 --- a/drivers/local/util_windows.go +++ b/drivers/local/util_windows.go @@ -45,7 +45,11 @@ func getDiskUsage(path string) (model.DiskUsage, error) { return model.DiskUsage{}, err } return model.DiskUsage{ - TotalSpace: totalBytes, - FreeSpace: freeBytes, + TotalSpace: int64(totalBytes), + UsedSpace: int64(totalBytes - freeBytes), }, nil } + +func isCrossDeviceError(err error) bool { + return errors.Is(err, windows.ERROR_NOT_SAME_DEVICE) +} diff --git a/drivers/mediafire/driver.go b/drivers/mediafire/driver.go index bd2502590..49d431021 100644 --- a/drivers/mediafire/driver.go +++ b/drivers/mediafire/driver.go @@ -60,20 +60,24 @@ func (d *Mediafire) GetAddition() driver.Additional { // Init initializes the MediaFire driver with session token and cookie validation func (d *Mediafire) Init(ctx context.Context) error { - if d.SessionToken == "" { - return fmt.Errorf("Init :: [MediaFire] {critical} missing sessionToken") - } - if d.Cookie == "" { return fmt.Errorf("Init :: [MediaFire] {critical} missing Cookie") } + + // If SessionToken is empty, try to get it from cookie + if d.SessionToken == "" { + if _, err := d.getSessionToken(ctx); err != nil { + return fmt.Errorf("Init :: [MediaFire] {critical} failed to get session token from cookie: %w", err) + } + } + // Setup rate limiter if rate limit is configured if d.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) } + // Validate and refresh session token if needed if _, err := d.getSessionToken(ctx); err != nil { - d.renewToken(ctx) // Avoids 10 mins token expiry (6- 9) @@ -387,8 +391,8 @@ func (d *Mediafire) Put(ctx context.Context, dstDir model.Obj, file model.FileSt } } else { pollKey = checkResp.Response.ResumableUpload.UploadKey - up(100.0) } + defer up(100.0) pollResp, err := d.pollUpload(ctx, pollKey) if err != nil { @@ -412,18 +416,18 @@ func (d *Mediafire) GetDetails(ctx context.Context) (*model.StorageDetails, erro if err != nil { return nil, err } - used, err := strconv.ParseUint(resp.Response.UserInfo.UsedStorageSize, 10, 64) + used, err := strconv.ParseInt(resp.Response.UserInfo.UsedStorageSize, 10, 64) if err != nil { return nil, err } - total, err := strconv.ParseUint(resp.Response.UserInfo.StorageLimit, 10, 64) + total, err := strconv.ParseInt(resp.Response.UserInfo.StorageLimit, 10, 64) if err != nil { return nil, err } return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: total - used, + UsedSpace: used, }, }, nil } diff --git a/drivers/mediafire/meta.go b/drivers/mediafire/meta.go index e80b11f1e..1a3d37c8f 100644 --- a/drivers/mediafire/meta.go +++ b/drivers/mediafire/meta.go @@ -15,6 +15,7 @@ Final opts by @Suyunjing @j2rong4cn @KirCute @Da3zKi7 */ import ( + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/op" ) @@ -23,8 +24,8 @@ type Addition struct { driver.RootPath //driver.RootID - SessionToken string `json:"session_token" required:"true" type:"string" help:"Required for MediaFire API"` - Cookie string `json:"cookie" required:"true" type:"string" help:"Required for navigation"` + SessionToken string `json:"session_token" required:"false" type:"string" help:"Optional for MediaFire API, can be auto-acquired from cookie"` + Cookie string `json:"cookie" required:"true" type:"string" help:"Required for MediaFire API authentication"` OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` @@ -36,7 +37,6 @@ type Addition struct { var config = driver.Config{ Name: "MediaFire", LocalSort: false, - OnlyLinkMFile: false, OnlyProxy: false, NoCache: false, NoUpload: false, @@ -50,13 +50,12 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { return &Mediafire{ - appBase: "https://app.mediafire.com", - apiBase: "https://www.mediafire.com/api/1.5", - hostBase: "https://www.mediafire.com", - maxRetries: 3, - secChUa: "\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"139\", \"Google Chrome\";v=\"139\"", - secChUaPlatform: "Windows", - userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36", + appBase: "https://app.mediafire.com", + apiBase: "https://www.mediafire.com/api/1.5", + hostBase: "https://www.mediafire.com", + maxRetries: 3, + userAgent: base.UserAgent, } }) } + diff --git a/drivers/mediafire/types.go b/drivers/mediafire/types.go index 4a59ae568..eebc85460 100644 --- a/drivers/mediafire/types.go +++ b/drivers/mediafire/types.go @@ -244,3 +244,4 @@ type MediafireUserInfoResponse struct { CurrentAPIVersion string `json:"current_api_version"` } `json:"response"` } + diff --git a/drivers/mediafire/util.go b/drivers/mediafire/util.go index 6ded6bdf0..24259d0bc 100644 --- a/drivers/mediafire/util.go +++ b/drivers/mediafire/util.go @@ -15,6 +15,7 @@ Final opts by @Suyunjing @j2rong4cn @KirCute @Da3zKi7 */ import ( + "compress/gzip" "context" "encoding/json" "fmt" @@ -60,7 +61,7 @@ func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { } req.Header.Set("Accept", "*/*") - req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") + req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("Accept-Language", "en-US,en;q=0.9") req.Header.Set("Content-Length", "0") req.Header.Set("Cookie", d.Cookie) @@ -83,7 +84,19 @@ func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) + var body []byte + // Handle gzip decompression if needed + if resp.Header.Get("Content-Encoding") == "gzip" { + gzipReader, err := gzip.NewReader(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzipReader.Close() + body, _ = io.ReadAll(gzipReader) + } else { + body, err = io.ReadAll(resp.Body) + } + if err != nil { return "", err } @@ -468,39 +481,32 @@ func (d *Mediafire) uploadUnits(ctx context.Context, file model.FileStreamer, ch } var reader io.ReadSeeker - var rateLimitedRd io.Reader var unitHash string // Use lifecycle pattern for proper resource management threadG.GoWithLifecycle(errgroup.Lifecycle{ - Before: func(ctx context.Context) error { + Before: func(ctx context.Context) (err error) { // Skip already uploaded units if d.isUnitUploaded(intWords, unitID) { return ss.DiscardSection(start, size) } - - var err error reader, err = ss.GetSectionReader(start, size) - if err != nil { - return err - } - rateLimitedRd = driver.NewLimitedUploadStream(ctx, reader) - return nil + return }, - Do: func(ctx context.Context) error { + Do: func(ctx context.Context) (err error) { if reader == nil { return nil // Skip if reader is not initialized (already uploaded) } + reader.Seek(0, io.SeekStart) if unitHash == "" { - reader.Seek(0, io.SeekStart) var err error unitHash, err = utils.HashReader(utils.SHA256, reader) if err != nil { return err } + reader.Seek(0, io.SeekStart) } - reader.Seek(0, io.SeekStart) // Perform upload @@ -515,7 +521,7 @@ func (d *Mediafire) uploadUnits(ctx context.Context, file model.FileStreamer, ch } url := d.apiBase + "/upload/resumable.php" - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, rateLimitedRd) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } @@ -579,10 +585,10 @@ func (d *Mediafire) uploadUnits(ctx context.Context, file model.FileStreamer, ch finalUploadKey = uploadResp.Response.Doupload.Key keyMutex.Unlock() + up(float64(threadG.Success()+1) * 100 / float64(numUnits+1)) return nil }, After: func(err error) { - up(float64(threadG.Success()) * 100 / float64(numUnits)) if reader != nil { // Cleanup resources ss.FreeSectionReader(reader) @@ -594,7 +600,6 @@ func (d *Mediafire) uploadUnits(ctx context.Context, file model.FileStreamer, ch if err := threadG.Wait(); err != nil { return "", err } - return finalUploadKey, nil } @@ -727,3 +732,4 @@ func (d *Mediafire) getFileByHash(ctx context.Context, hash string) (*model.ObjT file := resp.Response.FileInfo[0] return d.fileToObj(file), nil } + diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index 996f4dce5..f06b3fa98 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -152,7 +152,7 @@ func (d *Mega) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { func (d *Mega) Remove(ctx context.Context, obj model.Obj) error { if node, ok := obj.(*MegaNode); ok { - return d.c.Delete(node.n, false) + return d.c.Delete(node.n, !d.MoveToTrash) } return fmt.Errorf("unable to convert dir to mega n") } @@ -195,6 +195,19 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea return fmt.Errorf("unable to convert dir to mega n") } +func (d *Mega) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.c.GetQuota() + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: int64(quota.Mstrg), + UsedSpace: int64(quota.Cstrg), + }, + }, nil +} + //func (d *Mega) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/mega/meta.go b/drivers/mega/meta.go index 552c71621..17db29d75 100644 --- a/drivers/mega/meta.go +++ b/drivers/mega/meta.go @@ -13,6 +13,7 @@ type Addition struct { Password string `json:"password" required:"true"` TwoFACode string `json:"two_fa_code" required:"false" help:"2FA 6-digit code, filling in the 2FA code alone will not support reloading driver"` TwoFASecret string `json:"two_fa_secret" required:"false" help:"2FA secret"` + MoveToTrash bool `json:"move_to_trash" default:"true" help:"move to trash when deleting files"` } var config = driver.Config{ diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go index 5e7a0d8db..35ebe96cc 100644 --- a/drivers/misskey/util.go +++ b/drivers/misskey/util.go @@ -57,18 +57,22 @@ func setBody(body interface{}) base.ReqCallback { } func handleFolderId(dir model.Obj) interface{} { - if dir.GetID() == "" { - return nil + if isRootFolder(dir) { + return nil // Root folder doesn't need folderId } return dir.GetID() } +func isRootFolder(dir model.Obj) bool { + return dir.GetID() == "" +} + // API layer methods func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { var files []MFile var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} @@ -85,7 +89,7 @@ func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) { var folders []MFolder var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} @@ -197,16 +201,24 @@ func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileSt Reader: stream, UpdateProgress: up, }) + + // Build form data, only add folderId if not root folder + formData := map[string]string{ + "name": stream.GetName(), + "comment": "", + "isSensitive": "false", + "force": "false", + } + + folderId := handleFolderId(dstDir) + if folderId != nil { + formData["folderId"] = folderId.(string) + } + req := base.RestyClient.R(). SetContext(ctx). SetFileReader("file", stream.GetName(), reader). - SetFormData(map[string]string{ - "folderId": handleFolderId(dstDir).(string), - "name": stream.GetName(), - "comment": "", - "isSensitive": "false", - "force": "false", - }). + SetFormData(formData). SetResult(&file). SetAuthToken(d.AccessToken) diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index c611cf029..bab15c1fd 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -330,7 +330,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if resp.StatusCode != http.StatusOK { return fmt.Errorf("upload err,code=%d", resp.StatusCode) } - up(100 * float64(threadG.Success()) / float64(len(parts))) + up(100 * float64(threadG.Success()+1) / float64(len(parts)+1)) initUpdload.PartInfos[i] = "" return nil }) @@ -356,6 +356,19 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre }, nil } +func (d *MoPan) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.client.UsedSpace() + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: int64(quota.Capacity), + UsedSpace: int64(quota.Used), + }, + }, nil +} + var _ driver.Driver = (*MoPan)(nil) var _ driver.MkdirResult = (*MoPan)(nil) var _ driver.MoveResult = (*MoPan)(nil) diff --git a/drivers/netease_music/driver.go b/drivers/netease_music/driver.go index b03c3b2cc..4fcf433c1 100644 --- a/drivers/netease_music/driver.go +++ b/drivers/netease_music/driver.go @@ -42,14 +42,11 @@ func (d *NeteaseMusic) Drop(ctx context.Context) error { return nil } -func (d *NeteaseMusic) Get(ctx context.Context, path string) (model.Obj, error) { - if path == "/" { - return &model.Object{ - IsFolder: true, - Path: path, - }, nil - } +func (Addition) GetRootPath() string { + return "/" +} +func (d *NeteaseMusic) Get(ctx context.Context, path string) (model.Obj, error) { fragments := strings.Split(path, "/") if len(fragments) > 1 { fileName := fragments[1] diff --git a/drivers/onedrive/driver.go b/drivers/onedrive/driver.go index 0af2f8374..e01c1f7c1 100644 --- a/drivers/onedrive/driver.go +++ b/drivers/onedrive/driver.go @@ -22,6 +22,7 @@ type Onedrive struct { AccessToken string root *Object mutex sync.Mutex + ref *Onedrive } func (d *Onedrive) Config() driver.Config { @@ -36,10 +37,22 @@ func (d *Onedrive) Init(ctx context.Context) error { if d.ChunkSize < 1 { d.ChunkSize = 5 } + if d.ref != nil { + return nil + } return d.refreshToken() } +func (d *Onedrive) InitReference(refStorage driver.Driver) error { + if ref, ok := refStorage.(*Onedrive); ok { + d.ref = ref + return nil + } + return errs.NotSupport +} + func (d *Onedrive) Drop(ctx context.Context) error { + d.ref = nil return nil } @@ -85,7 +98,9 @@ func (d *Onedrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src, dir.GetID()), nil + obj := fileToObj(src, dir.GetID()) + obj.Path = path.Join(dir.GetPath(), obj.GetName()) + return obj, nil }) } @@ -218,9 +233,24 @@ func (d *Onedrive) GetDetails(ctx context.Context) (*model.StorageDetails, error return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: drive.Quota.Total, - FreeSpace: drive.Quota.Remaining, + UsedSpace: drive.Quota.Used, }, }, nil } +func (d *Onedrive) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +// GetDirectUploadInfo returns the direct upload info for OneDrive +func (d *Onedrive) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + return d.getDirectUploadInfo(ctx, path.Join(dstDir.GetPath(), fileName)) +} + var _ driver.Driver = (*Onedrive)(nil) diff --git a/drivers/onedrive/meta.go b/drivers/onedrive/meta.go index 3d9ca09eb..00f46bfcc 100644 --- a/drivers/onedrive/meta.go +++ b/drivers/onedrive/meta.go @@ -7,19 +7,20 @@ import ( type Addition struct { driver.RootPath - Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` - TenantID string `json:"tenant_id"` - IsSharepoint bool `json:"is_sharepoint"` - UseOnlineAPI bool `json:"use_online_api" default:"true"` - APIAddress string `json:"api_url_address" default:"https://api.oplist.org/onedrive/renewapi"` - ClientID string `json:"client_id"` - ClientSecret string `json:"client_secret"` - RedirectUri string `json:"redirect_uri" required:"true" default:"https://api.oplist.org/onedrive/callback"` - RefreshToken string `json:"refresh_token" required:"true"` - SiteId string `json:"site_id"` - ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` - CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` - DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` + IsSharepoint bool `json:"is_sharepoint"` + TenantID string `json:"tenant_id"` + UseOnlineAPI bool `json:"use_online_api" default:"true"` + APIAddress string `json:"api_url_address" default:"https://api.oplist.org/onedrive/renewapi"` + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + RedirectUri string `json:"redirect_uri" required:"true" default:"https://api.oplist.org/onedrive/callback"` + RefreshToken string `json:"refresh_token" required:"true"` + SiteId string `json:"site_id"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` + CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` + DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"` } var config = driver.Config{ diff --git a/drivers/onedrive/types.go b/drivers/onedrive/types.go index a00241eb9..78baa07e4 100644 --- a/drivers/onedrive/types.go +++ b/drivers/onedrive/types.go @@ -95,9 +95,9 @@ type DriveResp struct { DriveType string `json:"driveType"` Quota struct { Deleted uint64 `json:"deleted"` - Remaining uint64 `json:"remaining"` + Remaining int64 `json:"remaining"` State string `json:"state"` - Total uint64 `json:"total"` - Used uint64 `json:"used"` + Total int64 `json:"total"` + Used int64 `json:"used"` } `json:"quota"` } diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 8d71bd418..c3ce9c7dd 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -82,7 +82,6 @@ func (d *Onedrive) _refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, @@ -140,7 +139,10 @@ func (d *Onedrive) _refreshToken() error { return nil } -func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { +func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { + if d.ref != nil { + return d.ref.Request(url, method, callback, resp) + } req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { @@ -156,7 +158,7 @@ func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, return nil, err } if e.Error.Code != "" { - if e.Error.Code == "InvalidAuthenticationToken" { + if e.Error.Code == "InvalidAuthenticationToken" && !utils.IsBool(noRetry...) { err = d.refreshToken() if err != nil { return nil, err @@ -289,6 +291,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil return nil } }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), @@ -314,9 +317,36 @@ func (d *Onedrive) getDrive(ctx context.Context) (*DriveResp, error) { var resp DriveResp _, err := d.Request(api, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, &resp) + }, &resp, true) if err != nil { return nil, err } return &resp, nil } + +func (d *Onedrive) getDirectUploadInfo(ctx context.Context, path string) (*model.HttpDirectUploadInfo, error) { + // Create upload session + url := d.GetMetaUrl(false, path) + "/createUploadSession" + metadata := map[string]any{ + "item": map[string]any{ + "@microsoft.graph.conflictBehavior": "rename", + }, + } + + res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) + }, nil) + if err != nil { + return nil, err + } + + uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() + if uploadUrl == "" { + return nil, fmt.Errorf("failed to get upload URL from response") + } + return &model.HttpDirectUploadInfo{ + UploadURL: uploadUrl, + ChunkSize: d.ChunkSize * 1024 * 1024, // Convert MB to bytes + Method: "PUT", + }, nil +} diff --git a/drivers/onedrive_app/driver.go b/drivers/onedrive_app/driver.go index f28adde0a..063552308 100644 --- a/drivers/onedrive_app/driver.go +++ b/drivers/onedrive_app/driver.go @@ -85,7 +85,9 @@ func (d *OnedriveAPP) List(ctx context.Context, dir model.Obj, args model.ListAr return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src, dir.GetID()), nil + obj := fileToObj(src, dir.GetID()) + obj.Path = path.Join(dir.GetPath(), obj.GetName()) + return obj, nil }) } @@ -217,9 +219,23 @@ func (d *OnedriveAPP) GetDetails(ctx context.Context) (*model.StorageDetails, er return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: drive.Quota.Total, - FreeSpace: drive.Quota.Remaining, + UsedSpace: drive.Quota.Used, }, }, nil } +func (d *OnedriveAPP) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +func (d *OnedriveAPP) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + return d.getDirectUploadInfo(ctx, path.Join(dstDir.GetPath(), fileName)) +} + var _ driver.Driver = (*OnedriveAPP)(nil) diff --git a/drivers/onedrive_app/meta.go b/drivers/onedrive_app/meta.go index 32694e6d7..c3f0c02ec 100644 --- a/drivers/onedrive_app/meta.go +++ b/drivers/onedrive_app/meta.go @@ -7,14 +7,15 @@ import ( type Addition struct { driver.RootPath - Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` - ClientID string `json:"client_id" required:"true"` - ClientSecret string `json:"client_secret" required:"true"` - TenantID string `json:"tenant_id"` - Email string `json:"email"` - ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` - CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` - DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + Region string `json:"region" type:"select" required:"true" options:"global,cn,us,de" default:"global"` + ClientID string `json:"client_id" required:"true"` + ClientSecret string `json:"client_secret" required:"true"` + TenantID string `json:"tenant_id"` + Email string `json:"email"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` + CustomHost string `json:"custom_host" help:"Custom host for onedrive download link"` + DisableDiskUsage bool `json:"disable_disk_usage" default:"false"` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false" help:"Enable direct upload from client to OneDrive"` } var config = driver.Config{ diff --git a/drivers/onedrive_app/types.go b/drivers/onedrive_app/types.go index 438eddfb8..935017853 100644 --- a/drivers/onedrive_app/types.go +++ b/drivers/onedrive_app/types.go @@ -78,9 +78,9 @@ type DriveResp struct { DriveType string `json:"driveType"` Quota struct { Deleted uint64 `json:"deleted"` - Remaining uint64 `json:"remaining"` + Remaining int64 `json:"remaining"` State string `json:"state"` - Total uint64 `json:"total"` - Used uint64 `json:"used"` + Total int64 `json:"total"` + Used int64 `json:"used"` } `json:"quota"` } diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 4cc6b9c31..574afb381 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -88,7 +88,7 @@ func (d *OnedriveAPP) _accessToken() error { return nil } -func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { +func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { @@ -104,7 +104,7 @@ func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallba return nil, err } if e.Error.Code != "" { - if e.Error.Code == "InvalidAuthenticationToken" { + if e.Error.Code == "InvalidAuthenticationToken" && !utils.IsBool(noRetry...) { err = d.accessToken() if err != nil { return nil, err @@ -196,6 +196,7 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. return nil } }, + retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second), @@ -216,9 +217,36 @@ func (d *OnedriveAPP) getDrive(ctx context.Context) (*DriveResp, error) { var resp DriveResp _, err := d.Request(api, http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, &resp) + }, &resp, true) if err != nil { return nil, err } return &resp, nil } + +func (d *OnedriveAPP) getDirectUploadInfo(ctx context.Context, path string) (*model.HttpDirectUploadInfo, error) { + // Create upload session + url := d.GetMetaUrl(false, path) + "/createUploadSession" + metadata := map[string]any{ + "item": map[string]any{ + "@microsoft.graph.conflictBehavior": "rename", + }, + } + + res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) + }, nil) + if err != nil { + return nil, err + } + + uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() + if uploadUrl == "" { + return nil, fmt.Errorf("failed to get upload URL from response") + } + return &model.HttpDirectUploadInfo{ + UploadURL: uploadUrl, + ChunkSize: d.ChunkSize * 1024 * 1024, // Convert MB to bytes + Method: "PUT", + }, nil +} diff --git a/drivers/onedrive_sharelink/driver.go b/drivers/onedrive_sharelink/driver.go index 42d0f1904..0bc3e79b5 100644 --- a/drivers/onedrive_sharelink/driver.go +++ b/drivers/onedrive_sharelink/driver.go @@ -2,21 +2,34 @@ package onedrive_sharelink import ( "context" + "fmt" + "io" + "net/http" + stdpath "path" "strings" + "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/cron" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" log "github.com/sirupsen/logrus" ) +const headerTTL = 25 * time.Minute + type OnedriveSharelink struct { model.Storage cron *cron.Cron Addition + + headerMu sync.RWMutex + sg singleflight.Group[http.Header] } func (d *OnedriveSharelink) Config() driver.Config { @@ -38,17 +51,20 @@ func (d *OnedriveSharelink) Init(ctx context.Context) error { d.cron = cron.NewCron(time.Hour * 1) d.cron.Do(func() { var err error - d.Headers, err = d.getHeaders(ctx) + h, err := d.getHeaders(ctx) if err != nil { log.Errorf("%+v", err) + return } + d.storeHeaders(h) }) // Get initial headers - d.Headers, err = d.getHeaders(ctx) + h, err := d.getHeaders(ctx) if err != nil { return err } + d.storeHeaders(h) return nil } @@ -58,15 +74,16 @@ func (d *OnedriveSharelink) Drop(ctx context.Context) error { } func (d *OnedriveSharelink) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - path := dir.GetPath() - files, err := d.getFiles(ctx, path) + files, err := d.getFiles(ctx, dir.GetPath()) if err != nil { return nil, err } // Convert the slice of files to the required model.Obj format return utils.SliceConvert(files, func(src Item) (model.Obj, error) { - return fileToObj(src), nil + obj := fileToObj(src) + obj.Path = stdpath.Join(dir.GetPath(), obj.GetName()) + return obj, nil }) } @@ -76,21 +93,18 @@ func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model // Cut the first char and the last char uniqueId = uniqueId[1 : len(uniqueId)-1] url := d.downloadLinkPrefix + uniqueId - header := d.Headers - // If the headers are older than 30 minutes, get new headers - if d.HeaderTime < time.Now().Unix()-1800 { - var err error - log.Debug("headers are older than 30 minutes, get new headers") - header, err = d.getHeaders(ctx) - if err != nil { - return nil, err - } + header, err := d.getValidHeaders(ctx) + if err != nil { + return nil, err } return &model.Link{ URL: url, Header: header, + RangeReader: rangeReaderFunc(func(ctx context.Context, hr http_range.Range) (io.ReadCloser, error) { + return d.rangeReadWithRefresh(ctx, url, hr) + }), }, nil } @@ -129,3 +143,102 @@ func (d *OnedriveSharelink) Put(ctx context.Context, dstDir model.Obj, stream mo //} var _ driver.Driver = (*OnedriveSharelink)(nil) + +// rangeReadWithRefresh tries once with current headers, and if the response +// looks invalid (error status or html login page), it refreshes headers and retries. +func (d *OnedriveSharelink) rangeReadWithRefresh(ctx context.Context, url string, hr http_range.Range) (io.ReadCloser, error) { + tryOnce := func(header http.Header) (io.ReadCloser, error) { + h := cloneHeader(header) + if h == nil { + h = http.Header{} + } + h = http_range.ApplyRangeToHttpHeader(hr, h) + resp, err := net.RequestHttp(ctx, http.MethodGet, h, url) + if err != nil { + return nil, err + } + ct := strings.ToLower(resp.Header.Get("Content-Type")) + if strings.Contains(ct, "text/html") { + _ = resp.Body.Close() + return nil, fmt.Errorf("unexpected html response") + } + return resp.Body, nil + } + + header, err := d.getValidHeaders(ctx) + if err != nil { + return nil, err + } + if body, err := tryOnce(header); err == nil { + return body, nil + } + + // refresh and retry once + header, err = d.refreshHeaders(ctx) + if err != nil { + return nil, err + } + return tryOnce(header) +} + +type rangeReaderFunc func(ctx context.Context, hr http_range.Range) (io.ReadCloser, error) + +func (f rangeReaderFunc) RangeRead(ctx context.Context, hr http_range.Range) (io.ReadCloser, error) { + return f(ctx, hr) +} + +func cloneHeader(header http.Header) http.Header { + if header == nil { + return nil + } + return header.Clone() +} + +func (d *OnedriveSharelink) headerSnapshot() http.Header { + d.headerMu.RLock() + defer d.headerMu.RUnlock() + return cloneHeader(d.Headers) +} + +func (d *OnedriveSharelink) storeHeaders(header http.Header) { + if header == nil { + return + } + d.headerMu.Lock() + d.Headers = header + d.HeaderTime = time.Now().Unix() + d.headerMu.Unlock() +} + +func (d *OnedriveSharelink) headersExpired() bool { + d.headerMu.RLock() + defer d.headerMu.RUnlock() + return time.Since(time.Unix(d.HeaderTime, 0)) > headerTTL +} + +func (d *OnedriveSharelink) refreshHeaders(ctx context.Context) (http.Header, error) { + header, err, _ := d.sg.Do("refresh", func() (http.Header, error) { + h, e := d.getHeaders(ctx) + if e != nil { + return nil, e + } + d.storeHeaders(h) + return h, nil + }) + return header, err +} + +func (d *OnedriveSharelink) getValidHeaders(ctx context.Context) (http.Header, error) { + if h := d.headerSnapshot(); h != nil && !d.headersExpired() { + return h, nil + } + h, err := d.refreshHeaders(ctx) + if err != nil { + if h2 := d.headerSnapshot(); h2 != nil { + log.Warnf("onedrive_sharelink: use cached headers after refresh failure: %+v", err) + return h2, nil + } + return nil, err + } + return h, nil +} diff --git a/drivers/openlist/driver.go b/drivers/openlist/driver.go index b539bb5a8..2ca60ff61 100644 --- a/drivers/openlist/driver.go +++ b/drivers/openlist/driver.go @@ -95,6 +95,7 @@ func (d *OpenList) List(ctx context.Context, dir model.Obj, args model.ListArgs) file := model.ObjThumb{ Object: model.Object{ Name: f.Name, + Path: path.Join(dir.GetPath(), f.Name), Modified: f.Modified, Ctime: f.Created, Size: f.Size, @@ -110,19 +111,29 @@ func (d *OpenList) List(ctx context.Context, dir model.Obj, args model.ListArgs) func (d *OpenList) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp common.Resp[FsGetResp] + headers := map[string]string{ + "User-Agent": base.UserAgent, + } // if PassUAToUpsteam is true, then pass the user-agent to the upstream - userAgent := base.UserAgent if d.PassUAToUpsteam { - userAgent = args.Header.Get("user-agent") - if userAgent == "" { - userAgent = base.UserAgent + userAgent := args.Header.Get("user-agent") + if userAgent != "" { + headers["User-Agent"] = userAgent + } + } + // if PassIPToUpsteam is true, then pass the ip address to the upstream + if d.PassIPToUpsteam { + ip := args.IP + if ip != "" { + headers["X-Forwarded-For"] = ip + headers["X-Real-Ip"] = ip } } _, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(FsGetReq{ Path: file.GetPath(), Password: d.MetaPassword, - }).SetHeader("user-agent", userAgent) + }).SetHeaders(headers) }) if err != nil { return nil, err @@ -350,13 +361,21 @@ func (d *OpenList) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.O Name: []string{name}, PutIntoNewDir: args.PutIntoNewDir, SrcDir: dir, + Overwrite: args.Overwrite, }) }) return err } -//func (d *OpenList) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { -// return nil, errs.NotSupport -//} +func (d *OpenList) ResolveLinkCacheMode(_ string) driver.LinkCacheMode { + var mode driver.LinkCacheMode + if d.PassIPToUpsteam { + mode |= driver.LinkCacheIP + } + if d.PassUAToUpsteam { + mode |= driver.LinkCacheUA + } + return mode +} var _ driver.Driver = (*OpenList)(nil) diff --git a/drivers/openlist/meta.go b/drivers/openlist/meta.go index fc76c1424..16c6a155c 100644 --- a/drivers/openlist/meta.go +++ b/drivers/openlist/meta.go @@ -12,6 +12,7 @@ type Addition struct { Username string `json:"username"` Password string `json:"password"` Token string `json:"token"` + PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` } @@ -20,8 +21,8 @@ var config = driver.Config{ Name: "OpenList", LocalSort: true, DefaultRoot: "/", - CheckStatus: true, ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheAuto, } func init() { diff --git a/drivers/openlist/types.go b/drivers/openlist/types.go index f0960d6f6..ffc3e350a 100644 --- a/drivers/openlist/types.go +++ b/drivers/openlist/types.go @@ -167,4 +167,5 @@ type DecompressReq struct { Name []string `json:"name"` PutIntoNewDir bool `json:"put_into_new_dir"` SrcDir string `json:"src_dir"` + Overwrite bool `json:"overwrite"` } diff --git a/drivers/openlist_share/driver.go b/drivers/openlist_share/driver.go index 416987777..0b99edd07 100644 --- a/drivers/openlist_share/driver.go +++ b/drivers/openlist_share/driver.go @@ -68,6 +68,7 @@ func (d *OpenListShare) List(ctx context.Context, dir model.Obj, args model.List file := model.ObjThumb{ Object: model.Object{ Name: f.Name, + Path: stdpath.Join(dir.GetPath(), f.Name), Modified: f.Modified, Ctime: f.Created, Size: f.Size, diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index c728123c1..8b72c3638 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -36,7 +36,6 @@ func (d *PikPak) GetAddition() driver.Additional { } func (d *PikPak) Init(ctx context.Context) (err error) { - if d.Common == nil { d.Common = &Common{ client: base.NewRestyClient(), @@ -247,7 +246,7 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } params := resp.Resumable.Params - //endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") + // endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") // web 端上传 返回的endpoint 为 `mypikpak.net` | android 端上传 返回的endpoint 为 `vip-lixian-07.mypikpak.net`· if d.Addition.Platform == "android" { params.Endpoint = "mypikpak.net" @@ -260,6 +259,30 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return d.UploadByMultipart(ctx, ¶ms, stream.GetSize(), stream, up) } +func (d *PikPak) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := d.request("https://api-drive.mypikpak.com/drive/v1/about", http.MethodGet, func(req *resty.Request) { + req.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + // 离线下载文件 func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { requestBody := base.Json{ @@ -278,7 +301,6 @@ func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir req.SetContext(ctx). SetBody(requestBody) }, &resp) - if err != nil { return nil, err } @@ -325,7 +347,6 @@ func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase [] req.SetContext(ctx). SetQueryParams(params) }, &resp) - if err != nil { return nil, fmt.Errorf("failed to get offline list: %w", err) } diff --git a/drivers/pikpak/types.go b/drivers/pikpak/types.go index bea766fe3..6ae78a455 100644 --- a/drivers/pikpak/types.go +++ b/drivers/pikpak/types.go @@ -78,7 +78,7 @@ type Media struct { type UploadTaskData struct { UploadType string `json:"upload_type"` - //UPLOAD_TYPE_RESUMABLE + // UPLOAD_TYPE_RESUMABLE Resumable *struct { Kind string `json:"kind"` Params S3Params `json:"params"` @@ -195,3 +195,15 @@ type CaptchaTokenResponse struct { ExpiresIn int64 `json:"expires_in"` Url string `json:"url"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + UsageInTrash string `json:"usage_in_trash"` + IsUnlimited bool `json:"is_unlimited"` + Complimentary string `json:"complimentary"` + } `json:"quota"` + ExpiresAt string `json:"expires_at"` + UserType int `json:"user_type"` +} diff --git a/drivers/proton_drive/driver.go b/drivers/proton_drive/driver.go new file mode 100644 index 000000000..936ccee61 --- /dev/null +++ b/drivers/proton_drive/driver.go @@ -0,0 +1,286 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/ProtonMail/gopenpgp/v2/crypto" + proton_api_bridge "github.com/henrybear327/Proton-API-Bridge" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +type ProtonDrive struct { + model.Storage + Addition + + protonDrive *proton_api_bridge.ProtonDrive + + apiBase string + appVersion string + protonJson string + userAgent string + sdkVersion string + webDriveAV string + + c *proton.Client + + // userKR *crypto.KeyRing + addrKRs map[string]*crypto.KeyRing + addrData map[string]proton.Address + + MainShare *proton.Share + + DefaultAddrKR *crypto.KeyRing + MainShareKR *crypto.KeyRing +} + +func (d *ProtonDrive) Config() driver.Config { + return config +} + +func (d *ProtonDrive) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ProtonDrive) Init(ctx context.Context) (err error) { + defer func() { + if r := recover(); err == nil && r != nil { + err = fmt.Errorf("ProtonDrive initialization panic: %v", r) + } + }() + + if d.Email == "" { + return fmt.Errorf("email is required") + } + if d.Password == "" { + return fmt.Errorf("password is required") + } + + config := &common.Config{ + AppVersion: d.appVersion, + UserAgent: d.userAgent, + FirstLoginCredential: &common.FirstLoginCredentialData{ + Username: d.Email, + Password: d.Password, + TwoFA: d.TwoFACode, + }, + EnableCaching: true, + ConcurrentBlockUploadCount: setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers), + //ConcurrentFileCryptoCount: 2, + UseReusableLogin: d.UseReusableLogin && d.ReusableCredential != (common.ReusableCredentialData{}), + ReplaceExistingDraft: true, + ReusableCredential: &d.ReusableCredential, + } + + protonDrive, _, err := proton_api_bridge.NewProtonDrive( + ctx, + config, + d.authHandler, + func() {}, + ) + + if err != nil && config.UseReusableLogin { + config.UseReusableLogin = false + protonDrive, _, err = proton_api_bridge.NewProtonDrive(ctx, + config, + d.authHandler, + func() {}, + ) + if err == nil { + op.MustSaveDriverStorage(d) + } + } + + if err != nil { + return fmt.Errorf("failed to initialize ProtonDrive: %w", err) + } + + if err := d.initClient(ctx); err != nil { + return err + } + + d.protonDrive = protonDrive + d.MainShare = protonDrive.MainShare + if d.RootFolderID == "root" || d.RootFolderID == "" { + d.RootFolderID = protonDrive.RootLink.LinkID + } + d.MainShareKR = protonDrive.MainShareKR + d.DefaultAddrKR = protonDrive.DefaultAddrKR + + return nil +} + +func (d *ProtonDrive) Drop(ctx context.Context) error { + return nil +} + +func (d *ProtonDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + entries, err := d.protonDrive.ListDirectory(ctx, dir.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + } + + objects := make([]model.Obj, 0, len(entries)) + for _, entry := range entries { + obj := &model.Object{ + ID: entry.Link.LinkID, + Name: entry.Name, + Size: entry.Link.Size, + Modified: time.Unix(entry.Link.ModifyTime, 0), + IsFolder: entry.IsFolder, + } + objects = append(objects, obj) + } + + return objects, nil +} + +func (d *ProtonDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + link, err := d.getLink(ctx, file.GetID()) + if err != nil { + return nil, fmt.Errorf("failed get file link: %+v", err) + } + fileSystemAttrs, err := d.protonDrive.GetActiveRevisionAttrs(ctx, link) + if err != nil { + return nil, fmt.Errorf("failed get file revision: %+v", err) + } + // 解密后的文件大小 + size := fileSystemAttrs.Size + + rangeReaderFunc := func(rangeCtx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + length := httpRange.Length + if length < 0 || httpRange.Start+length > size { + length = size - httpRange.Start + } + reader, _, _, err := d.protonDrive.DownloadFile(rangeCtx, link, httpRange.Start) + if err != nil { + return nil, fmt.Errorf("failed start download: %+v", err) + } + return utils.ReadCloser{ + Reader: io.LimitReader(reader, length), + Closer: reader, + }, nil + } + + expiration := time.Minute + return &model.Link{ + RangeReader: stream.RateLimitRangeReaderFunc(rangeReaderFunc), + ContentLength: size, + Expiration: &expiration, + }, nil +} + +func (d *ProtonDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + id, err := d.protonDrive.CreateNewFolderByID(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } + + newDir := &model.Object{ + ID: id, + Name: dirName, + IsFolder: true, + Modified: time.Now(), + } + return newDir, nil +} + +func (d *ProtonDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.DirectMove(ctx, srcObj, dstDir) +} + +func (d *ProtonDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + return d.DirectRename(ctx, srcObj, newName) +} + +func (d *ProtonDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, fmt.Errorf("directory copy not supported") + } + + srcLink, err := d.getLink(ctx, srcObj.GetID()) + if err != nil { + return nil, err + } + + reader, linkSize, fileSystemAttrs, err := d.protonDrive.DownloadFile(ctx, srcLink, 0) + if err != nil { + return nil, fmt.Errorf("failed to download source file: %w", err) + } + defer reader.Close() + + actualSize := linkSize + if fileSystemAttrs != nil && fileSystemAttrs.Size > 0 { + actualSize = fileSystemAttrs.Size + } + + file := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{ + Name: srcObj.GetName(), + // Use the accurate and real size + Size: actualSize, + Modified: srcObj.ModTime(), + }, + Reader: reader, + } + defer file.Close() + return d.Put(ctx, dstDir, file, func(percentage float64) {}) +} + +func (d *ProtonDrive) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + return d.protonDrive.MoveFolderToTrashByID(ctx, obj.GetID(), false) + } else { + return d.protonDrive.MoveFileToTrashByID(ctx, obj.GetID()) + } +} + +func (d *ProtonDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return d.uploadFile(ctx, dstDir.GetID(), file, up) +} + +func (d *ProtonDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + about, err := d.protonDrive.About(ctx) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: about.MaxSpace, + UsedSpace: about.UsedSpace, + }, + }, nil +} + +var _ driver.Driver = (*ProtonDrive)(nil) diff --git a/drivers/proton_drive/meta.go b/drivers/proton_drive/meta.go new file mode 100644 index 000000000..acf656779 --- /dev/null +++ b/drivers/proton_drive/meta.go @@ -0,0 +1,56 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/henrybear327/Proton-API-Bridge/common" +) + +type Addition struct { + driver.RootID + Email string `json:"email" required:"true" type:"string"` + Password string `json:"password" required:"true" type:"string"` + TwoFACode string `json:"two_fa_code" type:"string"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"100"` + UseReusableLogin bool `json:"use_reusable_login" type:"bool" default:"true" help:"Use reusable login credentials instead of username/password"` + ReusableCredential common.ReusableCredentialData +} + +var config = driver.Config{ + Name: "ProtonDrive", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: "root", + NoLinkURL: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ProtonDrive{ + Addition: Addition{ + UseReusableLogin: true, + }, + apiBase: "https://drive.proton.me/api", + appVersion: "windows-drive@1.11.3+rclone+proton", + protonJson: "application/vnd.protonmail.v1+json", + sdkVersion: "js@0.3.0", + userAgent: "ProtonDrive/v1.70.0 (Windows NT 10.0.22000; Win64; x64)", + webDriveAV: "web-drive@5.2.0+0f69f7a8", + } + }) +} diff --git a/drivers/proton_drive/types.go b/drivers/proton_drive/types.go new file mode 100644 index 000000000..ed6dcd192 --- /dev/null +++ b/drivers/proton_drive/types.go @@ -0,0 +1,38 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +type MoveRequest struct { + ParentLinkID string `json:"ParentLinkID"` + NodePassphrase string `json:"NodePassphrase"` + NodePassphraseSignature *string `json:"NodePassphraseSignature"` + Name string `json:"Name"` + NameSignatureEmail string `json:"NameSignatureEmail"` + Hash string `json:"Hash"` + OriginalHash string `json:"OriginalHash"` + ContentHash *string `json:"ContentHash"` // Maybe null +} + +type RenameRequest struct { + Name string `json:"Name"` // PGP encrypted name + NameSignatureEmail string `json:"NameSignatureEmail"` // User's signature email + Hash string `json:"Hash"` // New name hash + OriginalHash string `json:"OriginalHash"` // Current name hash +} + +type RenameResponse struct { + Code int `json:"Code"` +} diff --git a/drivers/proton_drive/util.go b/drivers/proton_drive/util.go new file mode 100644 index 000000000..a59bb6cb9 --- /dev/null +++ b/drivers/proton_drive/util.go @@ -0,0 +1,670 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/henrybear327/go-proton-api" +) + +func (d *ProtonDrive) uploadFile(ctx context.Context, parentLinkID string, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + _, err := d.getLink(ctx, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to get parent link: %w", err) + } + + var reader io.Reader + // Use buffered reader with larger buffer for better performance + var bufferSize int + + // File > 100MB (default) + if file.GetSize() > d.ChunkSize*1024*1024 { + // 256KB for large files + bufferSize = 256 * 1024 + // File > 10MB + } else if file.GetSize() > 10*1024*1024 { + // 128KB for medium files + bufferSize = 128 * 1024 + } else { + // 64KB for small files + bufferSize = 64 * 1024 + } + + // reader = bufio.NewReader(file) + reader = bufio.NewReaderSize(file, bufferSize) + reader = &driver.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: reader, + Size: file.GetSize(), + }, + UpdateProgress: up, + } + reader = driver.NewLimitedUploadStream(ctx, reader) + + id, _, err := d.protonDrive.UploadFileByReader(ctx, parentLinkID, file.GetName(), file.ModTime(), reader, 0) + if err != nil { + return nil, fmt.Errorf("failed to upload file: %w", err) + } + + return &model.Object{ + ID: id, + Name: file.GetName(), + Size: file.GetSize(), + Modified: file.ModTime(), + IsFolder: false, + }, nil +} + +func (d *ProtonDrive) encryptFileName(ctx context.Context, name string, parentLinkID string) (string, error) { + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Temporary file (request) + tempReq := proton.CreateFileReq{ + SignatureAddress: d.MainShare.Creator, + } + + // Encrypt the filename + err = tempReq.SetName(name, d.DefaultAddrKR, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to encrypt filename: %w", err) + } + + return tempReq.Name, nil +} + +func (d *ProtonDrive) generateFileNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func (d *ProtonDrive) getOriginalNameHash(link *proton.Link) (string, error) { + if link == nil { + return "", fmt.Errorf("link cannot be nil") + } + + if link.Hash == "" { + return "", fmt.Errorf("link hash is empty") + } + + return link.Hash, nil +} + +func (d *ProtonDrive) getLink(ctx context.Context, linkID string) (*proton.Link, error) { + if linkID == "" { + return nil, fmt.Errorf("linkID cannot be empty") + } + + link, err := d.c.GetLink(ctx, d.MainShare.ShareID, linkID) + if err != nil { + return nil, err + } + + return &link, nil +} + +func (d *ProtonDrive) getLinkKR(ctx context.Context, link *proton.Link) (*crypto.KeyRing, error) { + if link == nil { + return nil, fmt.Errorf("link cannot be nil") + } + + // Root Link or Root Dir + if link.ParentLinkID == "" { + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + return link.GetKeyRing(d.MainShareKR, signatureVerificationKR) + } + + // Get parent keyring recursively + parentLink, err := d.getLink(ctx, link.ParentLinkID) + if err != nil { + return nil, err + } + + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return nil, err + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + + return link.GetKeyRing(parentNodeKR, signatureVerificationKR) +} + +var ( + ErrKeyPassOrSaltedKeyPassMustBeNotNil = errors.New("either keyPass or saltedKeyPass must be not nil") + ErrFailedToUnlockUserKeys = errors.New("failed to unlock user keys") +) + +func getAccountKRs(ctx context.Context, c *proton.Client, keyPass, saltedKeyPass []byte) (*crypto.KeyRing, map[string]*crypto.KeyRing, map[string]proton.Address, []byte, error) { + user, err := c.GetUser(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("user %#v", user) + + addrsArr, err := c.GetAddresses(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("addr %#v", addr) + + if saltedKeyPass == nil { + if keyPass == nil { + return nil, nil, nil, nil, ErrKeyPassOrSaltedKeyPassMustBeNotNil + } + + // Due to limitations, salts are stored using cacheCredentialToFile + salts, err := c.GetSalts(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("salts %#v", salts) + + saltedKeyPass, err = salts.SaltForKey(keyPass, user.Keys.Primary().ID) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("saltedKeyPass ok") + } + + userKR, addrKRs, err := proton.Unlock(user, addrsArr, saltedKeyPass, nil) + if err != nil { + return nil, nil, nil, nil, err + } else if userKR.CountDecryptionEntities() == 0 { + return nil, nil, nil, nil, ErrFailedToUnlockUserKeys + } + + addrs := make(map[string]proton.Address) + for _, addr := range addrsArr { + addrs[addr.Email] = addr + } + + return userKR, addrKRs, addrs, saltedKeyPass, nil +} + +func (d *ProtonDrive) getSignatureVerificationKeyring(emailAddresses []string, verificationAddrKRs ...*crypto.KeyRing) (*crypto.KeyRing, error) { + ret, err := crypto.NewKeyRing(nil) + if err != nil { + return nil, err + } + + for _, emailAddress := range emailAddresses { + if addr, ok := d.addrData[emailAddress]; ok { + if addrKR, exists := d.addrKRs[addr.ID]; exists { + err = d.addKeysFromKR(ret, addrKR) + if err != nil { + return nil, err + } + } + } + } + + for _, kr := range verificationAddrKRs { + err = d.addKeysFromKR(ret, kr) + if err != nil { + return nil, err + } + } + + if ret.CountEntities() == 0 { + return nil, fmt.Errorf("no keyring for signature verification") + } + + return ret, nil +} + +func (d *ProtonDrive) addKeysFromKR(kr *crypto.KeyRing, newKRs ...*crypto.KeyRing) error { + for i := range newKRs { + for _, key := range newKRs[i].GetKeys() { + err := kr.AddKey(key) + if err != nil { + return err + } + } + } + return nil +} + +func (d *ProtonDrive) DirectRename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + // fmt.Printf("DEBUG DirectRename: path=%s, newName=%s", srcObj.GetPath(), newName) + + if d.MainShare == nil || d.DefaultAddrKR == nil { + return nil, fmt.Errorf("missing required fields: MainShare=%v, DefaultAddrKR=%v", + d.MainShare != nil, d.DefaultAddrKR != nil) + } + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + srcLink, err := d.getLink(ctx, srcObj.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + parentLinkID := srcLink.ParentLinkID + if parentLinkID == "" { + return nil, fmt.Errorf("cannot rename root folder") + } + + encryptedName, err := d.encryptFileName(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateFileNameHash(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + renameReq := RenameRequest{ + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + } + + err = d.executeRenameAPI(ctx, srcLink.LinkID, renameReq) + if err != nil { + return nil, fmt.Errorf("rename API call failed: %w", err) + } + + return &model.Object{ + ID: srcLink.LinkID, + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) executeRenameAPI(ctx context.Context, linkID string, req RenameRequest) error { + renameURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/rename", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal rename request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", renameURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.ReusableCredential.UID) + httpReq.Header.Set("Authorization", "Bearer "+d.ReusableCredential.AccessToken) + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute rename request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("rename failed with status %d", resp.StatusCode) + } + + var renameResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&renameResp); err != nil { + return fmt.Errorf("failed to decode rename response: %w", err) + } + + if renameResp.Code != 1000 { + return fmt.Errorf("rename failed with code %d", renameResp.Code) + } + + return nil +} + +func (d *ProtonDrive) executeMoveAPI(ctx context.Context, linkID string, req MoveRequest) error { + // fmt.Printf("DEBUG Move Request - Name: %s\n", req.Name) + // fmt.Printf("DEBUG Move Request - Hash: %s\n", req.Hash) + // fmt.Printf("DEBUG Move Request - OriginalHash: %s\n", req.OriginalHash) + // fmt.Printf("DEBUG Move Request - ParentLinkID: %s\n", req.ParentLinkID) + + // fmt.Printf("DEBUG Move Request - Name length: %d\n", len(req.Name)) + // fmt.Printf("DEBUG Move Request - NameSignatureEmail: %s\n", req.NameSignatureEmail) + // fmt.Printf("DEBUG Move Request - ContentHash: %v\n", req.ContentHash) + // fmt.Printf("DEBUG Move Request - NodePassphrase length: %d\n", len(req.NodePassphrase)) + // fmt.Printf("DEBUG Move Request - NodePassphraseSignature length: %d\n", len(req.NodePassphraseSignature)) + + // fmt.Printf("DEBUG Move Request - SrcLinkID: %s\n", linkID) + // fmt.Printf("DEBUG Move Request - DstParentLinkID: %s\n", req.ParentLinkID) + // fmt.Printf("DEBUG Move Request - ShareID: %s\n", d.MainShare.ShareID) + + srcLink, _ := d.getLink(ctx, linkID) + if srcLink != nil && srcLink.ParentLinkID == req.ParentLinkID { + return fmt.Errorf("cannot move to same parent directory") + } + + moveURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/move", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal move request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", moveURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Authorization", "Bearer "+d.ReusableCredential.AccessToken) + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.ReusableCredential.UID) + httpReq.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute move request: %w", err) + } + defer resp.Body.Close() + + var moveResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&moveResp); err != nil { + return fmt.Errorf("failed to decode move response: %w", err) + } + + if moveResp.Code != 1000 { + return fmt.Errorf("move operation failed with code: %d", moveResp.Code) + } + + return nil +} + +func (d *ProtonDrive) DirectMove(ctx context.Context, srcObj model.Obj, dstDir model.Obj) (model.Obj, error) { + // fmt.Printf("DEBUG DirectMove: srcPath=%s, dstPath=%s", srcObj.GetPath(), dstDir.GetPath()) + + srcLink, err := d.getLink(ctx, srcObj.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + dstParentLinkID := dstDir.GetID() + + if srcObj.IsDir() { + // Check if destination is a descendant of source + if err := d.checkCircularMove(ctx, srcLink.LinkID, dstParentLinkID); err != nil { + return nil, err + } + } + + // Encrypt the filename for the new location + encryptedName, err := d.encryptFileName(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateNameHash(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + // Re-encrypt node passphrase for new parent context + reencryptedPassphrase, err := d.reencryptNodePassphrase(ctx, srcLink, dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to re-encrypt node passphrase: %w", err) + } + + moveReq := MoveRequest{ + ParentLinkID: dstParentLinkID, + NodePassphrase: reencryptedPassphrase, + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + ContentHash: nil, + + // *** Causes rejection *** + /* NodePassphraseSignature: srcLink.NodePassphraseSignature, */ + } + + //fmt.Printf("DEBUG MoveRequest validation:\n") + //fmt.Printf(" Name length: %d\n", len(moveReq.Name)) + //fmt.Printf(" Hash: %s\n", moveReq.Hash) + //fmt.Printf(" OriginalHash: %s\n", moveReq.OriginalHash) + //fmt.Printf(" NodePassphrase length: %d\n", len(moveReq.NodePassphrase)) + /* fmt.Printf(" NodePassphraseSignature length: %d\n", len(moveReq.NodePassphraseSignature)) */ + //fmt.Printf(" NameSignatureEmail: %s\n", moveReq.NameSignatureEmail) + + err = d.executeMoveAPI(ctx, srcLink.LinkID, moveReq) + if err != nil { + return nil, fmt.Errorf("move API call failed: %w", err) + } + + return &model.Object{ + ID: srcLink.LinkID, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) reencryptNodePassphrase(ctx context.Context, srcLink *proton.Link, dstParentLinkID string) (string, error) { + // Get source parent link with metadata + srcParentLink, err := d.getLink(ctx, srcLink.ParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get source parent link: %w", err) + } + + // Get source parent keyring using link object + srcParentKR, err := d.getLinkKR(ctx, srcParentLink) + if err != nil { + return "", fmt.Errorf("failed to get source parent keyring: %w", err) + } + + // Get destination parent link with metadata + dstParentLink, err := d.getLink(ctx, dstParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get destination parent link: %w", err) + } + + // Get destination parent keyring using link object + dstParentKR, err := d.getLinkKR(ctx, dstParentLink) + if err != nil { + return "", fmt.Errorf("failed to get destination parent keyring: %w", err) + } + + // Re-encrypt the node passphrase from source parent context to destination parent context + reencryptedPassphrase, err := reencryptKeyPacket(srcParentKR, dstParentKR, d.DefaultAddrKR, srcLink.NodePassphrase) + if err != nil { + return "", fmt.Errorf("failed to re-encrypt key packet: %w", err) + } + + return reencryptedPassphrase, nil +} + +func (d *ProtonDrive) generateNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Get signature verification keyring + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func reencryptKeyPacket(srcKR, dstKR, _ *crypto.KeyRing, passphrase string) (string, error) { // addrKR (3) + oldSplitMessage, err := crypto.NewPGPSplitMessageFromArmored(passphrase) + if err != nil { + return "", err + } + + sessionKey, err := srcKR.DecryptSessionKey(oldSplitMessage.KeyPacket) + if err != nil { + return "", err + } + + newKeyPacket, err := dstKR.EncryptSessionKey(sessionKey) + if err != nil { + return "", err + } + + newSplitMessage := crypto.NewPGPSplitMessage(newKeyPacket, oldSplitMessage.DataPacket) + + return newSplitMessage.GetArmored() +} + +func (d *ProtonDrive) checkCircularMove(ctx context.Context, srcLinkID, dstParentLinkID string) error { + currentLinkID := dstParentLinkID + + for currentLinkID != "" && currentLinkID != d.RootFolderID { + if currentLinkID == srcLinkID { + return fmt.Errorf("cannot move folder into itself or its subfolder") + } + + currentLink, err := d.getLink(ctx, currentLinkID) + if err != nil { + return err + } + currentLinkID = currentLink.ParentLinkID + } + + return nil +} + +func (d *ProtonDrive) authHandler(auth proton.Auth) { + if auth.AccessToken != d.ReusableCredential.AccessToken || auth.RefreshToken != d.ReusableCredential.RefreshToken { + d.ReusableCredential.UID = auth.UID + d.ReusableCredential.AccessToken = auth.AccessToken + d.ReusableCredential.RefreshToken = auth.RefreshToken + + if err := d.initClient(context.Background()); err != nil { + fmt.Printf("ProtonDrive: failed to reinitialize client after auth refresh: %v\n", err) + } + + op.MustSaveDriverStorage(d) + } +} + +func (d *ProtonDrive) initClient(ctx context.Context) error { + clientOptions := []proton.Option{ + proton.WithAppVersion(d.appVersion), + proton.WithUserAgent(d.userAgent), + } + manager := proton.New(clientOptions...) + d.c = manager.NewClient(d.ReusableCredential.UID, d.ReusableCredential.AccessToken, d.ReusableCredential.RefreshToken) + + saltedKeyPassBytes, err := base64.StdEncoding.DecodeString(d.ReusableCredential.SaltedKeyPass) + if err != nil { + return fmt.Errorf("failed to decode salted key pass: %w", err) + } + + _, addrKRs, addrs, _, err := getAccountKRs(ctx, d.c, nil, saltedKeyPassBytes) + if err != nil { + return fmt.Errorf("failed to get account keyrings: %w", err) + } + + d.addrKRs = addrKRs + d.addrData = addrs + + return nil +} diff --git a/drivers/qihoo360/driver.go b/drivers/qihoo360/driver.go new file mode 100644 index 000000000..0268c6758 --- /dev/null +++ b/drivers/qihoo360/driver.go @@ -0,0 +1,678 @@ +package qihoo360 + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type Qihoo360 struct { + model.Storage + Addition + authInfo *AuthResp + authExpire int64 +} + +func (d *Qihoo360) Config() driver.Config { + return config +} + +func (d *Qihoo360) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Qihoo360) Init(ctx context.Context) error { + // Test authentication + _, err := d.getAuth() + return err +} + +func (d *Qihoo360) Drop(ctx context.Context) error { + d.authInfo = nil + return nil +} + +func (d *Qihoo360) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + path := dir.GetPath() + if path == "" { + path = d.RootFolderPath + } + if path == "" { + path = "/" + } + + // Ensure directory paths end with / (required by API for non-root paths) + if path != "/" && !strings.HasSuffix(path, "/") { + path += "/" + } + + files, err := d.getFiles(path, 0, 100) + if err != nil { + return nil, err + } + + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return src, nil + }) +} + +func (d *Qihoo360) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + // Get file ID (nid) + nid := file.GetID() + if nid == "" { + return nil, fmt.Errorf("file id is empty") + } + + // Get download URL from API + downloadUrl, err := d.getDownloadUrl(nid) + if err != nil { + return nil, err + } + + if downloadUrl == "" { + return nil, fmt.Errorf("download url is empty") + } + + return &model.Link{ + URL: downloadUrl, + Header: http.Header{ + "User-Agent": []string{"yunpan_mcp_server"}, + }, + }, nil +} + +func (d *Qihoo360) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + userDetail, err := d.getUserDetail() + if err != nil { + return nil, err + } + + // Parse total and used sizes from strings to int64 + totalSize := int64(0) + usedSize := int64(0) + + if total, err := strconv.ParseInt(userDetail.Data.TotalSize, 10, 64); err == nil { + totalSize = total + } + if used, err := strconv.ParseInt(userDetail.Data.UsedSize, 10, 64); err == nil { + usedSize = used + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: totalSize, + UsedSpace: usedSize, + }, + }, nil +} + +func (d *Qihoo360) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + path := parentDir.GetPath() + if path == "" { + path = d.RootFolderPath + } + if path == "" { + path = "/" + } + + // Ensure path ends with / + if path[len(path)-1] != '/' { + path += "/" + } + // Ensure dirName ends with / + if dirName[len(dirName)-1] != '/' { + dirName += "/" + } + + fname := path + dirName + + params := map[string]string{ + "fname": fname, + } + + var resp CommonResp + _, err := d.request("File.makeDir", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("make dir failed: %s", resp.Errmsg) + } + + return nil, nil +} + +func (d *Qihoo360) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcPath := srcObj.GetPath() + if srcPath == "" { + // Try to construct path from name + srcPath = d.RootFolderPath + if srcPath == "" { + srcPath = "/" + } + if srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + srcPath += srcObj.GetName() + if srcObj.IsDir() && srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + } + + dstPath := dstDir.GetPath() + if dstPath == "" { + dstPath = d.RootFolderPath + } + if dstPath == "" { + dstPath = "/" + } + if dstPath[len(dstPath)-1] != '/' { + dstPath += "/" + } + + params := map[string]string{ + "src_name": srcPath, + "new_name": dstPath, + } + + var resp CommonResp + _, err := d.request("File.move", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("move failed: %s", resp.Errmsg) + } + + return nil, nil +} + +func (d *Qihoo360) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + srcPath := srcObj.GetPath() + if srcPath == "" { + // Try to construct path from name + srcPath = d.RootFolderPath + if srcPath == "" { + srcPath = "/" + } + if srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + srcPath += srcObj.GetName() + if srcObj.IsDir() && srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + } + + // new_name should be just the name, not full path + if srcObj.IsDir() && newName[len(newName)-1] != '/' { + newName += "/" + } + + params := map[string]string{ + "src_name": srcPath, + "new_name": newName, + } + + var resp CommonResp + _, err := d.request("File.rename", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("rename failed: %s", resp.Errmsg) + } + + return nil, nil +} + +func (d *Qihoo360) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // Copy is not documented in ecs_mcp_server + return nil, errs.NotSupport +} + +func (d *Qihoo360) Remove(ctx context.Context, obj model.Obj) error { + srcPath := obj.GetPath() + if srcPath == "" { + // Try to construct path from name + srcPath = d.RootFolderPath + if srcPath == "" { + srcPath = "/" + } + if srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + srcPath += obj.GetName() + if obj.IsDir() && srcPath[len(srcPath)-1] != '/' { + srcPath += "/" + } + } + + params := map[string]string{ + "fname": srcPath, + } + + var resp CommonResp + // fname parameter is excluded from sign calculation + _, err := d.request("File.delete", params, &resp, "fname") + if err != nil { + return err + } + + if resp.Errno != 0 { + return fmt.Errorf("remove failed: %s", resp.Errmsg) + } + + return nil +} + +func (d *Qihoo360) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + dstPath := dstDir.GetPath() + if dstPath == "" { + dstPath = d.RootFolderPath + } + if dstPath == "" { + dstPath = "/" + } + if dstPath[len(dstPath)-1] != '/' { + dstPath += "/" + } + + fname := dstPath + file.GetName() + fsize := file.GetSize() + now := time.Now().Unix() + + // Calculate file hash + const chunkSize = 524288 // 512KB per chunk + numChunks := (fsize + chunkSize - 1) / chunkSize + + var blockHashes []string + var blocks []struct { + data []byte + offset int64 + size int64 + hash string + } + + // Read file and calculate chunk hashes + for i := int64(0); i < numChunks; i++ { + size := chunkSize + if i == numChunks-1 { + size = int(fsize - i*chunkSize) + } + + buf := make([]byte, size) + n, err := io.ReadFull(file, buf) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + buf = buf[:n] + + // Calculate SHA1 hash for this block + hash := sha1.Sum(buf) + blockHash := hex.EncodeToString(hash[:]) + blockHashes = append(blockHashes, blockHash) + + blocks = append(blocks, struct { + data []byte + offset int64 + size int64 + hash string + }{ + data: buf, + offset: i * chunkSize, + size: int64(n), + hash: blockHash, + }) + } + + // Calculate file hash (SHA1 of concatenated block hashes) + fhashStr := strings.Join(blockHashes, "") + fhash := sha1.Sum([]byte(fhashStr)) + fhashHex := hex.EncodeToString(fhash[:]) + + // Get upload address + uploadAddr, err := d.getUploadAddr(fname, fsize, fhashHex, now, now) + if err != nil { + return nil, err + } + + // Check for instant upload (server already has the file) + // When file exists, HTTP is null + httpVal, httpOk := uploadAddr.Data.HTTP.(string) + if !httpOk || httpVal == "" { + return nil, nil // Instant upload success (file already exists) + } + + // Build upload host + uploadHost := httpVal + if uploadAddr.Data.IsHttps == 1 { + uploadHost = "https://" + uploadHost + } else { + uploadHost = "http://" + uploadHost + } + + // Get upload token + var tk string + if tkVal, ok := uploadAddr.Data.Tk.(string); ok { + tk = tkVal + } + + // Prepare block info for preload + blockInfoList := make([]BlockInfo, len(blocks)) + for i, block := range blocks { + blockInfoList[i] = BlockInfo{ + BHash: block.hash, + BIdx: i + 1, + BOffset: block.offset, + BSize: block.size, + } + } + + // Preload - send block info + preloadResp, err := d.preloadBlocks(ctx, uploadHost, fname, fsize, fhashHex, now, now, tk, blockInfoList) + if err != nil { + return nil, err + } + + // Upload each block + for i, block := range blocks { + blockInfo := preloadResp.Data.BlockInfo[i] + // Note: use user token (d.authInfo.Data.Token), not blockInfo.Token + err = d.uploadBlock(ctx, uploadHost, block.data, block.hash, i+1, block.offset, block.size, + fname, fsize, blockInfo.Q, blockInfo.T, d.authInfo.Data.Token, preloadResp.Data.Tid) + if err != nil { + return nil, fmt.Errorf("upload block %d failed: %w", i+1, err) + } + + // Update progress + if up != nil { + up(float64(block.size)) + } + } + + // Commit - merge blocks + // Note: use user token (d.authInfo.Data.Token), not blockInfo.Token + commitResp, err := d.commitUpload(ctx, uploadHost, preloadResp.Data.BlockInfo[0].Q, preloadResp.Data.BlockInfo[0].T, + d.authInfo.Data.Token, preloadResp.Data.Tid) + if err != nil { + return nil, err + } + + // If autoCommit is true (non-zero), file is already added (instant upload), use data from commit + if commitResp.Data.AutoCommit != 0 { + return &File{ + Name: file.GetName(), + Type: "0", + Nid: commitResp.Data.Nid, + CountSize: fmt.Sprintf("%d", commitResp.Data.Size), + CreateTimeTS: fmt.Sprintf("%d", commitResp.Data.CreateTime), + ModifyTimeTS: fmt.Sprintf("%d", commitResp.Data.ModifyTime), + Path: fname, + }, nil + } + + // Call Sync.addFileToApi to finalize the upload and get file info + addFileResp, err := d.addFileToApi(commitResp.Data.Tk) + if err != nil { + return nil, err + } + + // Set the full path + addFileResp.Data.File.Path = fname + + return &addFileResp.Data.File, nil +} + +func (d *Qihoo360) preloadBlocks(ctx context.Context, uploadHost, fname string, fsize int64, fhash string, fctime, fmtime int64, tk string, blocks []BlockInfo) (*PreloadResp, error) { + // Build query parameters + queryParams := map[string]string{ + "method": "Upload.request4Web", + "owner_qid": d.authInfo.Data.Qid, + "qid": d.authInfo.Data.Qid, + "devtype": "ecs_openapi", + "devid": "node-sdk-v16.20.2", // device id + "v": "1.0.1", + "ofmt": "json", + "devname": "EYUN_WEB_UPLOAD", + "rtick": fmt.Sprintf("%d", time.Now().Unix()), + } + + // Build URL + url := fmt.Sprintf("%s/intf.php", uploadHost) + for k, v := range queryParams { + if strings.Contains(url, "?") { + url += "&" + } else { + url += "?" + } + url += k + "=" + v + } + + // Prepare block_info JSON + blockInfoMap := map[string]interface{}{ + "request": map[string]interface{}{ + "block_info": blocks, + }, + } + blockInfoJSON, _ := json.Marshal(blockInfoMap) + + // Create multipart form + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + + // Add form fields + writer.WriteField("owner_qid", d.authInfo.Data.Qid) + writer.WriteField("fname", fname) + writer.WriteField("fsize", fmt.Sprintf("%d", fsize)) + writer.WriteField("fctime", fmt.Sprintf("%d", fctime)) + writer.WriteField("fmtime", fmt.Sprintf("%d", fmtime)) + writer.WriteField("fhash", fhash) + writer.WriteField("qid", d.authInfo.Data.Qid) + writer.WriteField("fattr", "0") + writer.WriteField("token", d.authInfo.Data.Token) + writer.WriteField("tk", tk) + writer.WriteField("devtype", "ecs_openapi") + + // Add file part with block_info JSON + part, _ := writer.CreateFormFile("file", "block_info.json") + part.Write(blockInfoJSON) + writer.Close() + + // Send request + req, _ := http.NewRequestWithContext(ctx, "POST", url, &buf) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Access-Token", d.authInfo.Data.AccessToken) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + var preloadResp PreloadResp + if err := json.Unmarshal(body, &preloadResp); err != nil { + return nil, err + } + + if preloadResp.Errno != 0 { + return nil, fmt.Errorf("preload failed: %s", preloadResp.Errmsg) + } + + return &preloadResp, nil +} + +func (d *Qihoo360) uploadBlock(ctx context.Context, uploadHost string, data []byte, bhash string, bidx int, boffset, bsize int64, filename string, filesize int64, q, t, token, tid string) error { + // Build query parameters + queryParams := map[string]string{ + "method": "Upload.block4Web", + "owner_qid": d.authInfo.Data.Qid, + "qid": d.authInfo.Data.Qid, + "devtype": "ecs_openapi", + "devid": "node-sdk-v16.20.2", + "v": "1.0.1", + "ofmt": "json", + "devname": "EYUN_WEB_UPLOAD", + "rtick": fmt.Sprintf("%d", time.Now().Unix()), + } + + // Build URL + url := fmt.Sprintf("%s/intf.php", uploadHost) + for k, v := range queryParams { + if strings.Contains(url, "?") { + url += "&" + } else { + url += "?" + } + url += k + "=" + v + } + + // Create multipart form + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + + // Add chunk data + part, _ := writer.CreateFormFile("file", "chunk") + part.Write(data) + + // Add form fields + writer.WriteField("bhash", bhash) + writer.WriteField("bidx", strconv.Itoa(bidx)) + writer.WriteField("boffset", fmt.Sprintf("%d", boffset)) + writer.WriteField("bsize", fmt.Sprintf("%d", bsize)) + writer.WriteField("filename", filename) + writer.WriteField("filesize", fmt.Sprintf("%d", filesize)) + writer.WriteField("q", q) + writer.WriteField("t", t) + writer.WriteField("token", token) + writer.WriteField("tid", tid) + writer.Close() + + // Send request + req, _ := http.NewRequestWithContext(ctx, "POST", url, &buf) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Access-Token", d.authInfo.Data.AccessToken) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + var result CommonResp + if err := json.Unmarshal(body, &result); err != nil { + return err + } + + if result.Errno != 0 { + return fmt.Errorf("upload block failed: %s", result.Errmsg) + } + + return nil +} + +func (d *Qihoo360) commitUpload(ctx context.Context, uploadHost, q, t, token, tid string) (*CommitResp, error) { + // Build query parameters + queryParams := map[string]string{ + "method": "Upload.commit4Web", + "owner_qid": d.authInfo.Data.Qid, + "qid": d.authInfo.Data.Qid, + "devtype": "ecs_openapi", + "devid": "node-sdk-v16.20.2", + "v": "1.0.1", + "ofmt": "json", + "devname": "EYUN_WEB_UPLOAD", + "rtick": fmt.Sprintf("%d", time.Now().Unix()), + } + + // Build URL + url := fmt.Sprintf("%s/intf.php", uploadHost) + for k, v := range queryParams { + if strings.Contains(url, "?") { + url += "&" + } else { + url += "?" + } + url += k + "=" + v + } + + // Create multipart form + var buf bytes.Buffer + writer := multipart.NewWriter(&buf) + + // Add form fields + writer.WriteField("q", q) + writer.WriteField("t", t) + writer.WriteField("token", token) + writer.WriteField("tid", tid) + writer.Close() + + // Send request + req, _ := http.NewRequestWithContext(ctx, "POST", url, &buf) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Access-Token", d.authInfo.Data.AccessToken) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + var result CommitResp + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + + if result.Errno != 0 { + return nil, fmt.Errorf("commit failed: %s", result.Errmsg) + } + + return &result, nil +} + +func (d *Qihoo360) addFileToApi(tk string) (*AddFileResp, error) { + params := map[string]string{ + "qid": d.authInfo.Data.Qid, + "tk": tk, + } + + var resp AddFileResp + _, err := d.request("Sync.addFileToApi", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("add file to api failed: %s", resp.Errmsg) + } + + return &resp, nil +} + +var _ driver.Driver = (*Qihoo360)(nil) diff --git a/drivers/qihoo360/meta.go b/drivers/qihoo360/meta.go new file mode 100644 index 000000000..57d261f37 --- /dev/null +++ b/drivers/qihoo360/meta.go @@ -0,0 +1,24 @@ +package qihoo360 + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + APIKey string `json:"api_key" required:"true" help:"360 AI Cloud API Key (yunpan_ prefix)"` +} + +var config = driver.Config{ + Name: "Qihoo360", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Qihoo360{} + }) +} diff --git a/drivers/qihoo360/types.go b/drivers/qihoo360/types.go new file mode 100644 index 000000000..a61e4789e --- /dev/null +++ b/drivers/qihoo360/types.go @@ -0,0 +1,167 @@ +package qihoo360 + +import ( + "strconv" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type File struct { + Name string `json:"name"` + Type string `json:"type"` // "1" for directory, "0" for file + Nid string `json:"nid"` + CountSize string `json:"count_size"` + CreateTimeTS string `json:"create_time"` + ModifyTimeTS string `json:"modify_time"` + Path string `json:"-"` // Full path, not from API +} + +func (f File) GetName() string { + return f.Name +} + +func (f File) GetSize() int64 { + size, _ := strconv.ParseInt(f.CountSize, 10, 64) + return size +} + +func (f File) ModTime() time.Time { + timestamp, _ := strconv.ParseInt(f.ModifyTimeTS, 10, 64) + return time.Unix(timestamp, 0) +} + +func (f File) CreateTime() time.Time { + timestamp, _ := strconv.ParseInt(f.CreateTimeTS, 10, 64) + return time.Unix(timestamp, 0) +} + +func (f File) IsDir() bool { + return f.Type == "1" +} + +func (f File) GetID() string { + return f.Nid +} + +func (f File) GetPath() string { + return f.Path +} + +func (f File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +type FileListResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + NodeList []File `json:"node_list"` + } `json:"data"` +} + +type AuthResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + AccessTokenExpire int64 `json:"access_token_expire"` + Qid string `json:"qid"` + } `json:"data"` + TraceId string `json:"trace_id"` +} + +type CommonResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` +} + +type UploadAddrResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + HTTP interface{} `json:"http"` // can be string or null + IsHttps int `json:"is_https"` // 0 or 1, not boolean + Tk interface{} `json:"tk"` // can be string or null + Addr2 string `json:"addr_2"` + NodeInfo []File `json:"node_info"` // returned when file exists (instant upload) + AutoCommit int `json:"autoCommit"` + FileHash string `json:"fhash"` + FileName string `json:"fname"` + FileSize string `json:"fsize"` + IsCreateDir bool `json:"is_createdir"` + } `json:"data"` +} + +type BlockInfo struct { + BHash string `json:"bhash"` + BIdx int `json:"bidx"` + BOffset int64 `json:"boffset"` + BSize int64 `json:"bsize"` + Q string `json:"q,omitempty"` + T string `json:"t,omitempty"` + Token string `json:"token,omitempty"` + Tid string `json:"tid,omitempty"` +} + +type PreloadResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + BlockInfo []BlockInfo `json:"block_info"` + Tid string `json:"tid"` + Tk string `json:"tk"` + HTTP string `json:"http"` + Addr2 string `json:"addr_2"` + IsHttps bool `json:"is_https"` + } `json:"data"` +} + +type DownloadUrlResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + DownloadUrl string `json:"downloadUrl"` + } `json:"data"` +} + +type CommitResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + Nid string `json:"nid"` + Name string `json:"fname"` + Size int64 `json:"fsize"` + CreateTime int64 `json:"fctime"` + ModifyTime int64 `json:"fmtime"` + Tk string `json:"tk"` + AutoCommit int `json:"autoCommit"` // 0 or 1 + } `json:"data"` +} + +type AddFileResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + File File `json:"file"` + } `json:"data"` +} + +type UserDetailResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Data struct { + Name string `json:"name"` + TotalSize string `json:"total_size"` + UsedSize string `json:"used_size"` + AvailableSize int64 `json:"available_size"` + IsVip bool `json:"is_vip"` + VipDesc string `json:"vip_desc"` + ExpireDay int `json:"expire_day"` + Expire string `json:"expire"` + } `json:"data"` +} + +var _ model.Obj = (*File)(nil) diff --git a/drivers/qihoo360/util.go b/drivers/qihoo360/util.go new file mode 100644 index 000000000..f64b50f89 --- /dev/null +++ b/drivers/qihoo360/util.go @@ -0,0 +1,377 @@ +package qihoo360 + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "net/url" + pathpkg "path" + "sort" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + log "github.com/sirupsen/logrus" +) + +const ( + ApiUrl = "https://openapi.eyun.360.cn/intf.php" + ClientID = "e4757e933b6486c08ed206ecb6d5d9e684fcb4e2" + ClientSecret = "885fd3231f1c1e37c9f462261a09b8c38cde0c2b" + SecretKey = "e7b24b112a44fdd9ee93bdf998c6ca0e" +) + +// phpUrlEncode encodes a string in PHP/JS style used by SDK +// JavaScript's encodeURIComponent keeps - _ . ! ~ * ' ( ) unencoded, +// but the sign function encodes them again +func phpUrlEncode(str string) string { + // First, do standard encoding but keep certain chars + encoded := url.QueryEscape(str) + // url.QueryEscape already encodes most things, but we need to ensure + // these specific characters are encoded as the JS does + replacer := strings.NewReplacer( + "!", "%21", + "'", "%27", + "(", "%28", + ")", "%29", + "*", "%2A", + ",", "%2C", + "~", "%7E", + ) + encoded = replacer.Replace(encoded) + // %20 should be + (last step) + encoded = strings.ReplaceAll(encoded, "%20", "+") + return encoded +} + +// generateSign generates MD5 signature for API request +func generateSign(params map[string]string) string { + // Sort keys alphabetically + keys := make([]string, 0, len(params)) + for k := range params { + keys = append(keys, k) + } + sort.Strings(keys) + + // Build key=encodedValue string + pairs := make([]string, 0, len(keys)) + for _, k := range keys { + encodedValue := phpUrlEncode(params[k]) + pairs = append(pairs, fmt.Sprintf("%s=%s", k, encodedValue)) + } + str := strings.Join(pairs, "&") + + // Append secret key + str += SecretKey + + // Calculate MD5 + hash := md5.Sum([]byte(str)) + return hex.EncodeToString(hash[:]) +} + +func (d *Qihoo360) getAuth() (*AuthResp, error) { + // Check if we have cached auth and it's not expired (with 5 min buffer) + if d.authInfo != nil && d.authExpire > 0 && time.Now().Unix() < d.authExpire-300 { + return d.authInfo, nil + } + + params := map[string]string{ + "method": "Oauth.getAccessTokenByApiKey", + "client_id": ClientID, + "client_secret": ClientSecret, + "api_key": d.APIKey, + "grant_type": "authorization_code", + } + + // Build URL with query parameters (no sign needed for auth request) + var resp AuthResp + req := base.RestyClient.R().SetResult(&resp) + for k, v := range params { + req.SetQueryParam(k, v) + } + + res, err := req.Get(ApiUrl) + if err != nil { + // Clear auth cache on error to force re-authentication next time + d.authInfo = nil + d.authExpire = 0 + return nil, err + } + + log.Debugf("Auth Response: %s", res.String()) + + if resp.Errno != 0 { + // Clear auth cache on error to force re-authentication next time + d.authInfo = nil + d.authExpire = 0 + return nil, fmt.Errorf("auth failed: %s", resp.Errmsg) + } + + // Cache auth info + d.authInfo = &resp + // access_token_expire is already a Unix timestamp, not a duration + if resp.Data.AccessTokenExpire > 0 { + d.authExpire = resp.Data.AccessTokenExpire + } else { + // Default to 1 hour if not provided + d.authExpire = time.Now().Unix() + 3600 + } + + return &resp, nil +} + +func (d *Qihoo360) request(method string, params map[string]string, result interface{}, excluded ...string) ([]byte, error) { + return d.requestWithRetry(method, params, result, 0, excluded...) +} + +func (d *Qihoo360) requestWithRetry(method string, params map[string]string, result interface{}, retryCount int, excluded ...string) ([]byte, error) { + // Prevent infinite retry loops + const maxRetries = 2 + if retryCount >= maxRetries { + return nil, fmt.Errorf("max retries (%d) exceeded for method %s", maxRetries, method) + } + + // Get auth if not already authenticated or expired + if d.authInfo == nil || d.authExpire <= 0 || time.Now().Unix() >= d.authExpire-300 { + _, err := d.getAuth() + if err != nil { + return nil, err + } + } + + // Ensure authInfo is set before proceeding + if d.authInfo == nil { + return nil, fmt.Errorf("authentication failed: no auth info") + } + + // Build excluded params map + excludedMap := make(map[string]bool) + if len(excluded) > 0 { + for _, key := range excluded { + excludedMap[key] = true + } + } + + // Build params for sign (excluding specified params) + signParams := map[string]string{ + "method": method, + "access_token": d.authInfo.Data.AccessToken, + "qid": d.authInfo.Data.Qid, + } + + // Add params to sign if not excluded + for k, v := range params { + if !excludedMap[k] { + signParams[k] = v + } + } + + // Generate sign + sign := generateSign(signParams) + + log.Debugf("Request method: %s", method) + + // File.getList, Sync.getVerifiedDownLoadUrl, and Sync.getUploadFileAddr use GET + var err error + + if method == "File.getList" || method == "Sync.getVerifiedDownLoadUrl" || method == "Sync.getUploadFileAddr" || method == "User.getUserDetail" { + // GET request: params in query string + allParams := map[string]string{ + "method": method, + "access_token": d.authInfo.Data.AccessToken, + "qid": d.authInfo.Data.Qid, + "sign": sign, + } + for k, v := range params { + if !excludedMap[k] { + allParams[k] = v + } + } + req := base.RestyClient.R(). + SetQueryParams(allParams). + SetResult(result). + SetHeader("Access-Token", d.authInfo.Data.AccessToken) + if method == "Sync.getVerifiedDownLoadUrl" { + req.SetHeader("User-Agent", "yunpan_mcp_server") + } + _, err = req.Get(ApiUrl) + if err != nil { + return nil, err + } + } else { + // POST request: basic params in query, all params in form + queryParams := map[string]string{ + "method": method, + "access_token": d.authInfo.Data.AccessToken, + "qid": d.authInfo.Data.Qid, + "sign": sign, + } + + formData := make(map[string]string) + for k, v := range params { + formData[k] = v + } + + _, err = base.RestyClient.R(). + SetQueryParams(queryParams). + SetFormData(formData). + SetResult(result). + SetHeader("Access-Token", d.authInfo.Data.AccessToken). + SetHeader("Content-Type", "application/x-www-form-urlencoded"). + Post(ApiUrl) + if err != nil { + return nil, err + } + } + + log.Debugf("Response data received") + + // Check if we got an authentication error from the API + // If errno is -1 or -2, it usually means token is invalid/expired + if resp, ok := result.(*FileListResp); ok { + if resp.Errno == -1 || resp.Errno == -2 { + log.Debugf("Auth token expired (errno: %d), clearing cache and retrying (attempt %d)", resp.Errno, retryCount+1) + d.authInfo = nil + d.authExpire = 0 + // Retry with fresh auth + return d.requestWithRetry(method, params, result, retryCount+1, excluded...) + } + } else if resp, ok := result.(*DownloadUrlResp); ok { + if resp.Errno == -1 || resp.Errno == -2 { + log.Debugf("Auth token expired (errno: %d), clearing cache and retrying (attempt %d)", resp.Errno, retryCount+1) + d.authInfo = nil + d.authExpire = 0 + return d.requestWithRetry(method, params, result, retryCount+1, excluded...) + } + } else if resp, ok := result.(*UserDetailResp); ok { + if resp.Errno == -1 || resp.Errno == -2 { + log.Debugf("Auth token expired (errno: %d), clearing cache and retrying (attempt %d)", resp.Errno, retryCount+1) + d.authInfo = nil + d.authExpire = 0 + return d.requestWithRetry(method, params, result, retryCount+1, excluded...) + } + } else if resp, ok := result.(*CommonResp); ok { + if resp.Errno == -1 || resp.Errno == -2 { + log.Debugf("Auth token expired (errno: %d), clearing cache and retrying (attempt %d)", resp.Errno, retryCount+1) + d.authInfo = nil + d.authExpire = 0 + return d.requestWithRetry(method, params, result, retryCount+1, excluded...) + } + } + + return nil, nil +} + +func (d *Qihoo360) getFiles(path string, page int, pageSize int) ([]File, error) { + params := map[string]string{ + "path": path, + "page": fmt.Sprintf("%d", page), + "page_size": fmt.Sprintf("%d", pageSize), + } + + var resp FileListResp + _, err := d.request("File.getList", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("get files failed: %s", resp.Errmsg) + } + + // Normalize name display and full path for each file/dir + for i := range resp.Data.NodeList { + rawName := resp.Data.NodeList[i].Name + // Trim leading slash and trailing slash for dir name + trimmed := strings.TrimPrefix(rawName, "/") + trimmed = strings.TrimSuffix(trimmed, "/") + base := pathpkg.Base(trimmed) + resp.Data.NodeList[i].Name = base + + // Construct full path + var fullPath string + if path == "/" { + fullPath = "/" + base + } else { + fullPath = path + base + } + if resp.Data.NodeList[i].Type == "1" && !strings.HasSuffix(fullPath, "/") { + fullPath += "/" + } + resp.Data.NodeList[i].Path = fullPath + } + + return resp.Data.NodeList, nil +} + +func (d *Qihoo360) getDownloadUrl(nid string) (string, error) { + params := map[string]string{ + "nid": nid, + } + + var resp DownloadUrlResp + _, err := d.request("Sync.getVerifiedDownLoadUrl", params, &resp) + if err != nil { + return "", err + } + + if resp.Errno != 0 { + return "", fmt.Errorf("get download url failed: %s", resp.Errmsg) + } + + return resp.Data.DownloadUrl, nil +} + +func (d *Qihoo360) getUploadAddr(fname string, fsize int64, fhash string, fctime, fmtime int64) (*UploadAddrResp, error) { + // Build all query parameters + params := map[string]string{ + "owner_qid": d.authInfo.Data.Qid, + "fname": fname, + "fsize": fmt.Sprintf("%d", fsize), + "fctime": fmt.Sprintf("%d", fctime), + "fmtime": fmt.Sprintf("%d", fmtime), + "fhash": fhash, + "qid": d.authInfo.Data.Qid, + "fattr": "0", + "token": d.authInfo.Data.Token, + "tk": "", + "devtype": "ecs_openapi", + } + + // Calculate sign using only specific parameters (per SDK) + signParams := map[string]string{ + "fhash": fhash, + "qid": d.authInfo.Data.Qid, + "method": "Sync.getUploadFileAddr", + "fname": fname, + "fsize": fmt.Sprintf("%d", fsize), + "access_token": d.authInfo.Data.AccessToken, + } + params["sign"] = generateSign(signParams) + + var resp UploadAddrResp + _, err := d.request("Sync.getUploadFileAddr", params, &resp) + if err != nil { + return nil, err + } + if resp.Errno != 0 { + return nil, fmt.Errorf("get upload addr failed: %s", resp.Errmsg) + } + return &resp, nil +} + +func (d *Qihoo360) getUserDetail() (*UserDetailResp, error) { + params := map[string]string{} + + var resp UserDetailResp + _, err := d.request("User.getUserDetail", params, &resp) + if err != nil { + return nil, err + } + + if resp.Errno != 0 { + return nil, fmt.Errorf("get user detail failed: %s", resp.Errmsg) + } + + return &resp, nil +} diff --git a/drivers/quark_open/driver.go b/drivers/quark_open/driver.go index 7b6b31331..f0b8baf09 100644 --- a/drivers/quark_open/driver.go +++ b/drivers/quark_open/driver.go @@ -1,7 +1,6 @@ package quark_open import ( - "bytes" "context" "encoding/hex" "errors" @@ -9,12 +8,15 @@ import ( "hash" "io" "net/http" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" ) @@ -174,7 +176,7 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return err } // 如果预上传已经完成，直接返回--秒传 - if pre.Data.Finish == true { + if pre.Data.Finish { up(100) return nil } @@ -188,46 +190,48 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File } // part up + ss, err := streamPkg.NewStreamSectionReader(stream, int(pre.Data.PartSize), &up) + if err != nil { + return err + } total := stream.GetSize() - left := total - part := make([]byte, pre.Data.PartSize) // 用于存储每个分片的ETag，后续commit时需要 - etags := make([]string, len(partInfo)) + etags := make([]string, 0, len(partInfo)) // 遍历上传每个分片 - for i, urlInfo := range upUrlInfo.UploadUrls { + for i := range len(upUrlInfo.UploadUrls) { if utils.IsCanceled(ctx) { return ctx.Err() } - currentSize := int64(urlInfo.PartSize) - if left < currentSize { - part = part[:left] - } else { - part = part[:currentSize] - } - - // 读取分片数据 - n, err := io.ReadFull(stream, part) - if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + offset := int64(i) * pre.Data.PartSize + size := min(pre.Data.PartSize, total-offset) + rd, err := ss.GetSectionReader(offset, size) + if err != nil { return err } - - // 准备上传分片 - reader := driver.NewLimitedUploadStream(ctx, bytes.NewReader(part)) - etag, err := d.upPart(ctx, upUrlInfo, i, reader) + err = retry.Do(func() error { + rd.Seek(0, io.SeekStart) + etag, err := d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) + if err != nil { + return err + } + etags = append(etags, etag) + return nil + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second)) + ss.FreeSectionReader(rd) if err != nil { return fmt.Errorf("failed to upload part %d: %w", i, err) } - // 保存ETag，用于后续commit - etags[i] = etag - - // 更新剩余大小和进度 - left -= int64(n) - up(float64(total-left) / float64(total) * 100) + up(95 * float64(offset+size) / float64(total)) } + defer up(100) return d.upFinish(ctx, pre, partInfo, etags) } diff --git a/drivers/quark_open/util.go b/drivers/quark_open/util.go index 78f4e4a2c..788ca0e99 100644 --- a/drivers/quark_open/util.go +++ b/drivers/quark_open/util.go @@ -341,8 +341,7 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber req.Header.Set("User-Agent", "Go-http-client/1.1") // 发送请求 - client := &http.Client{} - resp, err := client.Do(req) + resp, err := base.HttpClient.Do(req) if err != nil { return "", err } @@ -441,7 +440,6 @@ func (d *QuarkOpen) _refreshToken() (string, string, error) { u := d.APIAddress var resp RefreshTokenOnlineAPIResp _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index f7ebfc603..a1371243c 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -1,7 +1,6 @@ package quark import ( - "bytes" "context" "encoding/hex" "hash" @@ -13,9 +12,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" - log "github.com/sirupsen/logrus" ) type QuarkOrUC struct { @@ -159,56 +159,63 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File if err != nil { return err } - log.Debugln("hash: ", md5Str, sha1Str) // hash finish, err := d.upHash(md5Str, sha1Str, pre.Data.TaskId) if err != nil { return err } if finish { + up(100) return nil } // part up + ss, err := streamPkg.NewStreamSectionReader(stream, pre.Metadata.PartSize, &up) + if err != nil { + return err + } total := stream.GetSize() - left := total partSize := int64(pre.Metadata.PartSize) - part := make([]byte, partSize) - count := int(total / partSize) - if total%partSize > 0 { - count++ - } - md5s := make([]string, 0, count) - partNumber := 1 - for left > 0 { + uploadNums := int((total + partSize - 1) / partSize) + md5s := make([]string, 0, uploadNums) + for partIndex := range uploadNums { if utils.IsCanceled(ctx) { return ctx.Err() } - if left < partSize { - part = part[:left] - } - n, err := io.ReadFull(stream, part) + offset := int64(partIndex) * partSize + size := min(partSize, total-offset) + rd, err := ss.GetSectionReader(offset, size) if err != nil { return err } - left -= int64(n) - log.Debugf("left: %d", left) - reader := driver.NewLimitedUploadStream(ctx, bytes.NewReader(part)) - m, err := d.upPart(ctx, pre, stream.GetMimetype(), partNumber, reader) - // m, err := driver.UpPart(pre, file.GetMIMEType(), partNumber, bytes, account, md5Str, sha1Str) + err = retry.Do(func() error { + rd.Seek(0, io.SeekStart) + m, err := d.upPart(ctx, pre, stream.GetMimetype(), partIndex+1, driver.NewLimitedUploadStream(ctx, rd)) + if err != nil { + return err + } + if m == "finish" { + up(100) + return nil + } + md5s = append(md5s, m) + return nil + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second)) + ss.FreeSectionReader(rd) if err != nil { return err } - if m == "finish" { - return nil - } - md5s = append(md5s, m) - partNumber++ - up(100 * float64(total-left) / float64(total)) + up(95 * float64(offset+size) / float64(total)) } + up(97) err = d.upCommit(pre, md5s) if err != nil { return err } + defer up(100) return d.upFinish(pre) } @@ -220,7 +227,7 @@ func (d *QuarkOrUC) GetDetails(ctx context.Context) (*model.StorageDetails, erro return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: memberInfo.Data.TotalCapacity, - FreeSpace: memberInfo.Data.TotalCapacity - memberInfo.Data.UseCapacity, + UsedSpace: memberInfo.Data.UseCapacity, }, }, nil } diff --git a/drivers/quark_uc/types.go b/drivers/quark_uc/types.go index cea2b8830..07286e293 100644 --- a/drivers/quark_uc/types.go +++ b/drivers/quark_uc/types.go @@ -265,8 +265,8 @@ type MemberResp struct { Data struct { MemberType string `json:"member_type"` CreatedAt uint64 `json:"created_at"` - SecretUseCapacity uint64 `json:"secret_use_capacity"` - UseCapacity uint64 `json:"use_capacity"` + SecretUseCapacity int64 `json:"secret_use_capacity"` + UseCapacity int64 `json:"use_capacity"` IsNewUser bool `json:"is_new_user"` MemberStatus struct { Vip string `json:"VIP"` @@ -274,8 +274,8 @@ type MemberResp struct { MiniVip string `json:"MINI_VIP"` SuperVip string `json:"SUPER_VIP"` } `json:"member_status"` - SecretTotalCapacity uint64 `json:"secret_total_capacity"` - TotalCapacity uint64 `json:"total_capacity"` + SecretTotalCapacity int64 `json:"secret_total_capacity"` + TotalCapacity int64 `json:"total_capacity"` } `json:"data"` Metadata struct { RangeSize int `json:"range_size"` diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index 01f63ef0c..87798c6ef 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "fmt" + "html" "io" "net/http" "strconv" @@ -70,9 +71,11 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { page := 1 size := 100 query := map[string]string{ - "pdir_fid": parent, - "_size": strconv.Itoa(size), - "_fetch_total": "1", + "pdir_fid": parent, + "_size": strconv.Itoa(size), + "_fetch_total": "1", + "fetch_all_file": "1", + "fetch_risk_file_name": "1", } if d.OrderBy != "none" { query["_sort"] = "file_type:asc," + d.OrderBy + ":" + d.OrderDirection @@ -87,6 +90,7 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { return nil, err } for _, file := range resp.Data.List { + file.FileName = html.UnescapeString(file.FileName) if d.OnlyListVideoFile { // 开启后 只列出视频文件和文件夹 if file.IsDir() || file.Category == 1 { @@ -229,25 +233,30 @@ x-oss-user-agent:aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit // } //} u := fmt.Sprintf("https://%s.%s/%s", pre.Data.Bucket, pre.Data.UploadUrl[7:], pre.Data.ObjKey) - res, err := base.RestyClient.R().SetContext(ctx). - SetHeaders(map[string]string{ - "Authorization": resp.Data.AuthKey, - "Content-Type": mineType, - "Referer": "https://pan.quark.cn/", - "x-oss-date": timeStr, - "x-oss-user-agent": "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit", - }). - SetQueryParams(map[string]string{ - "partNumber": strconv.Itoa(partNumber), - "uploadId": pre.Data.UploadId, - }).SetBody(bytes).Put(u) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u, bytes) if err != nil { return "", err } - if res.StatusCode() != 200 { - return "", fmt.Errorf("up status: %d, error: %s", res.StatusCode(), res.String()) + req.Header.Set("Authorization", resp.Data.AuthKey) + req.Header.Set("Content-Type", mineType) + req.Header.Set("Referer", "https://pan.quark.cn/") + req.Header.Set("x-oss-date", timeStr) + req.Header.Set("x-oss-user-agent", "aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit") + q := req.URL.Query() + q.Add("partNumber", strconv.Itoa(partNumber)) + q.Add("uploadId", pre.Data.UploadId) + req.URL.RawQuery = q.Encode() + res, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + respBody, _ := io.ReadAll(res.Body) + return "", fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(respBody)) } - return res.Header().Get("Etag"), nil + return res.Header.Get("Etag"), nil } func (d *QuarkOrUC) upCommit(pre UpPreResp, md5s []string) error { diff --git a/drivers/quark_uc_tv/util.go b/drivers/quark_uc_tv/util.go index d68a2f3c3..c0da6eb6b 100644 --- a/drivers/quark_uc_tv/util.go +++ b/drivers/quark_uc_tv/util.go @@ -8,6 +8,7 @@ import ( "errors" "net/http" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -70,8 +71,16 @@ func (d *QuarkUCTV) request(ctx context.Context, pathname string, method string, return nil, err } // 判断 是否需要 刷新 access_token - if e.Status == -1 && e.Errno == 10001 { - // token 过期 + errInfoLower := strings.ToLower(strings.TrimSpace(e.ErrorInfo)) + maybeTokenInvalid := + (e.Status == -1 && (e.Errno == 10001 || e.Errno == 11001)) || + (errInfoLower != "" && + (strings.Contains(errInfoLower, "access token") || + strings.Contains(errInfoLower, "access_token") || + strings.Contains(errInfoLower, "token无效") || + strings.Contains(errInfoLower, "token 无效"))) + if maybeTokenInvalid { + // token 过期 / 无效 err = d.getRefreshTokenByTV(ctx, d.Addition.RefreshToken, true) if err != nil { return nil, err diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index d19ead477..a9edef10b 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -10,6 +10,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/cron" @@ -24,9 +25,10 @@ import ( type S3 struct { model.Storage Addition - Session *session.Session - client *s3.S3 - linkClient *s3.S3 + Session *session.Session + client *s3.S3 + linkClient *s3.S3 + directUploadClient *s3.S3 config driver.Config cron *cron.Cron @@ -52,16 +54,18 @@ func (d *S3) Init(ctx context.Context) error { if err != nil { log.Errorln("Doge init session error:", err) } - d.client = d.getClient(false) - d.linkClient = d.getClient(true) + d.client = d.getClient(ClientTypeNormal) + d.linkClient = d.getClient(ClientTypeLink) + d.directUploadClient = d.getClient(ClientTypeDirectUpload) }) } err := d.initSession() if err != nil { return err } - d.client = d.getClient(false) - d.linkClient = d.getClient(true) + d.client = d.getClient(ClientTypeNormal) + d.linkClient = d.getClient(ClientTypeLink) + d.directUploadClient = d.getClient(ClientTypeDirectUpload) return nil } @@ -210,4 +214,33 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up return err } +func (d *S3) GetDirectUploadTools() []string { + if !d.EnableDirectUpload { + return nil + } + return []string{"HttpDirect"} +} + +func (d *S3) GetDirectUploadInfo(ctx context.Context, _ string, dstDir model.Obj, fileName string, _ int64) (any, error) { + if !d.EnableDirectUpload { + return nil, errs.NotImplement + } + path := getKey(stdpath.Join(dstDir.GetPath(), fileName), false) + req, _ := d.directUploadClient.PutObjectRequest(&s3.PutObjectInput{ + Bucket: &d.Bucket, + Key: &path, + }) + if req == nil { + return nil, fmt.Errorf("failed to create PutObject request") + } + link, err := req.Presign(time.Hour * time.Duration(d.SignURLExpire)) + if err != nil { + return nil, err + } + return &model.HttpDirectUploadInfo{ + UploadURL: link, + Method: "PUT", + }, nil +} + var _ driver.Driver = (*S3)(nil) diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 59c3bbb05..b2439c683 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -21,6 +21,8 @@ type Addition struct { ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` + EnableDirectUpload bool `json:"enable_direct_upload" default:"false"` + DirectUploadHost string `json:"direct_upload_host" required:"false"` } func init() { diff --git a/drivers/s3/util.go b/drivers/s3/util.go index b58971760..c84380786 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -41,9 +41,15 @@ func (d *S3) initSession() error { return err } -func (d *S3) getClient(link bool) *s3.S3 { +const ( + ClientTypeNormal = iota + ClientTypeLink + ClientTypeDirectUpload +) + +func (d *S3) getClient(clientType int) *s3.S3 { client := s3.New(d.Session) - if link && d.CustomHost != "" { + if clientType == ClientTypeLink && d.CustomHost != "" { client.Handlers.Build.PushBack(func(r *request.Request) { if r.HTTPRequest.Method != http.MethodGet { return @@ -58,6 +64,20 @@ func (d *S3) getClient(link bool) *s3.S3 { } }) } + if clientType == ClientTypeDirectUpload && d.DirectUploadHost != "" { + client.Handlers.Build.PushBack(func(r *request.Request) { + if r.HTTPRequest.Method != http.MethodPut { + return + } + split := strings.SplitN(d.DirectUploadHost, "://", 2) + if utils.SliceContains([]string{"http", "https"}, split[0]) { + r.HTTPRequest.URL.Scheme = split[0] + r.HTTPRequest.URL.Host = split[1] + } else { + r.HTTPRequest.URL.Host = d.DirectUploadHost + } + }) + } return client } @@ -78,8 +98,8 @@ func getPlaceholderName(placeholder string) string { return placeholder } -func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { - prefix = getKey(prefix, true) +func (d *S3) listV1(dirPath string, args model.ListArgs) ([]model.Obj, error) { + prefix := getKey(dirPath, true) log.Debugf("list: %s", prefix) files := make([]model.Obj, 0) marker := "" @@ -97,7 +117,7 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { for _, object := range listObjectsResult.CommonPrefixes { name := path.Base(strings.Trim(*object.Prefix, "/")) file := model.Object{ - //Id: *object.Key, + Path: path.Join(dirPath, name), Name: name, Modified: d.Modified, IsFolder: true, @@ -110,7 +130,7 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { continue } file := model.Object{ - //Id: *object.Key, + Path: path.Join(dirPath, name), Name: name, Size: *object.Size, Modified: *object.LastModified, @@ -129,8 +149,8 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { return files, nil } -func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { - prefix = getKey(prefix, true) +func (d *S3) listV2(dirPath string, args model.ListArgs) ([]model.Obj, error) { + prefix := getKey(dirPath, true) files := make([]model.Obj, 0) var continuationToken, startAfter *string for { @@ -149,7 +169,7 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { for _, object := range listObjectsResult.CommonPrefixes { name := path.Base(strings.Trim(*object.Prefix, "/")) file := model.Object{ - //Id: *object.Key, + Path: path.Join(dirPath, name), Name: name, Modified: d.Modified, IsFolder: true, @@ -165,7 +185,7 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { continue } file := model.Object{ - //Id: *object.Key, + Path: path.Join(dirPath, name), Name: name, Size: *object.Size, Modified: *object.LastModified, @@ -197,9 +217,10 @@ func (d *S3) copy(ctx context.Context, src string, dst string, isDir bool) error func (d *S3) copyFile(ctx context.Context, src string, dst string) error { srcKey := getKey(src, false) dstKey := getKey(dst, false) + encodedKey := strings.ReplaceAll(url.PathEscape(d.Bucket+"/"+srcKey), "+", "%2B") input := &s3.CopyObjectInput{ Bucket: &d.Bucket, - CopySource: aws.String(url.PathEscape(d.Bucket + "/" + srcKey)), + CopySource: aws.String(encodedKey), Key: &dstKey, } _, err := d.client.CopyObject(input) diff --git a/drivers/seafile/driver.go b/drivers/seafile/driver.go index 73c21645e..221ac4361 100644 --- a/drivers/seafile/driver.go +++ b/drivers/seafile/driver.go @@ -4,10 +4,11 @@ import ( "context" "fmt" "net/http" + stdpath "path" "strings" - "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" @@ -18,7 +19,7 @@ type Seafile struct { Addition authorization string - libraryMap map[string]*LibraryInfo + root model.Obj } func (d *Seafile) Config() driver.Config { @@ -31,46 +32,103 @@ func (d *Seafile) GetAddition() driver.Additional { func (d *Seafile) Init(ctx context.Context) error { d.Address = strings.TrimSuffix(d.Address, "/") + err := d.getToken() + if err != nil { + return err + } d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) - d.libraryMap = make(map[string]*LibraryInfo) - return d.getToken() + if d.RepoId != "" { + library, err := d.getLibraryInfo(d.RepoId) + if err != nil { + return err + } + library.path = d.RootFolderPath + library.ObjMask = model.Locked + d.root = &LibraryInfo{ + LibraryItemResp: library, + } + return nil + } + if len(d.RootFolderPath) <= 1 { + d.root = &model.Object{ + Name: "root", + Path: d.RootFolderPath, + IsFolder: true, + Modified: d.Modified, + Mask: model.Locked, + } + return nil + } + + var resp []LibraryItemResp + _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { + req.SetResult(&resp) + }) + if err != nil { + return err + } + for _, library := range resp { + p, found := strings.CutPrefix(d.RootFolderPath[1:], library.Name) + if !found { + continue + } + if p == "" { + p = "/" + } else if p[0] != '/' { + continue + } + // d.RepoId = library.Id + // d.RootFolderPath = p + + library.path = p + library.ObjMask = model.Locked + d.root = &LibraryInfo{ + LibraryItemResp: library, + } + return nil + } + return fmt.Errorf("Library for root folder path %q not found", d.RootFolderPath) } func (d *Seafile) Drop(ctx context.Context) error { + d.root = nil return nil } +func (d *Seafile) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root == nil { + return nil, errs.StorageNotInit + } + return d.root, nil +} + func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) (result []model.Obj, err error) { path := dir.GetPath() - if path == d.RootFolderPath { - libraries, err := d.listLibraries() - if err != nil { - return nil, err - } - if path == "/" && d.RepoId == "" { - return utils.SliceConvert(libraries, func(f LibraryItemResp) (model.Obj, error) { - return &model.Object{ - Name: f.Name, - Modified: time.Unix(f.Modified, 0), - Size: f.Size, - IsFolder: true, - }, nil - }) - } - } - var repo *LibraryInfo - repo, path, err = d.getRepoAndPath(path) - if err != nil { - return nil, err - } - if repo.Encrypted { - err = d.decryptLibrary(repo) - if err != nil { - return nil, err + switch o := dir.(type) { + default: + var resp []LibraryItemResp + _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { + req.SetResult(&resp) + }) + return utils.SliceConvert(resp, func(f LibraryItemResp) (model.Obj, error) { + f.path = path + return &LibraryInfo{ + LibraryItemResp: f, + }, nil + }) + case *LibraryInfo: + if o.Encrypted { + err = d.decryptLibrary(o) + if err != nil { + return nil, err + } } + case *RepoItemResp: + // do nothing } - var resp []RepoDirItemResp - _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/dir/", repo.Id), func(req *resty.Request) { + + var resp []RepoItemResp + _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/dir/", dir.GetID()), func(req *resty.Request) { req.SetResult(&resp).SetQueryParams(map[string]string{ "p": path, }) @@ -78,27 +136,17 @@ func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) if err != nil { return nil, err } - return utils.SliceConvert(resp, func(f RepoDirItemResp) (model.Obj, error) { - return &model.ObjThumb{ - Object: model.Object{ - Name: f.Name, - Modified: time.Unix(f.Modified, 0), - Size: f.Size, - IsFolder: f.Type == "dir", - }, - // Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, - }, nil + return utils.SliceConvert(resp, func(f RepoItemResp) (model.Obj, error) { + f.path = stdpath.Join(path, f.Name) + f.repoID = dir.GetID() + return &f, nil }) } func (d *Seafile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - repo, path, err := d.getRepoAndPath(file.GetPath()) - if err != nil { - return nil, err - } - res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/file/", file.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": file.GetPath(), "reuse": "1", }) }) @@ -111,14 +159,9 @@ func (d *Seafile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Seafile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - repo, path, err := d.getRepoAndPath(parentDir.GetPath()) - if err != nil { - return err - } - path, _ = utils.JoinBasePath(path, dirName) - _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/dir/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/dir/", parentDir.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": stdpath.Join(parentDir.GetPath(), dirName), }).SetFormData(map[string]string{ "operation": "mkdir", }) @@ -127,34 +170,22 @@ func (d *Seafile) MakeDir(ctx context.Context, parentDir model.Obj, dirName stri } func (d *Seafile) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - repo, path, err := d.getRepoAndPath(srcObj.GetPath()) - if err != nil { - return err - } - dstRepo, dstPath, err := d.getRepoAndPath(dstDir.GetPath()) - if err != nil { - return err - } - _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", srcObj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": srcObj.GetPath(), }).SetFormData(map[string]string{ "operation": "move", - "dst_repo": dstRepo.Id, - "dst_dir": dstPath, + "dst_repo": dstDir.GetID(), + "dst_dir": dstDir.GetPath(), }) }, true) return err } func (d *Seafile) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - repo, path, err := d.getRepoAndPath(srcObj.GetPath()) - if err != nil { - return err - } - _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", srcObj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": srcObj.GetPath(), }).SetFormData(map[string]string{ "operation": "rename", "newname": newName, @@ -164,47 +195,31 @@ func (d *Seafile) Rename(ctx context.Context, srcObj model.Obj, newName string) } func (d *Seafile) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - repo, path, err := d.getRepoAndPath(srcObj.GetPath()) - if err != nil { - return err - } - dstRepo, dstPath, err := d.getRepoAndPath(dstDir.GetPath()) - if err != nil { - return err - } - _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", srcObj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": srcObj.GetPath(), }).SetFormData(map[string]string{ "operation": "copy", - "dst_repo": dstRepo.Id, - "dst_dir": dstPath, + "dst_repo": dstDir.GetID(), + "dst_dir": dstDir.GetPath(), }) }) return err } func (d *Seafile) Remove(ctx context.Context, obj model.Obj) error { - repo, path, err := d.getRepoAndPath(obj.GetPath()) - if err != nil { - return err - } - _, err = d.request(http.MethodDelete, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { + _, err := d.request(http.MethodDelete, fmt.Sprintf("/api2/repos/%s/file/", obj.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": obj.GetPath(), }) }) return err } func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - repo, path, err := d.getRepoAndPath(dstDir.GetPath()) - if err != nil { - return err - } - res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/upload-link/", repo.Id), func(req *resty.Request) { + res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/upload-link/", dstDir.GetID()), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": path, + "p": dstDir.GetPath(), }) }) if err != nil { @@ -220,7 +235,7 @@ func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame }) req.SetFileReader("file", s.GetName(), r). SetFormData(map[string]string{ - "parent_dir": path, + "parent_dir": dstDir.GetPath(), "replace": "1", }). SetContext(ctx) diff --git a/drivers/seafile/types.go b/drivers/seafile/types.go index 47cb322df..29a51ce84 100644 --- a/drivers/seafile/types.go +++ b/drivers/seafile/types.go @@ -1,6 +1,11 @@ package seafile -import "time" +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) type AuthTokenResp struct { Token string `json:"token"` @@ -13,7 +18,41 @@ type RepoItemResp struct { Size int64 `json:"size"` Modified int64 `json:"mtime"` Permission string `json:"permission"` + + path string + model.ObjMask + repoID string +} + +func (l *RepoItemResp) IsDir() bool { + return l.Type == "dir" +} +func (l *RepoItemResp) GetPath() string { + return l.path +} +func (l *RepoItemResp) GetName() string { + return l.Name +} +func (l *RepoItemResp) ModTime() time.Time { + return time.Unix(l.Modified, 0) +} +func (l *RepoItemResp) CreateTime() time.Time { + return l.ModTime() +} +func (l *RepoItemResp) GetSize() int64 { + return l.Size } +func (l *RepoItemResp) GetID() string { + if l.repoID != "" { + return l.repoID + } + return l.Id +} +func (l *RepoItemResp) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +var _ model.Obj = (*RepoItemResp)(nil) type LibraryItemResp struct { RepoItemResp @@ -33,12 +72,12 @@ type LibraryItemResp struct { SizeFormatted string `json:"size_formatted"` } -type RepoDirItemResp struct { - RepoItemResp -} - type LibraryInfo struct { LibraryItemResp decryptedTime time.Time decryptedSuccess bool -} \ No newline at end of file +} + +func (l *LibraryInfo) IsDir() bool { + return true +} diff --git a/drivers/seafile/util.go b/drivers/seafile/util.go index 6b5d09939..f2c0b6659 100644 --- a/drivers/seafile/util.go +++ b/drivers/seafile/util.go @@ -7,9 +7,6 @@ import ( "strings" "time" - "github.com/OpenListTeam/OpenList/v4/internal/errs" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/go-resty/resty/v2" ) @@ -71,73 +68,12 @@ func (d *Seafile) request(method string, pathname string, callback base.ReqCallb return res.Body(), nil } -func (d *Seafile) getRepoAndPath(fullPath string) (repo *LibraryInfo, path string, err error) { - libraryMap := d.libraryMap - repoId := d.Addition.RepoId - if repoId != "" { - if len(repoId) == 36 /* uuid */ { - for _, library := range libraryMap { - if library.Id == repoId { - return library, fullPath, nil - } - } - } - } else { - var repoName string - str := fullPath[1:] - pos := strings.IndexRune(str, '/') - if pos == -1 { - repoName = str - } else { - repoName = str[:pos] - } - path = utils.FixAndCleanPath(fullPath[1+len(repoName):]) - if library, ok := libraryMap[repoName]; ok { - return library, path, nil - } - } - return nil, "", errs.ObjectNotFound -} - -func (d *Seafile) listLibraries() (resp []LibraryItemResp, err error) { - repoId := d.Addition.RepoId - if repoId == "" { - _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { - req.SetResult(&resp) - }) - } else { - var oneResp LibraryItemResp - _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/", repoId), func(req *resty.Request) { - req.SetResult(&oneResp) - }) - if err == nil { - resp = append(resp, oneResp) - } - } - if err != nil { - return nil, err - } - libraryMap := make(map[string]*LibraryInfo) - var putLibraryMap func(library LibraryItemResp, index int) - putLibraryMap = func(library LibraryItemResp, index int) { - name := library.Name - if index > 0 { - name = fmt.Sprintf("%s (%d)", name, index) - } - if _, exist := libraryMap[name]; exist { - putLibraryMap(library, index+1) - } else { - libraryInfo := LibraryInfo{} - data, _ := utils.Json.Marshal(library) - _ = utils.Json.Unmarshal(data, &libraryInfo) - libraryMap[name] = &libraryInfo - } - } - for _, library := range resp { - putLibraryMap(library, 0) - } - d.libraryMap = libraryMap - return resp, nil +func (d *Seafile) getLibraryInfo(repoId string) (LibraryItemResp, error) { + var oneResp LibraryItemResp + _, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/", repoId), func(req *resty.Request) { + req.SetResult(&oneResp) + }) + return oneResp, err } var repoPwdNotConfigured = errors.New("library password not configured") diff --git a/drivers/sftp/driver.go b/drivers/sftp/driver.go index 17db40380..3aa39537e 100644 --- a/drivers/sftp/driver.go +++ b/drivers/sftp/driver.go @@ -69,15 +69,10 @@ func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* Limiter: stream.ServerDownloadLimit, Ctx: ctx, } - if !d.Config().OnlyLinkMFile { - return &model.Link{ - RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), - SyncClosers: utils.NewSyncClosers(remoteFile), - }, nil - } return &model.Link{ - MFile: mFile, - SyncClosers: utils.NewSyncClosers(remoteFile), + RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), + SyncClosers: utils.NewSyncClosers(remoteFile), + RequireReference: true, }, nil } @@ -136,12 +131,12 @@ func (d *SFTP) GetDetails(ctx context.Context) (*model.StorageDetails, error) { } return nil, err } - total := stat.Blocks * stat.Bsize - free := stat.Bfree * stat.Bsize + total := int64(stat.Blocks * stat.Bsize) + free := int64(stat.Bfree * stat.Bsize) return &model.StorageDetails{ DiskUsage: model.DiskUsage{ TotalSpace: total, - FreeSpace: free, + UsedSpace: total - free, }, }, nil } diff --git a/drivers/sftp/meta.go b/drivers/sftp/meta.go index 9dada9efa..1c9bd3e33 100644 --- a/drivers/sftp/meta.go +++ b/drivers/sftp/meta.go @@ -16,12 +16,12 @@ type Addition struct { } var config = driver.Config{ - Name: "SFTP", - LocalSort: true, - OnlyLinkMFile: false, - DefaultRoot: "/", - CheckStatus: true, - NoLinkURL: true, + Name: "SFTP", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: "/", + CheckStatus: true, + NoLinkURL: true, } func init() { diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go index 0948834e0..00a32f001 100644 --- a/drivers/sftp/types.go +++ b/drivers/sftp/types.go @@ -11,15 +11,16 @@ import ( func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { symlink := f.Mode()&os.ModeSymlink != 0 + path := stdpath.Join(dir, f.Name()) if !symlink { return &model.Object{ + Path: path, Name: f.Name(), Size: f.Size(), Modified: f.ModTime(), IsFolder: f.IsDir(), }, nil } - path := stdpath.Join(dir, f.Name()) // set target path target, err := d.client.ReadLink(path) if err != nil { @@ -32,6 +33,7 @@ func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { if err != nil { if d.IgnoreSymlinkError { return &model.Object{ + Path: path, Name: f.Name(), Size: f.Size(), Modified: f.ModTime(), diff --git a/drivers/smb/driver.go b/drivers/smb/driver.go index 3aeffbeba..88325e5cf 100644 --- a/drivers/smb/driver.go +++ b/drivers/smb/driver.go @@ -3,6 +3,7 @@ package smb import ( "context" "errors" + "path" "path/filepath" "strings" @@ -54,17 +55,17 @@ func (d *SMB) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m return nil, err } d.updateLastConnTime() - var files []model.Obj + files := make([]model.Obj, 0, len(rawFiles)) for _, f := range rawFiles { - file := model.ObjThumb{ - Object: model.Object{ - Name: f.Name(), - Modified: f.ModTime(), - Size: f.Size(), - IsFolder: f.IsDir(), - Ctime: f.(*smb2.FileStat).CreationTime, - }, + file := model.Object{ + Path: path.Join(fullPath, f.Name()), + Name: f.Name(), + Modified: f.ModTime(), + Size: f.Size(), + IsFolder: f.IsDir(), + Ctime: f.(*smb2.FileStat).CreationTime, } + files = append(files, &file) } return files, nil @@ -86,15 +87,10 @@ func (d *SMB) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*m Limiter: stream.ServerDownloadLimit, Ctx: ctx, } - if !d.Config().OnlyLinkMFile { - return &model.Link{ - RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), - SyncClosers: utils.NewSyncClosers(remoteFile), - }, nil - } return &model.Link{ - MFile: mFile, - SyncClosers: utils.NewSyncClosers(remoteFile), + RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), mFile), + SyncClosers: utils.NewSyncClosers(remoteFile), + RequireReference: true, }, nil } @@ -213,10 +209,12 @@ func (d *SMB) GetDetails(ctx context.Context) (*model.StorageDetails, error) { if err != nil { return nil, err } + total := int64(stat.BlockSize() * stat.TotalBlockCount()) + free := int64(stat.BlockSize() * stat.AvailableBlockCount()) return &model.StorageDetails{ DiskUsage: model.DiskUsage{ - TotalSpace: stat.BlockSize() * stat.TotalBlockCount(), - FreeSpace: stat.BlockSize() * stat.AvailableBlockCount(), + TotalSpace: total, + UsedSpace: total - free, }, }, nil } diff --git a/drivers/smb/meta.go b/drivers/smb/meta.go index 87a98277e..ce0e5d3b7 100644 --- a/drivers/smb/meta.go +++ b/drivers/smb/meta.go @@ -14,12 +14,12 @@ type Addition struct { } var config = driver.Config{ - Name: "SMB", - LocalSort: true, - OnlyLinkMFile: false, - DefaultRoot: ".", - NoCache: true, - NoLinkURL: true, + Name: "SMB", + LocalSort: true, + OnlyProxy: true, + DefaultRoot: ".", + NoCache: true, + NoLinkURL: true, } func init() { diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go index 010defa90..422f0e1b1 100644 --- a/drivers/strm/driver.go +++ b/drivers/strm/driver.go @@ -12,8 +12,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" + log "github.com/sirupsen/logrus" ) type Strm struct { @@ -39,6 +41,9 @@ func (d *Strm) Init(ctx context.Context) error { if d.Paths == "" { return errors.New("paths is required") } + if d.SaveStrmToLocal && len(d.SaveStrmLocalPath) <= 0 { + return errors.New("SaveStrmLocalPath is required") + } d.pathMap = make(map[string][]string) for _, path := range strings.Split(d.Paths, "\n") { path = strings.TrimSpace(path) @@ -47,6 +52,13 @@ func (d *Strm) Init(ctx context.Context) error { } k, v := getPair(path) d.pathMap[k] = append(d.pathMap[k], v) + if d.SaveStrmToLocal { + err := InsertStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d) + if err != nil { + log.Errorf("insert strmTrie error: %v", err) + continue + } + } } if len(d.pathMap) == 1 { for k := range d.pathMap { @@ -58,26 +70,55 @@ func (d *Strm) Init(ctx context.Context) error { d.autoFlatten = false } - d.supportSuffix = supportSuffix() - if d.FilterFileTypes != "" { - types := strings.Split(d.FilterFileTypes, ",") + var supportTypes []string + if d.FilterFileTypes == "" { + d.FilterFileTypes = "mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" + } + supportTypes = strings.Split(d.FilterFileTypes, ",") + d.supportSuffix = map[string]struct{}{} + for _, ext := range supportTypes { + ext = strings.ToLower(strings.TrimSpace(ext)) + if ext != "" { + d.supportSuffix[ext] = struct{}{} + } + } + + var downloadTypes []string + if d.DownloadFileTypes == "" { + d.DownloadFileTypes = "ass,srt,vtt,sub,strm" + } + downloadTypes = strings.Split(d.DownloadFileTypes, ",") + d.downloadSuffix = map[string]struct{}{} + for _, ext := range downloadTypes { + ext = strings.ToLower(strings.TrimSpace(ext)) + if ext != "" { + d.downloadSuffix[ext] = struct{}{} + } + } + + if d.Version != 5 { + types := strings.Split("mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac", ",") for _, ext := range types { - ext = strings.ToLower(strings.TrimSpace(ext)) - if ext != "" { + if _, ok := d.supportSuffix[ext]; !ok { d.supportSuffix[ext] = struct{}{} + supportTypes = append(supportTypes, ext) } } - } + d.FilterFileTypes = strings.Join(supportTypes, ",") - d.downloadSuffix = downloadSuffix() - if d.DownloadFileTypes != "" { - downloadTypes := strings.Split(d.DownloadFileTypes, ",") - for _, ext := range downloadTypes { - ext = strings.ToLower(strings.TrimSpace(ext)) - if ext != "" { + types = strings.Split("ass,srt,vtt,sub,strm", ",") + for _, ext := range types { + if _, ok := d.downloadSuffix[ext]; !ok { d.downloadSuffix[ext] = struct{}{} + downloadTypes = append(downloadTypes, ext) } } + d.DownloadFileTypes = strings.Join(downloadTypes, ",") + d.PathPrefix = "/d" + d.Version = 5 + } + if len(d.SaveLocalMode) == 0 { + d.SaveLocalMode = SaveLocalInsertMode } return nil } @@ -86,17 +127,17 @@ func (d *Strm) Drop(ctx context.Context) error { d.pathMap = nil d.downloadSuffix = nil d.supportSuffix = nil + for _, path := range strings.Split(d.Paths, "\n") { + RemoveStrm(utils.FixAndCleanPath(strings.TrimSpace(path)), d) + } return nil } +func (Addition) GetRootPath() string { + return "/" +} + func (d *Strm) Get(ctx context.Context, path string) (model.Obj, error) { - if utils.PathEqual(path, "/") { - return &model.Object{ - Name: "Root", - IsFolder: true, - Path: "/", - }, nil - } root, sub := d.getRootAndPath(path) dsts, ok := d.pathMap[root] if !ok { @@ -156,7 +197,7 @@ func (d *Strm) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* if file.GetID() == "strm" { link := d.getLink(ctx, file.GetPath()) return &model.Link{ - MFile: strings.NewReader(link), + RangeReader: stream.GetRangeReaderFromMFile(int64(len(link)), strings.NewReader(link)), }, nil } // ftp,s3 diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go new file mode 100644 index 000000000..78ec1e7e6 --- /dev/null +++ b/drivers/strm/hook.go @@ -0,0 +1,237 @@ +package strm + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "os" + stdpath "path" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" + "github.com/tchap/go-patricia/v2/patricia" +) + +var strmTrie = patricia.NewTrie() + +func UpdateLocalStrm(ctx context.Context, path string, objs []model.Obj) { + path = utils.FixAndCleanPath(path) + updateLocal := func(driver *Strm, basePath string, objs []model.Obj) { + relParent := strings.TrimPrefix(basePath, utils.GetActualMountPath(driver.MountPath)) + localParentPath := stdpath.Join(driver.SaveStrmLocalPath, relParent) + for _, obj := range objs { + localPath := stdpath.Join(localParentPath, obj.GetName()) + generateStrm(ctx, driver, obj, localPath) + } + deleteExtraFiles(driver, localParentPath, objs) + } + + _ = strmTrie.VisitPrefixes(patricia.Prefix(path), func(needPathPrefix patricia.Prefix, item patricia.Item) error { + strmDrivers := item.([]*Strm) + needPath := string(needPathPrefix) + restPath := strings.TrimPrefix(path, needPath) + if len(restPath) > 0 && restPath[0] != '/' { + return nil + } + for _, strmDriver := range strmDrivers { + strmObjs := strmDriver.convert2strmObjs(ctx, path, objs) + updateLocal(strmDriver, stdpath.Join(stdpath.Base(needPath), restPath), strmObjs) + } + return nil + }) +} + +func InsertStrm(dstPath string, d *Strm) error { + prefix := patricia.Prefix(strings.TrimRight(dstPath, "/")) + existing := strmTrie.Get(prefix) + + if existing == nil { + if !strmTrie.Insert(prefix, []*Strm{d}) { + return errors.New("failed to insert strm") + } + return nil + } + if lst, ok := existing.([]*Strm); ok { + strmTrie.Set(prefix, append(lst, d)) + } else { + return errors.New("invalid trie item type") + } + + return nil +} + +func RemoveStrm(dstPath string, d *Strm) { + prefix := patricia.Prefix(strings.TrimRight(dstPath, "/")) + existing := strmTrie.Get(prefix) + if existing == nil { + return + } + lst, ok := existing.([]*Strm) + if !ok { + return + } + if len(lst) == 1 && lst[0] == d { + strmTrie.Delete(prefix) + return + } + + for i, di := range lst { + if di == d { + newList := append(lst[:i], lst[i+1:]...) + strmTrie.Set(prefix, newList) + return + } + } +} + +func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) { + if !obj.IsDir() { + if utils.Exists(localPath) && driver.SaveLocalMode == SaveLocalInsertMode { + return + } + link, err := driver.Link(ctx, obj, model.LinkArgs{}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err) + return + } + defer link.Close() + size := link.ContentLength + if size <= 0 { + size = obj.GetSize() + } + rrf, err := stream.GetRangeReaderFromLink(size, link) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to get range reader: %v", localPath, err) + return + } + rc, err := rrf.RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to read range: %v", localPath, err) + return + } + defer rc.Close() + same, err := isSameContent(localPath, size, rc) + if err != nil { + log.Warnf("failed to compare content of obj %s: %v", localPath, err) + return + } + if same { + return + } + rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1}) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to reread range: %v", localPath, err) + return + } + defer rc.Close() + file, err := utils.CreateNestedFile(localPath) + if err != nil { + log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err) + return + } + defer file.Close() + if _, err := utils.CopyWithBuffer(file, rc); err != nil { + log.Warnf("failed to generate strm of obj %s: copy failed: %v", localPath, err) + } + } +} + +func isSameContent(localPath string, size int64, rc io.Reader) (bool, error) { + info, err := os.Stat(localPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + if info.Size() != size { + return false, nil + } + localFile, err := os.Open(localPath) + if err != nil { + return false, err + } + defer localFile.Close() + h1 := sha256.New() + h2 := sha256.New() + if _, err := io.Copy(h1, localFile); err != nil { + return false, err + } + if _, err := io.Copy(h2, rc); err != nil { + return false, err + } + return bytes.Equal(h1.Sum(nil), h2.Sum(nil)), nil +} + +func deleteExtraFiles(driver *Strm, localPath string, objs []model.Obj) { + if driver.SaveLocalMode != SaveLocalSyncMode { + return + } + localFiles, localDirs, err := getLocalDirsAndFiles(localPath) + if err != nil { + log.Errorf("Failed to read local files from %s: %v", localPath, err) + return + } + + fileSet := make(map[string]struct{}) + dirSet := make(map[string]struct{}) + for _, obj := range objs { + objPath := stdpath.Join(localPath, obj.GetName()) + if obj.IsDir() { + dirSet[objPath] = struct{}{} + } else { + fileSet[objPath] = struct{}{} + } + } + + for _, localFile := range localFiles { + if _, exists := fileSet[localFile]; !exists { + err := os.Remove(localFile) + if err != nil { + log.Errorf("Failed to delete file: %s, error: %v\n", localFile, err) + } else { + log.Infof("Deleted file %s", localFile) + } + } + } + + for _, localDir := range localDirs { + if _, exists := dirSet[localDir]; !exists { + err := os.RemoveAll(localDir) + if err != nil { + log.Errorf("Failed to delete directory: %s, error: %v\n", localDir, err) + } else { + log.Infof("Deleted directory %s", localDir) + } + } + } +} + +func getLocalDirsAndFiles(localPath string) ([]string, []string, error) { + var files, dirs []string + entries, err := os.ReadDir(localPath) + if err != nil { + return nil, nil, err + } + for _, entry := range entries { + fullPath := stdpath.Join(localPath, entry.Name()) + if entry.IsDir() { + dirs = append(dirs, fullPath) + } else { + files = append(files, fullPath) + } + } + return files, dirs, nil +} + +func init() { + op.RegisterObjsUpdateHook(UpdateLocalStrm) +} diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go index 0f3e0a994..843ff7c66 100644 --- a/drivers/strm/meta.go +++ b/drivers/strm/meta.go @@ -5,24 +5,35 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" ) +const ( + SaveLocalInsertMode = "insert" + SaveLocalUpdateMode = "update" + SaveLocalSyncMode = "sync" +) + type Addition struct { Paths string `json:"paths" required:"true" type:"text"` SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of the strm file"` - FilterFileTypes string `json:"filterFileTypes" type:"text" default:"strm" required:"false" help:"Supports suffix name of strm file"` - DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass" required:"false" help:"Files need to download with strm (usally subtitles)"` + PathPrefix string `json:"PathPrefix" type:"text" required:"false" default:"/d" help:"Path prefix"` + DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Files need to download with strm (usally subtitles)"` + FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Supports suffix name of strm file"` EncodePath bool `json:"encodePath" default:"true" required:"true" help:"encode the path in the strm file"` - LocalModel bool `json:"localModel" default:"false" help:"enable local mode"` + WithoutUrl bool `json:"withoutUrl" default:"false" help:"strm file content without URL prefix"` + WithSign bool `json:"withSign" default:"false"` + SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"save strm file locally"` + SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"save strm file local path"` + SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"save strm file locally mode" options:"insert,update,sync" default:"insert"` + Version int } var config = driver.Config{ - Name: "Strm", - LocalSort: true, - NoCache: true, - NoUpload: true, - DefaultRoot: "/", - OnlyLinkMFile: true, - OnlyProxy: true, - NoLinkURL: true, + Name: "Strm", + LocalSort: true, + OnlyProxy: true, + NoCache: true, + NoUpload: true, + DefaultRoot: "/", + NoLinkURL: true, } func init() { diff --git a/drivers/strm/types.go b/drivers/strm/types.go deleted file mode 100644 index dfa427d42..000000000 --- a/drivers/strm/types.go +++ /dev/null @@ -1,36 +0,0 @@ -package strm - -func supportSuffix() map[string]struct{} { - return map[string]struct{}{ - // video - "mp4": {}, - "mkv": {}, - "flv": {}, - "avi": {}, - "wmv": {}, - "ts": {}, - "rmvb": {}, - "webm": {}, - // audio - "mp3": {}, - "flac": {}, - "aac": {}, - "wav": {}, - "ogg": {}, - "m4a": {}, - "wma": {}, - "alac": {}, - } -} - -func downloadSuffix() map[string]struct{} { - return map[string]struct{}{ - // strm - "strm": {}, - // subtitles - "ass": {}, - "srt": {}, - "vtt": {}, - "sub": {}, - } -} diff --git a/drivers/strm/util.go b/drivers/strm/util.go index cfa441db8..b9a40da5e 100644 --- a/drivers/strm/util.go +++ b/drivers/strm/util.go @@ -3,7 +3,6 @@ package strm import ( "context" "fmt" - stdpath "path" "strings" @@ -19,6 +18,7 @@ func (d *Strm) listRoot() []model.Obj { var objs []model.Obj for k := range d.pathMap { obj := model.Object{ + Path: "/" + k, Name: k, IsFolder: true, Modified: d.Modified, @@ -58,20 +58,24 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([] if err != nil { return nil, err } + return d.convert2strmObjs(ctx, reqPath, objs), nil +} +func (d *Strm) convert2strmObjs(ctx context.Context, reqPath string, objs []model.Obj) []model.Obj { var validObjs []model.Obj for _, obj := range objs { id, name, path := "", obj.GetName(), "" size := int64(0) if !obj.IsDir() { path = stdpath.Join(reqPath, obj.GetName()) - ext := strings.ToLower(utils.Ext(name)) - if _, ok := d.supportSuffix[ext]; ok { + sourceExt := utils.SourceExt(name) + ext := strings.ToLower(sourceExt) + if _, ok := d.downloadSuffix[ext]; ok { + size = obj.GetSize() + } else if _, ok := d.supportSuffix[ext]; ok { id = "strm" - name = strings.TrimSuffix(name, ext) + "strm" + name = strings.TrimSuffix(name, sourceExt) + "strm" size = int64(len(d.getLink(ctx, path))) - } else if _, ok := d.downloadSuffix[ext]; ok { - size = obj.GetSize() } else { continue } @@ -84,13 +88,11 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([] Modified: obj.ModTime(), IsFolder: obj.IsDir(), } - thumb, ok := model.GetThumb(obj) if !ok { validObjs = append(validObjs, &objRes) continue } - validObjs = append(validObjs, &model.ObjThumb{ Object: objRes, Thumbnail: model.Thumbnail{ @@ -98,7 +100,7 @@ func (d *Strm) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([] }, }) } - return validObjs, nil + return validObjs } func (d *Strm) getLink(ctx context.Context, path string) string { @@ -106,11 +108,18 @@ func (d *Strm) getLink(ctx context.Context, path string) string { if d.EncodePath { finalPath = utils.EncodePath(path, true) } - if d.EnableSign { + if d.WithSign { signPath := sign.Sign(path) finalPath = fmt.Sprintf("%s?sign=%s", finalPath, signPath) } - if d.LocalModel { + pathPrefix := d.PathPrefix + if len(pathPrefix) > 0 { + finalPath = stdpath.Join(pathPrefix, finalPath) + } + if !strings.HasPrefix(finalPath, "/") { + finalPath = "/" + finalPath + } + if d.WithoutUrl { return finalPath } apiUrl := d.SiteUrl @@ -119,8 +128,7 @@ func (d *Strm) getLink(ctx context.Context, path string) string { } else { apiUrl = common.GetApiUrl(ctx) } - - return fmt.Sprintf("%s/d%s", + return fmt.Sprintf("%s%s", apiUrl, finalPath) } diff --git a/drivers/teldrive/driver.go b/drivers/teldrive/driver.go index 541d2e3be..d420eb4d0 100644 --- a/drivers/teldrive/driver.go +++ b/drivers/teldrive/driver.go @@ -6,6 +6,8 @@ import ( "math" "net/http" "net/url" + "path" + "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -16,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" + "golang.org/x/sync/errgroup" ) type Teldrive struct { @@ -52,19 +55,60 @@ func (d *Teldrive) Drop(ctx context.Context) error { } func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - var listResp ListResp + var firstResp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "path": dir.GetPath(), - "limit": "1000", // overide default 500, TODO pagination + "limit": "500", + "page": "1", }) - }, &listResp) + }, &firstResp) + if err != nil { return nil, err } - return utils.SliceConvert(listResp.Items, func(src Object) (model.Obj, error) { + pagesData := make([][]Object, firstResp.Meta.TotalPages) + pagesData[0] = firstResp.Items + + if firstResp.Meta.TotalPages > 1 { + g, _ := errgroup.WithContext(ctx) + g.SetLimit(8) + + for i := 2; i <= firstResp.Meta.TotalPages; i++ { + page := i + g.Go(func() error { + var resp ListResp + err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "path": dir.GetPath(), + "limit": "500", + "page": strconv.Itoa(page), + }) + }, &resp) + + if err != nil { + return err + } + + pagesData[page-1] = resp.Items + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + } + + var allItems []Object + for _, items := range pagesData { + allItems = append(allItems, items...) + } + + return utils.SliceConvert(allItems, func(src Object) (model.Obj, error) { return &model.Object{ + Path: path.Join(dir.GetPath(), src.Name), ID: src.ID, Name: src.Name, Size: func() int64 { @@ -182,7 +226,7 @@ func (d *Teldrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStr } if totalParts <= 1 { - return d.doSingleUpload(ctx, dstDir, file, up, totalParts, chunkSize, fileId) + return d.doSingleUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) } return d.doMultiUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) diff --git a/drivers/teldrive/meta.go b/drivers/teldrive/meta.go index 23bae5f94..cc7a5dbf7 100644 --- a/drivers/teldrive/meta.go +++ b/drivers/teldrive/meta.go @@ -11,6 +11,7 @@ type Addition struct { Cookie string `json:"cookie" type:"string" required:"true" help:"access_token=xxx"` UseShareLink bool `json:"use_share_link" type:"bool" default:"false" help:"Create share link when getting link to support 302. If disabled, you need to enable web proxy."` ChunkSize int64 `json:"chunk_size" type:"number" default:"10" help:"Chunk size in MiB"` + RandomChunkName bool `json:"random_chunk_name" type:"bool" default:"true" help:"Random chunk name"` UploadConcurrency int64 `json:"upload_concurrency" type:"number" default:"4" help:"Concurrency upload requests"` } diff --git a/drivers/teldrive/upload.go b/drivers/teldrive/upload.go index 168d9beff..b94f5fc93 100644 --- a/drivers/teldrive/upload.go +++ b/drivers/teldrive/upload.go @@ -1,6 +1,8 @@ package teldrive import ( + "crypto/md5" + "encoding/hex" "fmt" "io" "net/http" @@ -16,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + "github.com/google/uuid" "github.com/pkg/errors" "golang.org/x/net/context" "golang.org/x/sync/errgroup" @@ -38,6 +41,11 @@ func (d *Teldrive) touch(name, path string) error { return nil } +func getMD5Hash(text string) string { + hash := md5.Sum([]byte(text)) + return hex.EncodeToString(hash[:]) +} + func (d *Teldrive) createFileOnUploadSuccess(name, id, path string, uploadedFileParts []FilePart, totalSize int64) error { remoteFileParts, err := d.getFilePart(id) if err != nil { @@ -101,12 +109,10 @@ func (d *Teldrive) getFilePart(fileId string) ([]FilePart, error) { return uploadedParts, nil } -func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback, resp interface{}) error { +func (d *Teldrive) singleUploadRequest(ctx context.Context, fileId string, callback base.ReqCallback, resp any) error { url := d.Address + "/api/uploads/" + fileId client := resty.New().SetTimeout(0) - ctx := context.Background() - req := client.R(). SetContext(ctx) req.SetHeader("Cookie", d.Cookie) @@ -135,16 +141,18 @@ func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback, } func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up model.UpdateProgress, - totalParts int, chunkSize int64, fileId string) error { + maxRetried, totalParts int, chunkSize int64, fileId string) error { totalSize := file.GetSize() var fileParts []FilePart var uploaded int64 = 0 - ss, err := stream.NewStreamSectionReader(file, int(totalSize), &up) + var partName string + chunkSize = min(totalSize, chunkSize) + ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up) if err != nil { return err } - + chunkCnt := 0 for uploaded < totalSize { if utils.IsCanceled(ctx) { return ctx.Err() @@ -154,6 +162,7 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo if err != nil { return err } + chunkCnt += 1 filePart := &FilePart{} if err := retry.Do(func() error { @@ -161,13 +170,19 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo return err } - if err := d.singleUploadRequest(fileId, func(req *resty.Request) { + if d.RandomChunkName { + partName = getMD5Hash(uuid.New().String()) + } else { + partName = file.GetName() + if totalParts > 1 { + partName = fmt.Sprintf("%s.part.%03d", file.GetName(), chunkCnt) + } + } + + if err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ - "partName": func() string { - digits := len(fmt.Sprintf("%d", totalParts)) - return file.GetName() + fmt.Sprintf(".%0*d", digits, 1) - }(), - "partNo": strconv.Itoa(1), + "partName": partName, + "partNo": strconv.Itoa(chunkCnt), "fileName": file.GetName(), } req.SetQueryParams(uploadParams) @@ -179,7 +194,8 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo return nil }, - retry.Attempts(3), + retry.Context(ctx), + retry.Attempts(uint(maxRetried)), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)); err != nil { return err @@ -188,8 +204,11 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo if filePart.Name != "" { fileParts = append(fileParts, *filePart) uploaded += curChunkSize - up(float64(uploaded) / float64(totalSize)) + up(float64(uploaded) / float64(totalSize) * 100) ss.FreeSectionReader(rd) + } else { + // For common situation this code won't reach + return fmt.Errorf("[Teldrive] upload chunk %d failed: filePart Somehow missing", chunkCnt) } } @@ -317,6 +336,7 @@ func (d *Teldrive) doMultiUpload(ctx context.Context, dstDir model.Obj, file mod func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task chunkTask, totalParts, maxRetried int) (*FilePart, error) { filePart := &FilePart{} retryCount := 0 + var partName string defer task.ss.FreeSectionReader(task.reader) for { @@ -330,12 +350,22 @@ func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task ch return &existingPart, nil } - err := d.singleUploadRequest(fileId, func(req *resty.Request) { + if _, err := task.reader.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + if d.RandomChunkName { + partName = getMD5Hash(uuid.New().String()) + } else { + partName = task.fileName + if totalParts > 1 { + partName = fmt.Sprintf("%s.part.%03d", task.fileName, task.chunkIdx) + } + } + + err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ - "partName": func() string { - digits := len(fmt.Sprintf("%d", totalParts)) - return task.fileName + fmt.Sprintf(".%0*d", digits, task.chunkIdx) - }(), + "partName": partName, "partNo": strconv.Itoa(task.chunkIdx), "fileName": task.fileName, } diff --git a/drivers/template/meta.go b/drivers/template/meta.go index a546e676b..f525b5a95 100644 --- a/drivers/template/meta.go +++ b/drivers/template/meta.go @@ -16,7 +16,6 @@ type Addition struct { var config = driver.Config{ Name: "Template", LocalSort: false, - OnlyLinkMFile: false, OnlyProxy: false, NoCache: false, NoUpload: false, diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index 3c9d265c2..5323b1887 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -7,7 +7,6 @@ import ( "encoding/hex" "fmt" "io" - "math" stdpath "path" "strconv" @@ -63,7 +62,9 @@ func (d *Terabox) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil + obj := fileToObj(src) + obj.Path = stdpath.Join(dir.GetPath(), obj.Name) + return obj, nil }) } @@ -193,7 +194,7 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt streamSize := stream.GetSize() chunkSize := calculateChunkSize(streamSize) chunkByteData := make([]byte, chunkSize) - count := int(math.Ceil(float64(streamSize) / float64(chunkSize))) + count := int((streamSize + chunkSize - 1) / chunkSize) left := streamSize uploadBlockList := make([]string, 0, count) h := md5.New() diff --git a/drivers/terabox/util.go b/drivers/terabox/util.go index 518fd4f9d..69944d346 100644 --- a/drivers/terabox/util.go +++ b/drivers/terabox/util.go @@ -88,7 +88,7 @@ func (d *Terabox) request(rurl string, method string, callback base.ReqCallback, return nil, err } errno := utils.Json.Get(res.Body(), "errno").ToInt() - if errno == 4000023 || errno == 4500016 { + if errno == 4000023 || errno == 450016 { // reget jsToken err = d.resetJsToken() if err != nil { diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 7f537ac64..492b9814f 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "net/url" "strconv" "strings" @@ -68,6 +69,7 @@ func (x *Thunder) Init(ctx context.Context) (err error) { PackageName: "com.xunlei.downloadprovider", UserAgent: "ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)", + Space: x.Space, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) @@ -167,6 +169,7 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) { UserAgent: x.UserAgent, DownloadUserAgent: x.DownloadUserAgent, UseVideoUrl: x.UseVideoUrl, + Space: x.Space, refreshCTokenCk: func(token string) { x.CaptchaToken = token @@ -281,7 +284,7 @@ func (xc *XunLeiCommon) Link(ctx context.Context, file model.Obj, args model.Lin _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", file.GetID()) - //r.SetQueryParam("space", "") + r.SetQueryParam("space", xc.Space) }, &lFile) if err != nil { return nil, err @@ -322,6 +325,7 @@ func (xc *XunLeiCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirNam "kind": FOLDER, "name": dirName, "parent_id": parentDir.GetID(), + "space": xc.Space, }) }, nil) return err @@ -331,8 +335,9 @@ func (xc *XunLeiCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) erro _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ - "to": base.Json{"parent_id": dstDir.GetID()}, - "ids": []string{srcObj.GetID()}, + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + "space": xc.Space, }) }, nil) return err @@ -342,7 +347,10 @@ func (xc *XunLeiCommon) Rename(ctx context.Context, srcObj model.Obj, newName st _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", srcObj.GetID()) - r.SetBody(&base.Json{"name": newName}) + r.SetBody(&base.Json{ + "name": newName, + "space": xc.Space, + }) }, nil) return err } @@ -351,8 +359,9 @@ func (xc *XunLeiCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) erro _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ - "to": base.Json{"parent_id": dstDir.GetID()}, - "ids": []string{srcObj.GetID()}, + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + "space": xc.Space, }) }, nil) return err @@ -362,6 +371,7 @@ func (xc *XunLeiCommon) Remove(ctx context.Context, obj model.Obj) error { _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", obj.GetID()) + r.SetQueryParam("space", xc.Space) r.SetBody("{}") }, nil) return err @@ -387,6 +397,7 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi "size": file.GetSize(), "hash": gcid, "upload_type": UPLOAD_TYPE_RESUMABLE, + "space": xc.Space, }) }, &resp) if err != nil { @@ -422,6 +433,32 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi return nil } +func (xc *XunLeiCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string @@ -430,7 +467,7 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model. _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(map[string]string{ - "space": "", + "space": xc.Space, "__type": "drive", "refresh": "true", "__sync": "true", @@ -440,6 +477,17 @@ func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model. "limit": "100", "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, }) + // 获取硬盘挂载目录等 + if xc.Space != "" { + r.SetQueryParamsFromValues(url.Values{ + "with": []string{ + "withCategoryDiskMountPath", + "withCategoryDriveCachePath", + "withCategoryHistoryDownloadPath", + "withReadOnlyFS", + }, + }) + } }, &fileList) if err != nil { return nil, err @@ -576,6 +624,7 @@ func (xc *XunLeiCommon) OfflineDownload(ctx context.Context, fileUrl string, par "name": fileName, "parent_id": parentDir.GetID(), "upload_type": UPLOAD_TYPE_URL, + "space": xc.Space, "url": base.Json{ "url": fileUrl, }, @@ -602,6 +651,7 @@ func (xc *XunLeiCommon) OfflineList(ctx context.Context, nextPageToken string) ( "type": "offline", "limit": "10000", "page_token": nextPageToken, + "space": xc.Space, }) }, &resp) @@ -618,6 +668,7 @@ func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string SetQueryParams(map[string]string{ "task_ids": strings.Join(taskIDs, ","), "delete_files": strconv.FormatBool(deleteFiles), + "space": xc.Space, }) }, nil) if err != nil { diff --git a/drivers/thunder/meta.go b/drivers/thunder/meta.go index e5c0f977d..be9c712fd 100644 --- a/drivers/thunder/meta.go +++ b/drivers/thunder/meta.go @@ -46,6 +46,8 @@ type ExpertAddition struct { //优先使用视频链接代替下载链接 UseVideoUrl bool `json:"use_video_url"` + + Space string `json:"space" default:"" help:"device id for remote device"` } // 登录特征,用于判断是否重新登录 @@ -80,6 +82,8 @@ type Addition struct { CreditKey string `json:"credit_key" help:"credit key,used for login"` // 登录设备ID DeviceID string `json:"device_id" default:""` + + Space string `json:"space" default:"" help:"device id for remote device"` } // 登录特征,用于判断是否重新登录 @@ -90,7 +94,6 @@ func (i *Addition) GetIdentity() string { var config = driver.Config{ Name: "Thunder", LocalSort: true, - OnlyProxy: true, } var configExpert = driver.Config{ diff --git a/drivers/thunder/types.go b/drivers/thunder/types.go index 7b3ad5692..fcfa1fb1c 100644 --- a/drivers/thunder/types.go +++ b/drivers/thunder/types.go @@ -347,3 +347,21 @@ type ReviewData struct { Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } + +type AboutResponse struct { + // Kind string `json:"kind"` + Quota struct { + // Kind string `json:"kind"` + Limit string `json:"limit"` + Usage string `json:"usage"` + // UsageInTrash string `json:"usage_in_trash"` + // PlayTimesLimit string `json:"play_times_limit"` + // PlayTimesUsage string `json:"play_times_usage"` + // IsUnlimited bool `json:"is_unlimited"` + // UpgradeType string `json:"upgrade_type"` + } `json:"quota"` + // ExpiresAt string `json:"expires_at"` + // Quotas struct { + // } `json:"quotas"` + // IsSearchFlushed bool `json:"is_search_flushed"` +} diff --git a/drivers/thunder/util.go b/drivers/thunder/util.go index f4733613b..1fa01a90e 100644 --- a/drivers/thunder/util.go +++ b/drivers/thunder/util.go @@ -68,6 +68,7 @@ type Common struct { UserAgent string DownloadUserAgent string UseVideoUrl bool + Space string // 验证码token刷新成功回调 refreshCTokenCk func(token string) diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index bf1843a41..e60522e52 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "strconv" "strings" "time" @@ -542,6 +543,32 @@ func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream return nil } +func (xc *XunLeiBrowserCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, dir model.Obj, path string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string @@ -840,7 +867,7 @@ func (xc *XunLeiBrowserCommon) OfflineList(ctx context.Context, nextPageToken st func (xc *XunLeiBrowserCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string) error { queryParams := map[string]string{ "task_ids": strings.Join(taskIDs, ","), - "_t": fmt.Sprintf("%d", time.Now().UnixMilli()), + "_t": strconv.FormatInt(time.Now().UnixMilli(), 10), } if xc.UseFluentPlay { queryParams["space"] = ThunderBrowserDriveFluentPlayFolderType diff --git a/drivers/thunder_browser/types.go b/drivers/thunder_browser/types.go index 6b2a41023..a5fda2a5d 100644 --- a/drivers/thunder_browser/types.go +++ b/drivers/thunder_browser/types.go @@ -376,3 +376,10 @@ type ReviewData struct { Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + } `json:"quota"` +} diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index 86ff22bdc..acbb82513 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -423,6 +423,32 @@ func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.F return nil } +func (xc *XunLeiXCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiXCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string diff --git a/drivers/thunderx/types.go b/drivers/thunderx/types.go index e5fbaa241..728fdea31 100644 --- a/drivers/thunderx/types.go +++ b/drivers/thunderx/types.go @@ -303,3 +303,10 @@ type Media struct { IsVisible bool `json:"is_visible"` Category string `json:"category"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + } `json:"quota"` +} diff --git a/drivers/url_tree/driver.go b/drivers/url_tree/driver.go index 36579d3cd..c1af0b3fb 100644 --- a/drivers/url_tree/driver.go +++ b/drivers/url_tree/driver.go @@ -44,6 +44,10 @@ func (d *Urls) Drop(ctx context.Context) error { return nil } +func (Addition) GetRootPath() string { + return "/" +} + func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) { d.mutex.RLock() defer d.mutex.RUnlock() diff --git a/drivers/url_tree/meta.go b/drivers/url_tree/meta.go index 1ec57ee0a..9d2f182ba 100644 --- a/drivers/url_tree/meta.go +++ b/drivers/url_tree/meta.go @@ -6,10 +6,6 @@ import ( ) type Addition struct { - // Usually one of two - // driver.RootPath - // driver.RootID - // define other UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://raw.githubusercontent.com/OpenListTeam/OpenList/main/README.md\nhttps://raw.githubusercontent.com/OpenListTeam/OpenList/main/README_cn.md\nfolder:\n CONTRIBUTING.md:1635:https://raw.githubusercontent.com/OpenListTeam/OpenList/main/CONTRIBUTING.md\n CODE_OF_CONDUCT.md:2093:https://raw.githubusercontent.com/OpenListTeam/OpenList/main/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n [FileName:][FileSize:][Modified:]Url"` HeadSize bool `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."` Writable bool `json:"writable" type:"bool" default:"false"` @@ -20,6 +16,7 @@ var config = driver.Config{ LocalSort: true, NoCache: true, CheckStatus: true, + OnlyIndices: true, } func init() { diff --git a/drivers/uss/driver.go b/drivers/uss/driver.go index 8f33a5b0b..3606278d7 100644 --- a/drivers/uss/driver.go +++ b/drivers/uss/driver.go @@ -62,6 +62,7 @@ func (d *USS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m for obj := range objsChan { t := obj.Time f := model.Object{ + Path: path.Join(dir.GetPath(), obj.Name), Name: obj.Name, Size: obj.Size, Modified: t, diff --git a/drivers/virtual/driver.go b/drivers/virtual/driver.go index 1d14427c5..6e6bd0875 100644 --- a/drivers/virtual/driver.go +++ b/drivers/virtual/driver.go @@ -2,11 +2,11 @@ package virtual import ( "context" - "io" "time" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" ) @@ -42,16 +42,14 @@ func (d *Virtual) List(ctx context.Context, dir model.Obj, args model.ListArgs) return res, nil } -type DummyMFile struct { - io.Reader -} +type DummyMFile struct{} func (f DummyMFile) Read(p []byte) (n int, err error) { - return f.Reader.Read(p) + return random.Rand.Read(p) } func (f DummyMFile) ReadAt(p []byte, off int64) (n int, err error) { - return f.Reader.Read(p) + return random.Rand.Read(p) } func (DummyMFile) Seek(offset int64, whence int) (int64, error) { @@ -60,7 +58,7 @@ func (DummyMFile) Seek(offset int64, whence int) (int64, error) { func (d *Virtual) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { return &model.Link{ - MFile: DummyMFile{Reader: random.Rand}, + RangeReader: stream.GetRangeReaderFromMFile(file.GetSize(), DummyMFile{}), }, nil } diff --git a/drivers/virtual/meta.go b/drivers/virtual/meta.go index f567830a8..c270ec12b 100644 --- a/drivers/virtual/meta.go +++ b/drivers/virtual/meta.go @@ -14,11 +14,11 @@ type Addition struct { } var config = driver.Config{ - Name: "Virtual", - OnlyLinkMFile: true, - LocalSort: true, - NeedMs: true, - NoLinkURL: true, + Name: "Virtual", + LocalSort: true, + OnlyProxy: true, + NeedMs: true, + NoLinkURL: true, } func init() { diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index a6a18d85d..7a1b5db2c 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -2,11 +2,13 @@ package webdav import ( "context" + "fmt" "net/http" "os" "path" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/cron" @@ -54,6 +56,7 @@ func (d *WebDav) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } return utils.SliceConvert(files, func(src os.FileInfo) (model.Obj, error) { return &model.Object{ + Path: path.Join(dir.GetPath(), src.Name()), Name: src.Name(), Size: src.Size(), Modified: src.ModTime(), @@ -67,6 +70,22 @@ func (d *WebDav) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err != nil { return nil, err } + if args.Redirect { + // get the url after redirect + req := base.NoRedirectClient.R() + req.Header = header + req.SetDoNotParseResponse(true) + res, err := req.Get(url) + if err != nil { + return nil, err + } + _ = res.RawResponse.Body.Close() + if (res.StatusCode() == 302 || res.StatusCode() == 307 || res.StatusCode() == 308) && res.Header().Get("location") != "" { + url = res.Header().Get("location") + } else { + return nil, fmt.Errorf("redirect failed, status: %d", res.StatusCode()) + } + } return &model.Link{ URL: url, Header: header, diff --git a/drivers/webdav/meta.go b/drivers/webdav/meta.go index 5f8197cf6..6e7c27772 100644 --- a/drivers/webdav/meta.go +++ b/drivers/webdav/meta.go @@ -17,8 +17,8 @@ type Addition struct { var config = driver.Config{ Name: "WebDav", LocalSort: true, - OnlyProxy: true, DefaultRoot: "/", + PreferProxy: true, } func init() { diff --git a/drivers/weiyun/driver.go b/drivers/weiyun/driver.go index ef203cfa4..5178c9529 100644 --- a/drivers/weiyun/driver.go +++ b/drivers/weiyun/driver.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" weiyunsdkgo "github.com/foxxorcat/weiyun-sdk-go" + "github.com/go-resty/resty/v2" ) type WeiYun struct { @@ -67,7 +68,7 @@ func (d *WeiYun) Init(ctx context.Context) error { }) // qqCookie保活 - if d.client.LoginType() == 1 { + if d.client.LoginType() == weiyunsdkgo.AccountTypeQQ || d.client.LoginType() == weiyunsdkgo.AccountTypeQQOpenID { d.cron = cron.NewCron(time.Minute * 5) d.cron.Do(func() { _ = d.client.KeepAlive() @@ -391,6 +392,22 @@ func (d *WeiYun) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr }, nil } +func (d *WeiYun) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + info, err := d.client.DiskUserInfoGet(func(request *resty.Request) { + request.SetContext(ctx) + }) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: info.TotalSpace, + UsedSpace: info.UsedSpace, + }, + }, nil +} + // func (d *WeiYun) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport // } @@ -405,3 +422,4 @@ var _ driver.Remove = (*WeiYun)(nil) var _ driver.PutResult = (*WeiYun)(nil) var _ driver.RenameResult = (*WeiYun)(nil) +var _ driver.WithDetails = (*WeiYun)(nil) diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go index 6d9aabca8..3ff39c0cb 100644 --- a/drivers/wopan/driver.go +++ b/drivers/wopan/driver.go @@ -9,8 +9,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/go-resty/resty/v2" "github.com/OpenListTeam/wopan-sdk-go" + "github.com/go-resty/resty/v2" ) type Wopan struct { @@ -166,6 +166,23 @@ func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre return err } +func (d *Wopan) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.client.QueryCloudUsageInfo() + if err != nil { + return nil, err + } + total, err := strconv.ParseInt(quota.UsageInfo.ByteTotalSize, 10, 64) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: quota.UsageInfo.ByteUsedSize, + }, + }, nil +} + //func (d *Wopan) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/wps/driver.go b/drivers/wps/driver.go new file mode 100644 index 000000000..8a3ccb6ce --- /dev/null +++ b/drivers/wps/driver.go @@ -0,0 +1,91 @@ +package wps + +import ( + "context" + "fmt" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type Wps struct { + model.Storage + Addition + companyID string +} + +func (d *Wps) Config() driver.Config { + return config +} + +func (d *Wps) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Wps) Init(ctx context.Context) error { + if d.Cookie == "" { + return fmt.Errorf("cookie is empty") + } + return d.ensureCompanyID(ctx) +} + +func (d *Wps) Drop(ctx context.Context) error { + return nil +} + +func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]model.Obj, error) { + basePath := "/" + if dir != nil { + if p := dir.GetPath(); p != "" { + basePath = p + } + } + return d.list(ctx, basePath) +} + +func (d *Wps) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { + if file == nil { + return nil, errs.NotSupport + } + return d.link(ctx, file.GetPath()) +} + +func (d *Wps) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.makeDir(ctx, parentDir, dirName) +} + +func (d *Wps) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.move(ctx, srcObj, dstDir) +} + +func (d *Wps) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return d.rename(ctx, srcObj, newName) +} + +func (d *Wps) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.copy(ctx, srcObj, dstDir) +} + +func (d *Wps) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(ctx, obj) +} + +func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + return d.put(ctx, dstDir, file, up) +} + +func (d *Wps) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + quota, err := d.spaces(ctx) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: quota.Total, + UsedSpace: quota.Used, + }, + }, nil +} + +var _ driver.Driver = (*Wps)(nil) diff --git a/drivers/wps/meta.go b/drivers/wps/meta.go new file mode 100644 index 000000000..7a3362f3a --- /dev/null +++ b/drivers/wps/meta.go @@ -0,0 +1,26 @@ +package wps + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + driver.RootPath + Cookie string `json:"cookie" required:"true" type:"text"` + Mode string `json:"mode" type:"select" options:"Personal,Business" default:"Business"` +} + +var config = driver.Config{ + Name: "WPS", + LocalSort: true, + DefaultRoot: "/", + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Wps{} + }) +} diff --git a/drivers/wps/types.go b/drivers/wps/types.go new file mode 100644 index 000000000..a04df3d11 --- /dev/null +++ b/drivers/wps/types.go @@ -0,0 +1,107 @@ +package wps + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type workspaceResp struct { + Companies []struct { + ID int64 `json:"id"` + } `json:"companies"` +} + +type Group struct { + CompanyID int64 `json:"company_id"` + GroupID int64 `json:"group_id"` + Name string `json:"name"` + Type string `json:"type"` +} + +type groupsResp struct { + Groups []Group `json:"groups"` +} + +type filePerms struct { + Download int `json:"download"` +} + +type FileInfo struct { + GroupID int64 `json:"groupid"` + ParentID int64 `json:"parentid"` + Name string `json:"fname"` + Size int64 `json:"fsize"` + Type string `json:"ftype"` + Ctime int64 `json:"ctime"` + Mtime int64 `json:"mtime"` + ID int64 `json:"id"` + Deleted bool `json:"deleted"` + FilePerms filePerms `json:"file_perms_acl"` +} + +type filesResp struct { + Files []FileInfo `json:"files"` + NextOffset int `json:"next_offset"` +} + +type downloadResp struct { + URL string `json:"url"` + Result string `json:"result"` +} + +type spacesResp struct { + Id int64 `json:"id"` + Name string `json:"name"` + Result string `json:"result"` + Total int64 `json:"total"` + Used int64 `json:"used"` + UsedParts []struct { + Type string `json:"type"` + Used int64 `json:"used"` + } `json:"used_parts"` +} + +type Obj struct { + id string + name string + size int64 + ctime time.Time + mtime time.Time + isDir bool + hash utils.HashInfo + path string + canDownload bool +} + +func (o *Obj) GetSize() int64 { + return o.size +} + +func (o *Obj) GetName() string { + return o.name +} + +func (o *Obj) ModTime() time.Time { + return o.mtime +} + +func (o *Obj) CreateTime() time.Time { + return o.ctime +} + +func (o *Obj) IsDir() bool { + return o.isDir +} + +func (o *Obj) GetHash() utils.HashInfo { + return o.hash +} + +func (o *Obj) GetID() string { + return o.id +} + +func (o *Obj) GetPath() string { + return o.path +} diff --git a/drivers/wps/util.go b/drivers/wps/util.go new file mode 100644 index 000000000..6f8f342da --- /dev/null +++ b/drivers/wps/util.go @@ -0,0 +1,1072 @@ +package wps + +import ( + "bytes" + "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/go-resty/resty/v2" +) + +const endpoint = "https://365.kdocs.cn" +const personalEndpoint = "https://drive.wps.cn" + +type resolvedNode struct { + kind string + group Group + file *FileInfo +} + +type resolveCacheEntry struct { + node *resolvedNode + expire time.Time +} + +type resolveCacheStore struct { + mu sync.RWMutex + m map[string]resolveCacheEntry +} + +var resolveCaches sync.Map + +type apiResult struct { + Result string `json:"result"` + Msg string `json:"msg"` +} + +type uploadCreateUpdateResp struct { + apiResult + Method string `json:"method"` + URL string `json:"url"` + Store string `json:"store"` + Request struct { + Headers map[string]string `json:"headers"` + FormData map[string]string `json:"formData"` + } `json:"request"` + Response struct { + ExpectCode []int `json:"expect_code"` + ArgsETag string `json:"args_etag"` + ArgsKey string `json:"args_key"` + } `json:"response"` +} + +type uploadPutResp struct { + NewFilename string `json:"newfilename"` + Sha1 string `json:"sha1"` + MD5 string `json:"md5"` +} + +type personalGroupsResp struct { + apiResult + Groups []struct { + ID int64 `json:"id"` + Name string `json:"name"` + } `json:"groups"` +} + +type countingWriter struct { + n *int64 +} + +func (w countingWriter) Write(p []byte) (int, error) { + *w.n += int64(len(p)) + return len(p), nil +} + +func (d *Wps) isPersonal() bool { + return strings.TrimSpace(d.Mode) == "Personal" +} + +func (d *Wps) driveHost() string { + if d.isPersonal() { + return personalEndpoint + } + return endpoint +} + +func (d *Wps) drivePrefix() string { + if d.isPersonal() { + return "" + } + return "/3rd/drive" +} + +func (d *Wps) driveURL(path string) string { + return d.driveHost() + d.drivePrefix() + path +} + +func (d *Wps) origin() string { + return d.driveHost() +} + +func (d *Wps) canDownload(f *FileInfo) bool { + if f == nil || f.Type == "folder" { + return false + } + if f.FilePerms.Download != 0 { + return true + } + return d.isPersonal() +} + +func (d *Wps) request(ctx context.Context) *resty.Request { + return base.RestyClient.R(). + SetHeader("Cookie", d.Cookie). + SetHeader("Accept", "application/json"). + SetContext(ctx) +} + +func (d *Wps) jsonRequest(ctx context.Context) *resty.Request { + return d.request(ctx). + SetHeader("Content-Type", "application/json"). + SetHeader("Origin", d.origin()) +} + +func statusOK(code int, expect []int) bool { + if len(expect) == 0 { + return code >= 200 && code < 300 + } + for _, v := range expect { + if v == code { + return true + } + } + return false +} + +func respArg(arg string, resp *http.Response, body []byte) string { + arg = strings.TrimSpace(arg) + if arg == "" { + return "" + } + l := strings.ToLower(arg) + if strings.HasPrefix(l, "header.") { + h := strings.TrimSpace(arg[len("header."):]) + if h == "" { + return "" + } + return strings.TrimSpace(resp.Header.Get(h)) + } + if strings.HasPrefix(l, "body.") { + k := strings.TrimSpace(arg[len("body."):]) + if k == "" { + return "" + } + var m map[string]interface{} + if err := json.Unmarshal(body, &m); err != nil { + return "" + } + if v, ok := m[k]; ok { + if s, ok := v.(string); ok { + return strings.TrimSpace(s) + } + } + } + return "" +} + +func extractXMLTag(v, tag string) string { + s := strings.TrimSpace(v) + if s == "" { + return "" + } + lt := strings.ToLower(tag) + open := "<" + lt + ">" + clos := "" + ls := strings.ToLower(s) + i := strings.Index(ls, open) + if i < 0 { + return "" + } + i += len(open) + j := strings.Index(ls[i:], clos) + if j < 0 { + return "" + } + r := strings.TrimSpace(s[i : i+j]) + r = strings.ReplaceAll(r, """, "") + return strings.Trim(r, `"'`) +} + +func checkAPI(resp *resty.Response, result apiResult) error { + if result.Result != "" && result.Result != "ok" { + if result.Msg == "" { + result.Msg = "unknown error" + } + return fmt.Errorf("%s: %s", result.Result, result.Msg) + } + if resp != nil && resp.IsError() { + if result.Msg != "" { + return fmt.Errorf("%s", result.Msg) + } + return fmt.Errorf("http error: %d", resp.StatusCode()) + } + return nil +} + +func (d *Wps) ensureCompanyID(ctx context.Context) error { + if d.isPersonal() { + return nil + } + if d.companyID != "" { + return nil + } + var resp workspaceResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(endpoint + "/3rd/plussvr/compose/v1/users/self/workspaces?fields=name&comp_status=active") + if err != nil { + return err + } + if r != nil && r.IsError() { + return fmt.Errorf("http error: %d", r.StatusCode()) + } + if len(resp.Companies) == 0 { + return fmt.Errorf("no company id") + } + d.companyID = strconv.FormatInt(resp.Companies[0].ID, 10) + return nil +} + +func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { + if d.isPersonal() { + var resp personalGroupsResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(d.driveURL("/api/v3/groups")) + if err != nil { + return nil, err + } + if err := checkAPI(r, resp.apiResult); err != nil { + return nil, err + } + res := make([]Group, 0, len(resp.Groups)) + for _, g := range resp.Groups { + res = append(res, Group{GroupID: g.ID, Name: g.Name}) + } + return res, nil + } + if err := d.ensureCompanyID(ctx); err != nil { + return nil, err + } + var resp groupsResp + url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%s/users/self/groups/private", endpoint, d.companyID) + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return resp.Groups, nil +} + +func (d *Wps) getFiles(ctx context.Context, groupID, parentID int64) ([]FileInfo, error) { + var resp filesResp + var files []FileInfo + next_offset := 0 + for range 50 { + url := fmt.Sprintf("%s/api/v5/groups/%d/files", d.driveHost()+d.drivePrefix(), groupID) + r, err := d.request(ctx). + SetQueryParam("parentid", strconv.FormatInt(parentID, 10)). + SetQueryParam("offset", fmt.Sprint(next_offset)). + SetResult(&resp). + SetError(&resp). + Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + files = append(files, resp.Files...) + if resp.NextOffset == -1 { + break + } + next_offset = resp.NextOffset + } + return files, nil +} + +func parseTime(v int64) time.Time { + if v <= 0 { + return time.Time{} + } + return time.Unix(v, 0) +} + +func joinPath(basePath, name string) string { + if basePath == "" || basePath == "/" { + return "/" + name + } + return strings.TrimRight(basePath, "/") + "/" + name +} + +func normalizePath(path string) string { + clean := strings.TrimSpace(path) + if clean == "" || clean == "/" { + return "/" + } + return "/" + strings.Trim(clean, "/") +} + +func (d *Wps) resolveCacheStore() *resolveCacheStore { + if d == nil { + return nil + } + if v, ok := resolveCaches.Load(d); ok { + if s, ok := v.(*resolveCacheStore); ok { + return s + } + } + s := &resolveCacheStore{m: make(map[string]resolveCacheEntry)} + if v, loaded := resolveCaches.LoadOrStore(d, s); loaded { + if s2, ok := v.(*resolveCacheStore); ok { + return s2 + } + } + return s +} + +func (d *Wps) getResolveCache(path string) (*resolvedNode, bool) { + s := d.resolveCacheStore() + if s == nil { + return nil, false + } + s.mu.RLock() + e, ok := s.m[path] + s.mu.RUnlock() + if !ok || e.node == nil { + return nil, false + } + if !e.expire.IsZero() && time.Now().After(e.expire) { + s.mu.Lock() + delete(s.m, path) + s.mu.Unlock() + return nil, false + } + return e.node, true +} + +func (d *Wps) setResolveCache(path string, node *resolvedNode) { + s := d.resolveCacheStore() + if s == nil || node == nil { + return + } + s.mu.Lock() + s.m[path] = resolveCacheEntry{node: node, expire: time.Now().Add(10 * time.Minute)} + s.mu.Unlock() +} + +func (d *Wps) clearResolveCache() { + s := d.resolveCacheStore() + if s == nil { + return + } + s.mu.Lock() + if len(s.m) != 0 { + s.m = make(map[string]resolveCacheEntry) + } + s.mu.Unlock() +} + +func (d *Wps) resolvePath(ctx context.Context, path string) (*resolvedNode, error) { + cacheKey := normalizePath(path) + if n, ok := d.getResolveCache(cacheKey); ok { + return n, nil + } + clean := strings.TrimSpace(path) + if clean == "" { + clean = "/" + } + clean = strings.Trim(clean, "/") + if clean == "" { + n := &resolvedNode{kind: "root"} + d.setResolveCache("/", n) + return n, nil + } + seg := strings.Split(clean, "/") + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + var grp *Group + for i := range groups { + if groups[i].Name == seg[0] { + grp = &groups[i] + break + } + } + if grp == nil { + return nil, fmt.Errorf("group not found") + } + cur := "/" + seg[0] + gn := &resolvedNode{kind: "group", group: *grp} + d.setResolveCache(cur, gn) + if len(seg) == 1 { + return gn, nil + } + parentID := int64(0) + var lastNode *resolvedNode + for i := 1; i < len(seg); i++ { + files, err := d.getFiles(ctx, grp.GroupID, parentID) + if err != nil { + return nil, err + } + var found *FileInfo + for j := range files { + if files[j].Name == seg[i] { + found = &files[j] + break + } + } + if found == nil { + return nil, fmt.Errorf("path not found") + } + if i < len(seg)-1 && found.Type != "folder" { + return nil, fmt.Errorf("path not found") + } + fi := *found + parentID = fi.ID + cur = cur + "/" + seg[i] + kind := "file" + if fi.Type == "folder" { + kind = "folder" + } + n := &resolvedNode{kind: kind, group: *grp, file: &fi} + d.setResolveCache(cur, n) + lastNode = n + } + if lastNode == nil { + return nil, fmt.Errorf("path not found") + } + return lastNode, nil +} + +func (d *Wps) fileToObj(basePath string, f FileInfo) *Obj { + name := f.Name + path := joinPath(basePath, name) + obj := &Obj{ + id: path, + name: name, + size: f.Size, + ctime: parseTime(f.Ctime), + mtime: parseTime(f.Mtime), + isDir: f.Type == "folder", + path: path, + } + if !obj.isDir { + obj.canDownload = d.canDownload(&f) + } + return obj +} + +func (d *Wps) doJSON(ctx context.Context, method, url string, body interface{}) error { + var result apiResult + req := d.jsonRequest(ctx).SetBody(body).SetResult(&result).SetError(&result) + var ( + resp *resty.Response + err error + ) + switch method { + case http.MethodPost: + resp, err = req.Post(url) + case http.MethodPut: + resp, err = req.Put(url) + default: + return errs.NotSupport + } + if err != nil { + return err + } + return checkAPI(resp, result) +} + +func (d *Wps) list(ctx context.Context, basePath string) ([]model.Obj, error) { + if strings.TrimSpace(basePath) == "" { + basePath = "/" + } + node, err := d.resolvePath(ctx, basePath) + if err != nil { + return nil, err + } + if node.kind == "root" { + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0, len(groups)) + for _, g := range groups { + path := joinPath(basePath, g.Name) + obj := &Obj{ + id: path, + name: g.Name, + ctime: parseTime(0), + mtime: parseTime(0), + isDir: true, + path: path, + } + res = append(res, obj) + d.setResolveCache(normalizePath(path), &resolvedNode{kind: "group", group: g}) + } + d.setResolveCache("/", &resolvedNode{kind: "root"}) + return res, nil + } + if node.kind != "group" && node.kind != "folder" { + return nil, nil + } + parentID := int64(0) + if node.file != nil && node.kind == "folder" { + parentID = node.file.ID + } + files, err := d.getFiles(ctx, node.group.GroupID, parentID) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0, len(files)) + for _, f := range files { + res = append(res, d.fileToObj(basePath, f)) + path := normalizePath(joinPath(basePath, f.Name)) + fi := f + kind := "file" + if fi.Type == "folder" { + kind = "folder" + } + d.setResolveCache(path, &resolvedNode{kind: kind, group: node.group, file: &fi}) + } + return res, nil +} + +func (d *Wps) link(ctx context.Context, path string) (*model.Link, error) { + node, err := d.resolvePath(ctx, path) + if err != nil { + return nil, err + } + if node.kind != "file" || node.file == nil { + return nil, errs.NotSupport + } + if !d.canDownload(node.file) { + return nil, fmt.Errorf("no download permission") + } + url := fmt.Sprintf("%s/api/v5/groups/%d/files/%d/download?support_checksums=sha1", d.driveHost()+d.drivePrefix(), node.group.GroupID, node.file.ID) + var resp downloadResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + if resp.URL == "" { + return nil, fmt.Errorf("empty download url") + } + return &model.Link{URL: resp.URL, Header: http.Header{}}, nil +} + +func (d *Wps) makeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if parentDir == nil { + return errs.NotSupport + } + node, err := d.resolvePath(ctx, parentDir.GetPath()) + if err != nil { + return err + } + if node.kind != "group" && node.kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.file != nil && node.kind == "folder" { + parentID = node.file.ID + } + body := map[string]interface{}{ + "groupid": node.group.GroupID, + "name": dirName, + "parentid": parentID, + } + if err := d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/folder"), body); err != nil { + return err + } + d.clearResolveCache() + return nil +} + +func (d *Wps) move(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) + if err != nil { + return err + } + nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) + if err != nil { + return err + } + if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { + return errs.NotSupport + } + if nodeDst.kind != "group" && nodeDst.kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.file != nil && nodeDst.kind == "folder" { + targetParentID = nodeDst.file.ID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.file.ID}, + "target_groupid": nodeDst.group.GroupID, + "target_parentid": targetParentID, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/move", nodeSrc.group.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + d.clearResolveCache() + return nil +} + +func (d *Wps) rename(ctx context.Context, srcObj model.Obj, newName string) error { + if srcObj == nil { + return errs.NotSupport + } + node, err := d.resolvePath(ctx, srcObj.GetPath()) + if err != nil { + return err + } + if node.kind != "file" && node.kind != "folder" { + return errs.NotSupport + } + url := fmt.Sprintf("/api/v3/groups/%d/files/%d", node.group.GroupID, node.file.ID) + body := map[string]string{"fname": newName} + if err := d.doJSON(ctx, http.MethodPut, d.driveURL(url), body); err != nil { + return err + } + d.clearResolveCache() + return nil +} + +func (d *Wps) copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) + if err != nil { + return err + } + nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) + if err != nil { + return err + } + if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { + return errs.NotSupport + } + if nodeDst.kind != "group" && nodeDst.kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.file != nil && nodeDst.kind == "folder" { + targetParentID = nodeDst.file.ID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.file.ID}, + "groupid": nodeSrc.group.GroupID, + "target_groupid": nodeDst.group.GroupID, + "target_parentid": targetParentID, + "duplicated_name_model": 1, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/copy", nodeSrc.group.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + d.clearResolveCache() + return nil +} + +func (d *Wps) remove(ctx context.Context, obj model.Obj) error { + if obj == nil { + return errs.NotSupport + } + node, err := d.resolvePath(ctx, obj.GetPath()) + if err != nil { + return err + } + if node.kind != "file" && node.kind != "folder" { + return errs.NotSupport + } + + body := map[string]interface{}{ + "fileids": []int64{node.file.ID}, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/delete", node.group.GroupID) + + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + // 无法连续创建文件夹删除。如果一定要删除，每0.5s 尝试一次创建下一个删除请求，应当避免递归删除文件夹 + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + d.clearResolveCache() + return nil +} + +func cacheAndHash(file model.FileStreamer, up driver.UpdateProgress) (model.File, int64, string, string, error) { + h1 := sha1.New() + h256 := sha256.New() + size := file.GetSize() + var counted int64 + ws := []io.Writer{h1, h256} + if size <= 0 { + ws = append(ws, countingWriter{n: &counted}) + } + p := up + f, err := file.CacheFullAndWriter(&p, io.MultiWriter(ws...)) + if err != nil { + return nil, 0, "", "", err + } + if size <= 0 { + size = counted + } + return f, size, hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h256.Sum(nil)), nil +} + +func (d *Wps) createUpload(ctx context.Context, groupID, parentID int64, name string, size int64, sha1Hex, sha256Hex string) (*uploadCreateUpdateResp, error) { + body := map[string]string{ + "group_id": strconv.FormatInt(groupID, 10), + "name": name, + "parent_id": strconv.FormatInt(parentID, 10), + "sha1": sha1Hex, + "sha256": sha256Hex, + "size": strconv.FormatInt(size, 10), + } + var resp uploadCreateUpdateResp + r, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&resp). + SetError(&resp). + Put(d.driveURL("/api/v5/files/upload/create_update")) + if err != nil { + return nil, err + } + if err := checkAPI(r, resp.apiResult); err != nil { + return nil, err + } + if resp.URL == "" { + return nil, fmt.Errorf("empty upload url") + } + return &resp, nil +} + +func normalizeETag(v string) string { + v = strings.TrimSpace(v) + if strings.HasPrefix(v, "W/") { + v = strings.TrimSpace(strings.TrimPrefix(v, "W/")) + } + return strings.Trim(v, `"`) +} + +func (d *Wps) commitUpload(ctx context.Context, etag, key string, groupID, parentID int64, name, sha1Hex string, size int64, store string) error { + store = strings.TrimSpace(store) + if store == "" { + store = "ks3" + } + storeKey := "" + if key != "" { + storeKey = key + } + body := map[string]interface{}{ + "etag": etag, + "groupid": groupID, + "key": key, + "name": name, + "parentid": parentID, + "sha1": sha1Hex, + "size": size, + "store": store, + "storekey": storeKey, + } + return d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/file"), body) +} + +func (d *Wps) put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if dstDir == nil || file == nil { + return errs.NotSupport + } + if up == nil { + up = func(float64) {} + } + node, err := d.resolvePath(ctx, dstDir.GetPath()) + if err != nil { + return err + } + if node.kind != "group" && node.kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.file != nil && node.kind == "folder" { + parentID = node.file.ID + } + f, size, sha1Hex, sha256Hex, err := cacheAndHash(file, func(float64) {}) + if err != nil { + return err + } + if c, ok := f.(io.Closer); ok { + defer c.Close() + } + + // 在隐藏文件名前加_上传，这是WPS的限制，无法上传隐藏文件，也无法将任何文件重命名为隐藏文件，所有隐藏文件会被自动加上_ 上传 + // 甚至可以上传前缀是..的文件，但是单个点就是不行 + realName := file.GetName() + uploadName := realName + if strings.HasPrefix(realName, ".") { + uploadName = "_" + realName + } + + info, err := d.createUpload(ctx, node.group.GroupID, parentID, uploadName, size, sha1Hex, sha256Hex) + if err != nil { + return err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + rf := driver.NewLimitedUploadFile(ctx, f) + prog := driver.NewProgress(size, model.UpdateProgressWithRange(up, 0, 1)) + + method := strings.ToUpper(strings.TrimSpace(info.Method)) + if method == "" { + method = http.MethodPut + } + + var req *http.Request + if method == http.MethodPost && len(info.Request.FormData) > 0 { + if size == 0 { + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + return err + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + return err + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + return err + } + if err := mw.Close(); err != nil { + return err + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, bytes.NewReader(buf.Bytes())) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(buf.Len()) + req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10)) + } else { + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + req, err = http.NewRequestWithContext(ctx, method, info.URL, pr) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + go func() { + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + pw.CloseWithError(err) + return + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + pw.CloseWithError(err) + return + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + pw.CloseWithError(err) + return + } + if err := mw.Close(); err != nil { + pw.CloseWithError(err) + return + } + pw.Close() + }() + } + } else { + var body = io.TeeReader(rf, prog) + if size == 0 { + body = bytes.NewReader(nil) + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, body) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.ContentLength = size + req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) + } + + c := *base.RestyClient.GetClient() + c.Timeout = 0 + resp, err := (&c).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !statusOK(resp.StatusCode, info.Response.ExpectCode) { + io.Copy(io.Discard, resp.Body) + return fmt.Errorf("http error: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + etag := normalizeETag(respArg(info.Response.ArgsETag, resp, body)) + if etag == "" { + etag = normalizeETag(resp.Header.Get("ETag")) + } + + key := strings.TrimSpace(respArg(info.Response.ArgsKey, resp, body)) + if key == "" { + key = strings.TrimSpace(resp.Header.Get("x-obs-save-key")) + } + + var pr uploadPutResp + sha1FromServer := "" + if err := json.Unmarshal(body, &pr); err == nil { + sha1FromServer = strings.TrimSpace(pr.NewFilename) + if sha1FromServer == "" { + sha1FromServer = strings.TrimSpace(pr.Sha1) + } + if etag == "" && pr.MD5 != "" { + etag = strings.TrimSpace(pr.MD5) + } + } + + if sha1FromServer == "" { + if v := extractXMLTag(string(body), "ETag"); v != "" { + sha1FromServer = v + if etag == "" { + etag = v + } + } + } + if sha1FromServer == "" && key != "" && len(key) == 40 { + sha1FromServer = key + } + if sha1FromServer == "" { + sha1FromServer = sha1Hex + } + + if etag == "" { + return fmt.Errorf("empty etag") + } + if sha1FromServer == "" { + return fmt.Errorf("empty sha1") + } + + store := strings.TrimSpace(info.Store) + commitKey := "" + if strings.TrimSpace(info.Response.ArgsKey) != "" { + commitKey = key + if commitKey == "" { + commitKey = sha1FromServer + } + } + + if err := d.commitUpload(ctx, etag, commitKey, node.group.GroupID, parentID, uploadName, sha1FromServer, size, store); err != nil { + return err + } + + up(1) + return nil +} + +func (d *Wps) spaces(ctx context.Context) (*spacesResp, error) { + url := fmt.Sprintf("%s/api/v3/spaces", d.driveHost()+d.drivePrefix()) + var resp spacesResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return &resp, nil +} diff --git a/drivers/yandex_disk/driver.go b/drivers/yandex_disk/driver.go index b3591f380..2a7996e22 100644 --- a/drivers/yandex_disk/driver.go +++ b/drivers/yandex_disk/driver.go @@ -41,7 +41,9 @@ func (d *YandexDisk) List(ctx context.Context, dir model.Obj, args model.ListArg return nil, err } return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil + obj := fileToObj(src) + obj.Path = path.Join(dir.GetPath(), obj.Name) + return obj, nil }) } diff --git a/drivers/yandex_disk/types.go b/drivers/yandex_disk/types.go index 2481f051e..78e6203c5 100644 --- a/drivers/yandex_disk/types.go +++ b/drivers/yandex_disk/types.go @@ -42,7 +42,7 @@ type File struct { //Revision int64 `json:"revision"` } -func fileToObj(f File) model.Obj { +func fileToObj(f File) *model.Object { return &model.Object{ Name: f.Name, Size: f.Size, diff --git a/drivers/yandex_disk/util.go b/drivers/yandex_disk/util.go index e6f4cab06..2da2cb73a 100644 --- a/drivers/yandex_disk/util.go +++ b/drivers/yandex_disk/util.go @@ -23,7 +23,6 @@ func (d *YandexDisk) refreshToken() error { ErrorMessage string `json:"text"` } _, err := base.RestyClient.R(). - SetHeader("User-Agent", "Mozilla/5.0 (Macintosh; Apple macOS 15_5) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36 Chrome/138.0.0.0 Openlist/425.6.30"). SetResult(&resp). SetQueryParams(map[string]string{ "refresh_ui": d.RefreshToken, diff --git a/go.mod b/go.mod index 78416068b..c36ac1ca0 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,29 @@ module github.com/OpenListTeam/OpenList/v4 -go 1.23.4 +go 1.24.0 + +toolchain go1.24.13 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 + github.com/KarpelesLab/reflink v1.0.2 + github.com/KirCute/zip v1.0.1 github.com/OpenListTeam/go-cache v0.1.0 github.com/OpenListTeam/sftpd-openlist v1.0.1 - github.com/OpenListTeam/tache v0.2.0 + github.com/OpenListTeam/tache v0.2.2 github.com/OpenListTeam/times v0.1.0 github.com/OpenListTeam/wopan-sdk-go v0.1.5 github.com/ProtonMail/go-crypto v1.3.0 - github.com/SheltonZhu/115driver v1.1.1 + github.com/ProtonMail/gopenpgp/v2 v2.9.0 + github.com/SheltonZhu/115driver v1.2.3 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible + github.com/antchfx/htmlquery v1.3.5 + github.com/antchfx/xpath v1.3.5 github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.55.7 github.com/blevesearch/bleve/v2 v2.5.2 + github.com/bmatcuk/doublestar/v4 v4.9.1 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.21.0 github.com/charmbracelet/bubbletea v1.3.6 @@ -30,7 +38,7 @@ require ( github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 github.com/foxxorcat/mopan-sdk-go v0.1.6 - github.com/foxxorcat/weiyun-sdk-go v0.1.3 + github.com/foxxorcat/weiyun-sdk-go v0.1.4 github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 github.com/go-resty/resty/v2 v2.16.5 @@ -38,7 +46,9 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 github.com/hekmon/transmissionrpc/v3 v3.0.0 + github.com/henrybear327/go-proton-api v1.0.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/itsHenry35/gofakes3 v0.0.8 github.com/jlaffaye/ftp v0.2.1-0.20240918233326-1b970516f5d3 @@ -52,8 +62,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.9 github.com/pquerna/otp v1.5.0 + github.com/quic-go/quic-go v0.54.1 github.com/rclone/rclone v1.70.3 - github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d github.com/shirou/gopsutil/v4 v4.25.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.14.0 @@ -64,13 +74,12 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.6.0 - github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 - golang.org/x/crypto v0.40.0 + golang.org/x/crypto v0.46.0 golang.org/x/image v0.29.0 - golang.org/x/net v0.42.0 - golang.org/x/oauth2 v0.30.0 - golang.org/x/time v0.12.0 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 + golang.org/x/time v0.14.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 @@ -80,15 +89,26 @@ require ( ) require ( - cloud.google.com/go/compute/metadata v0.7.0 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect + github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect + github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect + github.com/ProtonMail/go-srp v0.0.7 // indirect + github.com/PuerkitoBio/goquery v1.10.3 // indirect github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/bradenaw/juniper v0.15.3 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cronokirby/saferith v0.33.0 // indirect github.com/ebitengine/purego v0.8.4 // indirect + github.com/emersion/go-message v0.18.2 // indirect + github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect github.com/geoffgarside/ber v1.2.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect @@ -101,13 +121,17 @@ require ( github.com/minio/minlz v1.0.0 // indirect github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/otiai10/mint v1.6.3 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/relvacode/iso8601 v1.6.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + go.uber.org/mock v0.5.0 // indirect + golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect + golang.org/x/mod v0.30.0 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect ) require ( - github.com/OpenListTeam/115-sdk-go v0.2.2 + github.com/OpenListTeam/115-sdk-go v0.2.3 github.com/STARRY-S/zip v0.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/blevesearch/go-faiss v1.0.25 // indirect @@ -137,7 +161,7 @@ require ( github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/yuin/goldmark v1.7.13 - go4.org v0.0.0-20230225012048-214862532bf5 + go4.org v0.0.0-20260112195520-a5071408f32f resty.dev/v3 v3.0.0-beta.2 // indirect ) @@ -194,6 +218,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect + github.com/henrybear327/Proton-API-Bridge v1.0.0 github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.5.0 github.com/jackc/pgpassfile v1.0.0 // indirect @@ -237,7 +262,6 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/otiai10/copy v1.14.1 github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -263,18 +287,22 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.4.0 // indirect golang.org/x/arch v0.18.0 // indirect - golang.org/x/sync v0.16.0 - golang.org/x/sys v0.34.0 - golang.org/x/term v0.33.0 // indirect - golang.org/x/text v0.27.0 - golang.org/x/tools v0.34.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.40.0 + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 + golang.org/x/tools v0.39.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) +replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0 + +replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed + // replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go diff --git a/go.sum b/go.sum index af6b00215..b9a4570bd 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbf cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -37,30 +38,52 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= +github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= +github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= +github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= +github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= +github.com/KirCute/zip v1.0.1/go.mod h1:xhF7dCB+Bjvy+5a56lenYCKBsH+gxDNPZSy5Cp+nlXk= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/OpenListTeam/115-sdk-go v0.2.2 h1:JCrGHqQjBX3laOA6Hw4CuBovSg7g+FC5s0LEAYsRciU= -github.com/OpenListTeam/115-sdk-go v0.2.2/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= +github.com/OpenListTeam/115-sdk-go v0.2.3 h1:nDNz0GxgliW+nT2Ds486k/rp/GgJj7Ngznc98ZBUwZo= +github.com/OpenListTeam/115-sdk-go v0.2.3/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= github.com/OpenListTeam/gsync v0.1.0/go.mod h1:h/Rvv9aX/6CdW/7B8di3xK3xNV8dUg45Fehrd/ksZ9s= github.com/OpenListTeam/sftpd-openlist v1.0.1 h1:j4S3iPFOpnXCUKRPS7uCT4mF2VCl34GyqvH6lqwnkUU= github.com/OpenListTeam/sftpd-openlist v1.0.1/go.mod h1:uO/wKnbvbdq3rBLmClMTZXuCnw7XW4wlAq4dZe91a40= -github.com/OpenListTeam/tache v0.2.0 h1:Q4MjuyECn0CZCf1ZF91JaVaZTaps1mOTAm8bFj8sr9Q= -github.com/OpenListTeam/tache v0.2.0/go.mod h1:qmnZ/VpY2DUlmjg3UoDeNFy/LRqrw0biN3hYEEGc/+A= +github.com/OpenListTeam/tache v0.2.2 h1:CWFn6sr1AIYaEjC8ONdKs+LrxHyuErheenAjEqRhh4k= +github.com/OpenListTeam/tache v0.2.2/go.mod h1:qmnZ/VpY2DUlmjg3UoDeNFy/LRqrw0biN3hYEEGc/+A= github.com/OpenListTeam/times v0.1.0 h1:qknxw+qj5CYKgXAwydA102UEpPcpU8TYNGRmwRyPYpg= github.com/OpenListTeam/times v0.1.0/go.mod h1:Jx7qen5NCYzKk2w14YuvU48YYMcPa1P9a+EJePC15Pc= github.com/OpenListTeam/wopan-sdk-go v0.1.5 h1:iKKcVzIqBgtGDbn0QbdWrCazSGxXFmYFyrnFBG+U8dI= github.com/OpenListTeam/wopan-sdk-go v0.1.5/go.mod h1:otynv0CgSNUClPpUgZ44qCZGcMRe0dc83Pkk65xAunI= +github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= +github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw= github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= +github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= +github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk= +github.com/ProtonMail/gopenpgp/v2 v2.9.0 h1:ruLzBmwe4dR1hdnrsEJ/S7psSBmV15gFttFUPP/+/kE= +github.com/ProtonMail/gopenpgp/v2 v2.9.0/go.mod h1:IldDyh9Hv1ZCCYatTuuEt1XZJ0OPjxLpTarDfglih7s= +github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= +github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.1.1 h1:9EMhe2ZJflGiAaZbYInw2jqxTcqZNF+DtVDsEy70aFU= -github.com/SheltonZhu/115driver v1.1.1/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= +github.com/SheltonZhu/115driver v1.2.3 h1:94XMP/ey7VXIlpoBLIJHEoXu7N8YsELZlXVbxWcDDvk= +github.com/SheltonZhu/115driver v1.2.3/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= @@ -72,6 +95,12 @@ github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAP github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 h1:8PmGpDEZl9yDpcdEr6Odf23feCxK3LNUNMxjXg41pZQ= github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/antchfx/htmlquery v1.3.5 h1:aYthDDClnG2a2xePf6tys/UyyM/kRcsFRm+ifhFKoU0= +github.com/antchfx/htmlquery v1.3.5/go.mod h1:5oyIPIa3ovYGtLqMPNjBF2Uf25NPCKsMjCnQ8lvjaoA= +github.com/antchfx/xpath v1.3.5 h1:PqbXLC3TkfeZyakF5eeh3NTWEbYl4VHNVeufANzDbKQ= +github.com/antchfx/xpath v1.3.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -156,6 +185,8 @@ github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFx github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= github.com/blevesearch/zapx/v16 v16.2.4 h1:tGgfvleXTAkwsD5mEzgM3zCS/7pgocTCnO1oyAUjlww= github.com/blevesearch/zapx/v16 v16.2.4/go.mod h1:Rti/REtuuMmzwsI8/C/qIzRaEoSK/wiFYw5e5ctUKKs= +github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= +github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4= @@ -164,6 +195,9 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo= +github.com/bradenaw/juniper v0.15.3/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0= github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -198,6 +232,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc h1:t8YjNUCt1DimB4HCIXBztwWMhgxr5yG5/YaRl9Afdfg= @@ -239,6 +274,10 @@ github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJL github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= +github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= +github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg= +github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= @@ -251,8 +290,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= -github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= -github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= +github.com/foxxorcat/weiyun-sdk-go v0.1.4 h1:X2tFvdqikkJ7awCBbMH7XXk7+uQoJlQksJz9CUU6ZgA= +github.com/foxxorcat/weiyun-sdk-go v0.1.4/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= @@ -313,6 +352,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -364,6 +405,8 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38 h1:lsK2GVgI2Ox0NkRpQnN09GBOH7jtsjFK5tcIgxXlLr0= +github.com/halalcloud/golang-sdk-lite v0.0.0-20251105081800-78cbb6786c38/go.mod h1:8x1h4rm3s8xMcTyJrq848sQ6BJnKzl57mDY4CNshdPM= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -385,6 +428,10 @@ github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg= +github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0= +github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= +github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= +github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -549,10 +596,6 @@ github.com/ncw/swift/v2 v2.0.4 h1:hHWVFxn5/YaTWAASmn4qyq2p6OyP/Hm3vMLzkjEqR7w= github.com/ncw/swift/v2 v2.0.4/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.1.1 h1:OJaYalXdliBUXPmC8CZGQ7oZDxzX1/5mQmgn0/GASew= github.com/nwaples/rardecode/v2 v2.1.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= -github.com/otiai10/copy v1.14.1 h1:5/7E6qsUMBaH5AnQ0sSLzzTg1oTECmcCmT6lvF45Na8= -github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I= -github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= -github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= @@ -585,8 +628,14 @@ github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQP github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= +github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE= github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo= +github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= +github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -599,8 +648,6 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= -github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= @@ -673,8 +720,6 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 h1:K8gF0eekWPEX+57l30ixxzGhHH/qscI3JCnuhbN6V4M= -github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9/go.mod h1:9BnoKCcgJ/+SLhfAXj15352hTOuVmG5Gzo8xNRINfqI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= @@ -702,8 +747,14 @@ go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5J go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= +go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= +go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= @@ -715,12 +766,15 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -756,6 +810,10 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -775,12 +833,16 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -788,6 +850,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -802,6 +866,8 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -821,6 +887,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -835,10 +902,13 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= @@ -846,6 +916,8 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -854,6 +926,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= @@ -861,10 +934,14 @@ golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -895,8 +972,10 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -932,6 +1011,8 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -941,10 +1022,14 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/archive/rardecode/rardecode.go b/internal/archive/rardecode/rardecode.go index 13a22e3e8..019841d3b 100644 --- a/internal/archive/rardecode/rardecode.go +++ b/internal/archive/rardecode/rardecode.go @@ -4,6 +4,7 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" @@ -21,7 +22,7 @@ func (RarDecoder) AcceptedExtensions() []string { func (RarDecoder) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{ - ".part1.rar": {".part%d.rar", 2}, + ".part1.rar": {PartFileFormat: regexp.MustCompile(`^.*\.part(\d+)\.rar$`), SecondPartIndex: 2}, } } diff --git a/internal/archive/sevenzip/sevenzip.go b/internal/archive/sevenzip/sevenzip.go index c59cf27c2..bf2cb78ad 100644 --- a/internal/archive/sevenzip/sevenzip.go +++ b/internal/archive/sevenzip/sevenzip.go @@ -2,6 +2,7 @@ package sevenzip import ( "io" + "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" @@ -18,7 +19,7 @@ func (SevenZip) AcceptedExtensions() []string { func (SevenZip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{ - ".7z.001": {".7z.%.3d", 2}, + ".7z.001": {PartFileFormat: regexp.MustCompile(`^.*\.7z\.(\d+)$`), SecondPartIndex: 2}, } } diff --git a/internal/archive/tool/base.go b/internal/archive/tool/base.go index 606094534..571ada748 100644 --- a/internal/archive/tool/base.go +++ b/internal/archive/tool/base.go @@ -2,13 +2,14 @@ package tool import ( "io" + "regexp" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" ) type MultipartExtension struct { - PartFileFormat string + PartFileFormat *regexp.Regexp SecondPartIndex int } diff --git a/internal/archive/zip/utils.go b/internal/archive/zip/utils.go index 0cd1a637a..4f367a631 100644 --- a/internal/archive/zip/utils.go +++ b/internal/archive/zip/utils.go @@ -4,22 +4,15 @@ import ( "bytes" "io" "io/fs" - stdpath "path" "strings" + "github.com/KirCute/zip" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" - "github.com/saintfish/chardet" - "github.com/yeka/zip" - "golang.org/x/text/encoding" - "golang.org/x/text/encoding/charmap" - "golang.org/x/text/encoding/japanese" - "golang.org/x/text/encoding/korean" - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/encoding/traditionalchinese" - "golang.org/x/text/encoding/unicode" - "golang.org/x/text/encoding/unicode/utf32" + "golang.org/x/text/encoding/ianaindex" "golang.org/x/text/transform" ) @@ -37,10 +30,11 @@ func (r *WrapReader) Files() []tool.SubFile { type WrapFileInfo struct { fs.FileInfo + efs bool } func (f *WrapFileInfo) Name() string { - return decodeName(f.FileInfo.Name()) + return decodeName(f.FileInfo.Name(), f.efs) } type WrapFile struct { @@ -48,11 +42,11 @@ type WrapFile struct { } func (f *WrapFile) Name() string { - return decodeName(f.f.Name) + return decodeName(f.f.Name, isEFS(f.f.Flags)) } func (f *WrapFile) FileInfo() fs.FileInfo { - return &WrapFileInfo{FileInfo: f.f.FileInfo()} + return &WrapFileInfo{FileInfo: f.f.FileInfo(), efs: isEFS(f.f.Flags)} } func (f *WrapFile) Open() (io.ReadCloser, error) { @@ -67,16 +61,33 @@ func (f *WrapFile) SetPassword(password string) { f.f.SetPassword(password) } -func getReader(ss []*stream.SeekableStream) (*zip.Reader, error) { - if len(ss) > 1 && stdpath.Ext(ss[1].GetName()) == ".z01" { - // FIXME: Incorrect parsing method for standard multipart zip format - ss = append(ss[1:], ss[0]) - } - reader, err := stream.NewMultiReaderAt(ss) +func makePart(ss *stream.SeekableStream) (zip.SizeReaderAt, error) { + ra, err := stream.NewReadAtSeeker(ss, 0) if err != nil { return nil, err } - return zip.NewReader(reader, reader.Size()) + return &inlineSizeReaderAt{ReaderAt: ra, size: ss.GetSize()}, nil +} + +func (z *Zip) getReader(ss []*stream.SeekableStream) (*zip.Reader, error) { + if len(ss) > 1 && z.traditionalSecondPartRegExp.MatchString(ss[1].GetName()) { + ss = append(ss[1:], ss[0]) + ras := make([]zip.SizeReaderAt, 0, len(ss)) + for _, s := range ss { + ra, err := makePart(s) + if err != nil { + return nil, err + } + ras = append(ras, ra) + } + return zip.NewMultipartReader(ras) + } else { + reader, err := stream.NewMultiReaderAt(ss) + if err != nil { + return nil, err + } + return zip.NewReader(reader, reader.Size()) + } } func filterPassword(err error) error { @@ -86,110 +97,29 @@ func filterPassword(err error) error { return err } -func decodeName(name string) string { - b := []byte(name) - detector := chardet.NewTextDetector() - results, err := detector.DetectAll(b) - if err != nil { +func decodeName(name string, efs bool) string { + if efs { return name } - var ce, re, enc encoding.Encoding - for _, r := range results { - if r.Confidence > 30 { - ce = getCommonEncoding(r.Charset) - if ce != nil { - break - } - } - if re == nil { - re = getEncoding(r.Charset) - } - } - if ce != nil { - enc = ce - } else if re != nil { - enc = re - } else { + enc, err := ianaindex.IANA.Encoding(setting.GetStr(conf.NonEFSZipEncoding)) + if err != nil { return name } - i := bytes.NewReader(b) + i := bytes.NewReader([]byte(name)) decoder := transform.NewReader(i, enc.NewDecoder()) content, _ := io.ReadAll(decoder) return string(content) } -func getCommonEncoding(name string) (enc encoding.Encoding) { - switch name { - case "UTF-8": - enc = unicode.UTF8 - case "UTF-16LE": - enc = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) - case "Shift_JIS": - enc = japanese.ShiftJIS - case "GB-18030": - enc = simplifiedchinese.GB18030 - case "EUC-KR": - enc = korean.EUCKR - case "Big5": - enc = traditionalchinese.Big5 - default: - enc = nil - } - return -} - -func getEncoding(name string) (enc encoding.Encoding) { - switch name { - case "UTF-8": - enc = unicode.UTF8 - case "UTF-16BE": - enc = unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM) - case "UTF-16LE": - enc = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) - case "UTF-32BE": - enc = utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM) - case "UTF-32LE": - enc = utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM) - case "ISO-8859-1": - enc = charmap.ISO8859_1 - case "ISO-8859-2": - enc = charmap.ISO8859_2 - case "ISO-8859-3": - enc = charmap.ISO8859_3 - case "ISO-8859-4": - enc = charmap.ISO8859_4 - case "ISO-8859-5": - enc = charmap.ISO8859_5 - case "ISO-8859-6": - enc = charmap.ISO8859_6 - case "ISO-8859-7": - enc = charmap.ISO8859_7 - case "ISO-8859-8": - enc = charmap.ISO8859_8 - case "ISO-8859-8-I": - enc = charmap.ISO8859_8I - case "ISO-8859-9": - enc = charmap.ISO8859_9 - case "windows-1251": - enc = charmap.Windows1251 - case "windows-1256": - enc = charmap.Windows1256 - case "KOI8-R": - enc = charmap.KOI8R - case "Shift_JIS": - enc = japanese.ShiftJIS - case "GB-18030": - enc = simplifiedchinese.GB18030 - case "EUC-JP": - enc = japanese.EUCJP - case "EUC-KR": - enc = korean.EUCKR - case "Big5": - enc = traditionalchinese.Big5 - case "ISO-2022-JP": - enc = japanese.ISO2022JP - default: - enc = nil - } - return +func isEFS(flags uint16) bool { + return (flags & 0x800) > 0 +} + +type inlineSizeReaderAt struct { + io.ReaderAt + size int64 +} + +func (i *inlineSizeReaderAt) Size() int64 { + return i.size } diff --git a/internal/archive/zip/zip.go b/internal/archive/zip/zip.go index 8ed83ac72..1dcb904d7 100644 --- a/internal/archive/zip/zip.go +++ b/internal/archive/zip/zip.go @@ -3,6 +3,7 @@ package zip import ( "io" stdpath "path" + "regexp" "strings" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" @@ -12,34 +13,39 @@ import ( ) type Zip struct { + traditionalSecondPartRegExp *regexp.Regexp } -func (Zip) AcceptedExtensions() []string { +func (z *Zip) AcceptedExtensions() []string { return []string{} } -func (Zip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { +func (z *Zip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { return map[string]tool.MultipartExtension{ - ".zip": {".z%.2d", 1}, - ".zip.001": {".zip.%.3d", 2}, + ".zip": {PartFileFormat: regexp.MustCompile(`^.*\.z(\d+)$`), SecondPartIndex: 1}, + ".zip.001": {PartFileFormat: regexp.MustCompile(`^.*\.zip\.(\d+)$`), SecondPartIndex: 2}, } } -func (Zip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { - zipReader, err := getReader(ss) +func (z *Zip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + zipReader, err := z.getReader(ss) if err != nil { return nil, err } + efs := true + if len(zipReader.File) > 0 { + efs = isEFS(zipReader.File[0].Flags) + } encrypted, tree := tool.GenerateMetaTreeFromFolderTraversal(&WrapReader{Reader: zipReader}) return &model.ArchiveMetaInfo{ - Comment: zipReader.Comment, + Comment: decodeName(zipReader.Comment, efs), Encrypted: encrypted, Tree: tree, }, nil } -func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { - zipReader, err := getReader(ss) +func (z *Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + zipReader, err := z.getReader(ss) if err != nil { return nil, err } @@ -57,7 +63,7 @@ func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]mod _ = rc.Close() passVerified = true } - name := strings.TrimSuffix(decodeName(file.Name), "/") + name := strings.TrimSuffix(decodeName(file.Name, isEFS(file.Flags)), "/") if strings.Contains(name, "/") { // 有些压缩包不压缩第一个文件夹 strs := strings.Split(name, "/") @@ -70,7 +76,7 @@ func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]mod } continue } - ret = append(ret, tool.MakeModelObj(&WrapFileInfo{FileInfo: file.FileInfo()})) + ret = append(ret, tool.MakeModelObj(&WrapFileInfo{FileInfo: file.FileInfo(), efs: isEFS(file.Flags)})) } if len(ret) == 0 && dir != nil { ret = append(ret, dir) @@ -81,13 +87,13 @@ func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]mod ret := make([]model.Obj, 0) exist := false for _, file := range zipReader.File { - name := decodeName(file.Name) + name := decodeName(file.Name, isEFS(file.Flags)) dir := stdpath.Dir(strings.TrimSuffix(name, "/")) + "/" if dir != innerPath { continue } exist = true - ret = append(ret, tool.MakeModelObj(&WrapFileInfo{file.FileInfo()})) + ret = append(ret, tool.MakeModelObj(&WrapFileInfo{file.FileInfo(), isEFS(file.Flags)})) } if !exist { return nil, errs.ObjectNotFound @@ -96,14 +102,14 @@ func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]mod } } -func (Zip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { - zipReader, err := getReader(ss) +func (z *Zip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + zipReader, err := z.getReader(ss) if err != nil { return nil, 0, err } innerPath := strings.TrimPrefix(args.InnerPath, "/") for _, file := range zipReader.File { - if decodeName(file.Name) == innerPath { + if decodeName(file.Name, isEFS(file.Flags)) == innerPath { if file.IsEncrypted() { file.SetPassword(args.Password) } @@ -117,8 +123,8 @@ func (Zip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io return nil, 0, errs.ObjectNotFound } -func (Zip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { - zipReader, err := getReader(ss) +func (z *Zip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + zipReader, err := z.getReader(ss) if err != nil { return err } @@ -128,5 +134,7 @@ func (Zip) Decompress(ss []*stream.SeekableStream, outputPath string, args model var _ tool.Tool = (*Zip)(nil) func init() { - tool.RegisterTool(Zip{}) + tool.RegisterTool(&Zip{ + traditionalSecondPartRegExp: regexp.MustCompile(`^.*\.z0*1$`), + }) } diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 2209c64f3..74e218f8f 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -39,7 +39,22 @@ func InitConfig() { if !filepath.IsAbs(dataDir) { flags.DataDir = filepath.Join(pwd, flags.DataDir) } - configPath := filepath.Join(flags.DataDir, "config.json") + // Determine config file path: use flags.ConfigPath if provided, otherwise default to /config.json + configPath := flags.ConfigPath + if configPath == "" { + configPath = filepath.Join(flags.DataDir, "config.json") + } else { + // if relative, resolve relative to working directory + if !filepath.IsAbs(configPath) { + if absPath, err := filepath.Abs(configPath); err == nil { + configPath = absPath + } else { + configPath = filepath.Join(pwd, configPath) + } + } + } + configPath = filepath.Clean(configPath) + conf.ConfigPath = configPath log.Infof("reading config file: %s", configPath) if !utils.Exists(configPath) { log.Infof("config file not exists, creating default config file") @@ -126,6 +141,10 @@ func InitConfig() { log.Fatalf("create temp dir error: %+v", err) } log.Debugf("config: %+v", conf.Conf) + + // Validate and display proxy configuration status + validateProxyConfig() + base.InitClient() initURL() } @@ -165,3 +184,14 @@ func CleanTempDir() { } } } + +// validateProxyConfig validates proxy configuration and displays status at startup +func validateProxyConfig() { + if conf.Conf.ProxyAddress != "" { + if _, err := url.Parse(conf.Conf.ProxyAddress); err == nil { + log.Infof("Proxy enabled: %s", conf.Conf.ProxyAddress) + } else { + log.Errorf("Invalid proxy address format: %s, error: %v", conf.Conf.ProxyAddress, err) + } + } +} diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 08024695c..7bff851de 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -148,9 +148,13 @@ func InitialSettings() []model.SettingItem { {Key: "audio_cover", Value: "https://res.oplist.org/logo/logo.svg", MigrationValue: "https://cdn.oplist.org/gh/OpenListTeam/Logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW}, {Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.PreviewDownloadByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.SharePreviewDownloadByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.SharePreviewArchivesByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.NonEFSZipEncoding, Value: "IBM437", Type: conf.TypeString, Group: model.PREVIEW}, // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, {Key: "package_download", Value: "true", Type: conf.TypeBool, Group: model.GLOBAL}, @@ -173,6 +177,9 @@ func InitialSettings() []model.SettingItem { {Key: conf.ShareArchivePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, {Key: conf.ShareForceProxy, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.ShareSummaryContent, Value: "@{{creator}} shared {{#each files}}{{#if @first}}\"{{filename this}}\"{{/if}}{{#if @last}}{{#unless (eq @index 0)}} and {{@index}} more files{{/unless}}{{/if}}{{/each}} from {{site_title}}: {{base_url}}/@s/{{id}}{{#if pwd}} , the share code is {{pwd}}{{/if}}{{#if expires}}, please access before {{dateLocaleString expires}}.{{/if}}", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PUBLIC}, + {Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, @@ -201,6 +208,7 @@ func InitialSettings() []model.SettingItem { // ldap settings {Key: conf.LdapLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PUBLIC}, {Key: conf.LdapServer, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapSkipTlsVerify, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapManagerDN, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapManagerPassword, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapUserSearchBase, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, diff --git a/internal/bootstrap/patch.go b/internal/bootstrap/patch.go index 1e76190c8..e8baeeac9 100644 --- a/internal/bootstrap/patch.go +++ b/internal/bootstrap/patch.go @@ -2,7 +2,6 @@ package bootstrap import ( "fmt" - "strings" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch" diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go index 44ba17376..5d4c814dc 100644 --- a/internal/bootstrap/patch/all.go +++ b/internal/bootstrap/patch/all.go @@ -4,7 +4,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_24_0" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_32_0" "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_41_0" - "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v3_all" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v4_1_8" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/patch/v4_1_9" ) type VersionPatches struct { @@ -34,9 +35,16 @@ var UpgradePatches = []VersionPatches{ }, }, { - Version: "v3.0.0", + Version: "v4.1.8", Patches: []func(){ - v3_all.RenameAlistV3Driver, + v4_1_8.FixAliasConfig, + }, + }, + { + Version: "v4.1.9", + Patches: []func(){ + v4_1_9.EnableWebDavProxy, + v4_1_9.ResetSkipTlsVerify, }, }, } diff --git a/internal/bootstrap/patch/v3_24_0/hash_password.go b/internal/bootstrap/patch/v3_24_0/hash_password.go index 881e3d47b..30a296c47 100644 --- a/internal/bootstrap/patch/v3_24_0/hash_password.go +++ b/internal/bootstrap/patch/v3_24_0/hash_password.go @@ -11,7 +11,8 @@ import ( func HashPwdForOldVersion() { users, _, err := op.GetUsers(1, -1) if err != nil { - utils.Log.Fatalf("[hash pwd for old version] failed get users: %v", err) + utils.Log.Errorf("[hash pwd for old version] failed get users: %v", err) + return } for i := range users { user := users[i] @@ -19,7 +20,7 @@ func HashPwdForOldVersion() { user.SetPassword(user.Password) user.Password = "" if err := db.UpdateUser(&user); err != nil { - utils.Log.Fatalf("[hash pwd for old version] failed update user: %v", err) + utils.Log.Errorf("[hash pwd for old version] failed update user: %v", err) } } } diff --git a/internal/bootstrap/patch/v3_32_0/update_authn.go b/internal/bootstrap/patch/v3_32_0/update_authn.go index fea4bbfb1..721c3f582 100644 --- a/internal/bootstrap/patch/v3_32_0/update_authn.go +++ b/internal/bootstrap/patch/v3_32_0/update_authn.go @@ -11,14 +11,15 @@ import ( func UpdateAuthnForOldVersion() { users, _, err := op.GetUsers(1, -1) if err != nil { - utils.Log.Fatalf("[update authn for old version] failed get users: %v", err) + utils.Log.Errorf("[update authn for old version] failed get users: %v", err) + return } for i := range users { user := users[i] if user.Authn == "" { user.Authn = "[]" if err := db.UpdateUser(&user); err != nil { - utils.Log.Fatalf("[update authn for old version] failed update user: %v", err) + utils.Log.Errorf("[update authn for old version] failed update user: %v", err) } } } diff --git a/internal/bootstrap/patch/v3_all/rename.go b/internal/bootstrap/patch/v3_all/rename.go deleted file mode 100644 index 323b7391f..000000000 --- a/internal/bootstrap/patch/v3_all/rename.go +++ /dev/null @@ -1,33 +0,0 @@ -package v3_all - -import ( - "github.com/OpenListTeam/OpenList/v4/internal/db" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" -) - -// Rename Alist V3 driver to OpenList -func RenameAlistV3Driver() { - storages, _, err := db.GetStorages(1, -1) - if err != nil { - utils.Log.Errorf("[RenameAlistV3Driver] failed to get storages: %s", err.Error()) - return - } - - updatedCount := 0 - for _, s := range storages { - if s.Driver == "AList V3" { - utils.Log.Warnf("[RenameAlistV3Driver] rename storage [%d]%s from Alist V3 to OpenList", s.ID, s.MountPath) - s.Driver = "OpenList" - err = db.UpdateStorage(&s) - if err != nil { - utils.Log.Errorf("[RenameAlistV3Driver] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) - } else { - updatedCount++ - } - } - } - - if updatedCount > 0 { - utils.Log.Infof("[RenameAlistV3Driver] updated %d storages from Alist V3 to OpenList", updatedCount) - } -} diff --git a/internal/bootstrap/patch/v4_1_8/alias.go b/internal/bootstrap/patch/v4_1_8/alias.go new file mode 100644 index 000000000..e41742654 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_8/alias.go @@ -0,0 +1,81 @@ +package v4_1_8 + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// FixAliasConfig upgrade the old version of the Addition of the Alias driver +func FixAliasConfig() { + storages, _, err := db.GetStorages(1, -1) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to get storages: %s", err.Error()) + return + } + for _, s := range storages { + if s.Driver != "Alias" { + continue + } + addition := make(map[string]any) + err = utils.Json.UnmarshalFromString(s.Addition, &addition) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to unmarshal addition of [%d]%s: %s", s.ID, s.MountPath, err.Error()) + continue + } + if _, ok := addition["read_conflict_policy"]; ok { + utils.Log.Infof("[FixAliasConfig] skip fixing [%d]%s because the addition already has \"read_conflict_policy\" key", s.ID, s.MountPath) + continue + } + var protectSameName, parallelWrite, writable bool + protectSameNameAny, ok := addition["protect_same_name"] + if ok { + delete(addition, "protect_same_name") + protectSameName, ok = protectSameNameAny.(bool) + } + if !ok { + protectSameName = false + } + parallelWriteAny, ok := addition["parallel_write"] + if ok { + delete(addition, "parallel_write") + parallelWrite, ok = parallelWriteAny.(bool) + } + if !ok { + parallelWrite = false + } + writableAny, ok := addition["writable"] + if ok { + delete(addition, "writable") + writable, ok = writableAny.(bool) + } + if !ok { + writable = false + } + if !writable { + addition["write_conflict_policy"] = "disabled" + addition["put_conflict_policy"] = "disabled" + } else if !protectSameName && !parallelWrite { + addition["write_conflict_policy"] = "first" + addition["put_conflict_policy"] = "first" + } else if protectSameName && !parallelWrite { + addition["write_conflict_policy"] = "deterministic" + addition["put_conflict_policy"] = "deterministic" + } else if !protectSameName && parallelWrite { + addition["write_conflict_policy"] = "all" + addition["put_conflict_policy"] = "all" + } else { + addition["write_conflict_policy"] = "deterministic_or_all" + addition["put_conflict_policy"] = "deterministic_or_all" + } + addition["read_conflict_policy"] = "first" + s.Addition, err = utils.Json.MarshalToString(addition) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to marshal addition of [%d]%s: %s", s.ID, s.MountPath, err.Error()) + continue + } + err = db.UpdateStorage(&s) + if err != nil { + utils.Log.Errorf("[FixAliasConfig] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) + } + } +} diff --git a/internal/bootstrap/patch/v4_1_9/skip_tls.go b/internal/bootstrap/patch/v4_1_9/skip_tls.go new file mode 100644 index 000000000..1d9858d95 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_9/skip_tls.go @@ -0,0 +1,32 @@ +package v4_1_9 + +import ( + "os" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +func ResetSkipTlsVerify() { + if !conf.Conf.TlsInsecureSkipVerify { + return + } + if !strings.HasPrefix(conf.Version, "v") { + return + } + + conf.Conf.TlsInsecureSkipVerify = false + + confBody, err := utils.Json.MarshalIndent(conf.Conf, "", " ") + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: marshal config error: %+v", err) + return + } + err = os.WriteFile(conf.ConfigPath, confBody, 0o777) + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: update config struct error: %+v", err) + return + } + utils.Log.Infof("[ResetSkipTlsVerify] succeeded to set tls_insecure_skip_verify to false") +} diff --git a/internal/bootstrap/patch/v4_1_9/webdav.go b/internal/bootstrap/patch/v4_1_9/webdav.go new file mode 100644 index 000000000..2fd97c783 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_9/webdav.go @@ -0,0 +1,30 @@ +package v4_1_9 + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +// EnableWebDavProxy updates Webdav driver storages to enable proxy +func EnableWebDavProxy() { + storages, _, err := db.GetStorages(1, -1) + if err != nil { + utils.Log.Errorf("[EnableWebDavProxy] failed to get storages: %s", err.Error()) + return + } + for _, s := range storages { + if s.Driver != "WebDav" { + continue + } + if !s.WebProxy { + s.WebProxy = true + } + if s.WebdavPolicy == "302_redirect" { + s.WebdavPolicy = "native_proxy" + } + err = db.UpdateStorage(&s) + if err != nil { + utils.Log.Errorf("[EnableWebDavProxy] failed to update storage [%d]%s: %s", s.ID, s.MountPath, err.Error()) + } + } +} diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go new file mode 100644 index 000000000..6740dba65 --- /dev/null +++ b/internal/bootstrap/run.go @@ -0,0 +1,404 @@ +package bootstrap + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "strconv" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/bootstrap/data" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/db" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server" + "github.com/OpenListTeam/OpenList/v4/server/middlewares" + "github.com/OpenListTeam/sftpd-openlist" + ftpserver "github.com/fclairamb/ftpserverlib" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/quic-go/quic-go/http3" + log "github.com/sirupsen/logrus" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" +) + +func Init() { + InitConfig() + Log() + InitDB() + data.InitData() + InitStreamLimit() + InitIndex() + InitUpgradePatch() +} + +func Release() { + db.Close() +} + +var ( + running bool + httpSrv *http.Server + httpRunning bool + httpsSrv *http.Server + httpsRunning bool + unixSrv *http.Server + unixRunning bool + quicSrv *http3.Server + quicRunning bool + s3Srv *http.Server + s3Running bool + ftpDriver *server.FtpMainDriver + ftpServer *ftpserver.FtpServer + ftpRunning bool + sftpDriver *server.SftpDriver + sftpServer *sftpd.SftpServer + sftpRunning bool +) + +// Called by OpenList-Mobile +func IsRunning(t string) bool { + switch t { + case "http": + return httpRunning + case "https": + return httpsRunning + case "unix": + return unixRunning + case "quic": + return quicRunning + case "s3": + return s3Running + case "sftp": + return sftpRunning + case "ftp": + return ftpRunning + } + return running +} + +func Start() { + if conf.Conf.DelayedStart != 0 { + utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart) + time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second) + } + InitOfflineDownloadTools() + LoadStorages() + InitTaskManager() + if !flags.Debug && !flags.Dev { + gin.SetMode(gin.ReleaseMode) + } + r := gin.New() + + // gin log + if conf.Conf.Log.Filter.Enable { + r.Use(middlewares.FilteredLogger()) + } else { + r.Use(gin.LoggerWithWriter(log.StandardLogger().Out)) + } + r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out)) + + server.Init(r) + var httpHandler http.Handler = r + if conf.Conf.Scheme.EnableH2c { + httpHandler = h2c.NewHandler(r, &http2.Server{}) + } + if conf.Conf.Scheme.HttpPort != -1 { + httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort) + fmt.Printf("start HTTP server @ %s\n", httpBase) + utils.Log.Infof("start HTTP server @ %s", httpBase) + httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler} + go func() { + httpRunning = true + err := httpSrv.ListenAndServe() + httpRunning = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("http", err) + utils.Log.Errorf("failed to start http: %s", err.Error()) + } else { + handleEndpointShutdownHooks("http") + } + }() + } + if conf.Conf.Scheme.HttpsPort != -1 { + httpsBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpsPort) + fmt.Printf("start HTTPS server @ %s\n", httpsBase) + utils.Log.Infof("start HTTPS server @ %s", httpsBase) + httpsSrv = &http.Server{Addr: httpsBase, Handler: r} + go func() { + httpsRunning = true + err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) + httpsRunning = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("https", err) + utils.Log.Errorf("failed to start https: %s", err.Error()) + } else { + handleEndpointShutdownHooks("https") + } + }() + if conf.Conf.Scheme.EnableH3 { + fmt.Printf("start HTTP3 (quic) server @ %s\n", httpsBase) + utils.Log.Infof("start HTTP3 (quic) server @ %s", httpsBase) + r.Use(func(c *gin.Context) { + if c.Request.TLS != nil { + port := conf.Conf.Scheme.HttpsPort + c.Header("Alt-Svc", fmt.Sprintf("h3=\":%d\"; ma=86400", port)) + } + c.Next() + }) + quicSrv = &http3.Server{Addr: httpsBase, Handler: r} + go func() { + quicRunning = true + err := quicSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) + quicRunning = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("quic", err) + utils.Log.Errorf("failed to start http3 (quic): %s", err.Error()) + } else { + handleEndpointShutdownHooks("quic") + } + }() + } + } + if conf.Conf.Scheme.UnixFile != "" { + fmt.Printf("start unix server @ %s\n", conf.Conf.Scheme.UnixFile) + utils.Log.Infof("start unix server @ %s", conf.Conf.Scheme.UnixFile) + unixSrv = &http.Server{Handler: httpHandler} + go func() { + listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile) + if err != nil { + utils.Log.Errorf("failed to listen unix: %+v", err) + return + } + unixRunning = true + // set socket file permission + mode, err := strconv.ParseUint(conf.Conf.Scheme.UnixFilePerm, 8, 32) + if err != nil { + utils.Log.Errorf("failed to parse socket file permission: %+v", err) + } else { + err = os.Chmod(conf.Conf.Scheme.UnixFile, os.FileMode(mode)) + if err != nil { + utils.Log.Errorf("failed to chmod socket file: %+v", err) + } + } + err = unixSrv.Serve(listener) + unixRunning = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("unix", err) + utils.Log.Errorf("failed to start unix: %s", err.Error()) + } else { + handleEndpointShutdownHooks("unix") + } + }() + } + if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { + s3r := gin.New() + s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) + server.InitS3(s3r) + s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) + fmt.Printf("start S3 server @ %s\n", s3Base) + utils.Log.Infof("start S3 server @ %s", s3Base) + go func() { + s3Running = true + var err error + if conf.Conf.S3.SSL { + s3Srv = &http.Server{Addr: s3Base, Handler: s3r} + err = s3Srv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) + } else { + s3Srv = &http.Server{Addr: s3Base, Handler: s3r} + err = s3Srv.ListenAndServe() + } + s3Running = false + if err != nil && !errors.Is(err, http.ErrServerClosed) { + handleEndpointStartFailedHooks("s3", err) + utils.Log.Errorf("failed to start s3 server: %s", err.Error()) + } else { + handleEndpointShutdownHooks("s3") + } + }() + } + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable { + var err error + ftpDriver, err = server.NewMainDriver() + if err != nil { + utils.Log.Errorf("failed to start ftp driver: %s", err.Error()) + } else { + fmt.Printf("start ftp server on %s\n", conf.Conf.FTP.Listen) + utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen) + go func() { + ftpServer = ftpserver.NewFtpServer(ftpDriver) + ftpRunning = true + err = ftpServer.ListenAndServe() + ftpRunning = false + if err != nil { + handleEndpointStartFailedHooks("ftp", err) + utils.Log.Errorf("problem ftp server listening: %s", err.Error()) + } else { + handleEndpointShutdownHooks("ftp") + } + }() + } + } + if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable { + var err error + sftpDriver, err = server.NewSftpDriver() + if err != nil { + utils.Log.Errorf("failed to start sftp driver: %s", err.Error()) + } else { + fmt.Printf("start sftp server on %s", conf.Conf.SFTP.Listen) + utils.Log.Infof("start sftp server on %s", conf.Conf.SFTP.Listen) + go func() { + sftpServer = sftpd.NewSftpServer(sftpDriver) + sftpRunning = true + err = sftpServer.RunServer() + sftpRunning = false + if err != nil { + handleEndpointStartFailedHooks("sftp", err) + utils.Log.Errorf("problem sftp server listening: %s", err.Error()) + } else { + handleEndpointShutdownHooks("sftp") + } + }() + } + } + running = true +} + +func Shutdown(timeout time.Duration) { + utils.Log.Println("Shutdown server...") + fs.ArchiveContentUploadTaskManager.RemoveAll() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + var wg sync.WaitGroup + if httpSrv != nil && conf.Conf.Scheme.HttpPort != -1 { + wg.Add(1) + go func() { + defer wg.Done() + if err := httpSrv.Shutdown(ctx); err != nil { + utils.Log.Error("HTTP server shutdown err: ", err) + } + httpSrv = nil + }() + } + if httpsSrv != nil && conf.Conf.Scheme.HttpsPort != -1 { + wg.Add(1) + go func() { + defer wg.Done() + if err := httpsSrv.Shutdown(ctx); err != nil { + utils.Log.Error("HTTPS server shutdown err: ", err) + } + httpsSrv = nil + }() + if quicSrv != nil && conf.Conf.Scheme.EnableH3 { + wg.Add(1) + go func() { + defer wg.Done() + if err := quicSrv.Shutdown(ctx); err != nil { + utils.Log.Error("HTTP3 (quic) server shutdown err: ", err) + } + quicSrv = nil + }() + } + } + if unixSrv != nil && conf.Conf.Scheme.UnixFile != "" { + wg.Add(1) + go func() { + defer wg.Done() + if err := unixSrv.Shutdown(ctx); err != nil { + utils.Log.Error("Unix server shutdown err: ", err) + } + unixSrv = nil + }() + } + if s3Srv != nil && conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { + wg.Add(1) + go func() { + defer wg.Done() + if err := s3Srv.Shutdown(ctx); err != nil { + utils.Log.Error("S3 server shutdown err: ", err) + } + s3Srv = nil + }() + } + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil { + wg.Add(1) + go func() { + defer wg.Done() + if ftpDriver != nil { + ftpDriver.Stop() + ftpDriver = nil + } + if err := ftpServer.Stop(); err != nil { + utils.Log.Error("FTP server shutdown err: ", err) + } + ftpServer = nil + }() + } + if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable && sftpServer != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := sftpServer.Close(); err != nil { + utils.Log.Error("SFTP server shutdown err: ", err) + } + sftpServer = nil + sftpDriver = nil + }() + } + wg.Wait() + utils.Log.Println("Server exit") + running = false +} + +type EndpointStartFailedHook func(string, string) + +type EndpointShutdownHook func(string) + +var ( + endpointStartFailedHooks map[string]EndpointStartFailedHook + endpointShutdownHooks map[string]EndpointShutdownHook +) + +func RegisterEndpointStartFailedHook(hook EndpointStartFailedHook) string { + id := uuid.NewString() + endpointStartFailedHooks[id] = hook + return id +} + +func RemoveEndpointStartFailedHook(id string) { + delete(endpointStartFailedHooks, id) +} + +func RegisterEndpointShutdownHook(hook EndpointShutdownHook) string { + id := uuid.NewString() + endpointShutdownHooks[id] = hook + return id +} + +func RemoveEndpointShutdownHook(id string) { + delete(endpointShutdownHooks, id) +} + +func handleEndpointStartFailedHooks(t string, err error) { + for _, hook := range endpointStartFailedHooks { + hook(t, err.Error()) + } +} + +func handleEndpointShutdownHooks(t string) { + for _, hook := range endpointShutdownHooks { + hook(t) + } +} + +func init() { + endpointShutdownHooks = make(map[string]EndpointShutdownHook) + endpointStartFailedHooks = make(map[string]EndpointStartFailedHook) +} diff --git a/internal/bootstrap/storage.go b/internal/bootstrap/storage.go index d111c7d37..1389095bf 100644 --- a/internal/bootstrap/storage.go +++ b/internal/bootstrap/storage.go @@ -25,6 +25,6 @@ func LoadStorages() { storages[i].MountPath, storages[i].Driver, storages[i].Order) } } - conf.StoragesLoaded = true + conf.SendStoragesLoadedSignal() }(storages) } diff --git a/internal/cache/keyed_cache.go b/internal/cache/keyed_cache.go new file mode 100644 index 000000000..07bf8cd9f --- /dev/null +++ b/internal/cache/keyed_cache.go @@ -0,0 +1,101 @@ +package cache + +import ( + "sync" + "time" +) + +type KeyedCache[T any] struct { + entries map[string]*CacheEntry[T] + mu sync.RWMutex + ttl time.Duration +} + +func NewKeyedCache[T any](ttl time.Duration) *KeyedCache[T] { + c := &KeyedCache[T]{ + entries: make(map[string]*CacheEntry[T]), + ttl: ttl, + } + gcFuncs = append(gcFuncs, c.GC) + return c +} + +func (c *KeyedCache[T]) Set(key string, value T) { + c.SetWithExpirable(key, value, ExpirationTime(time.Now().Add(c.ttl))) +} + +func (c *KeyedCache[T]) SetWithTTL(key string, value T, ttl time.Duration) { + c.SetWithExpirable(key, value, ExpirationTime(time.Now().Add(ttl))) +} + +func (c *KeyedCache[T]) SetWithExpirable(key string, value T, exp Expirable) { + c.mu.Lock() + defer c.mu.Unlock() + + c.entries[key] = &CacheEntry[T]{ + data: value, + Expirable: exp, + } +} + +func (c *KeyedCache[T]) Get(key string) (T, bool) { + c.mu.RLock() + entry, exists := c.entries[key] + if !exists { + c.mu.RUnlock() + return *new(T), false + } + + expired := entry.Expired() + c.mu.RUnlock() + + if !expired { + return entry.data, true + } + + c.mu.Lock() + if c.entries[key] == entry { + delete(c.entries, key) + c.mu.Unlock() + return *new(T), false + } + c.mu.Unlock() + return *new(T), false +} + +func (c *KeyedCache[T]) Delete(key string) { + c.mu.Lock() + defer c.mu.Unlock() + + delete(c.entries, key) +} + +func (c *KeyedCache[T]) Pop(key string) (T, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if entry, exists := c.entries[key]; exists { + delete(c.entries, key) + return entry.data, true + } + return *new(T), false +} + +func (c *KeyedCache[T]) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + c.entries = make(map[string]*CacheEntry[T]) +} + +func (c *KeyedCache[T]) GC() { + c.mu.Lock() + defer c.mu.Unlock() + expiredKeys := make([]string, 0, len(c.entries)) + for key, entry := range c.entries { + if entry.Expired() { + expiredKeys = append(expiredKeys, key) + } + } + for _, key := range expiredKeys { + delete(c.entries, key) + } +} diff --git a/internal/cache/type.go b/internal/cache/type.go new file mode 100644 index 000000000..4eccba52a --- /dev/null +++ b/internal/cache/type.go @@ -0,0 +1,18 @@ +package cache + +import "time" + +type Expirable interface { + Expired() bool +} + +type ExpirationTime time.Time + +func (e ExpirationTime) Expired() bool { + return time.Now().After(time.Time(e)) +} + +type CacheEntry[T any] struct { + Expirable + data T +} diff --git a/internal/cache/typed_cache.go b/internal/cache/typed_cache.go new file mode 100644 index 000000000..7ba126be8 --- /dev/null +++ b/internal/cache/typed_cache.go @@ -0,0 +1,113 @@ +package cache + +import ( + "sync" + "time" +) + +type TypedCache[T any] struct { + entries map[string]map[string]*CacheEntry[T] + mu sync.RWMutex + ttl time.Duration +} + +func NewTypedCache[T any](ttl time.Duration) *TypedCache[T] { + c := &TypedCache[T]{ + entries: make(map[string]map[string]*CacheEntry[T]), + ttl: ttl, + } + gcFuncs = append(gcFuncs, c.GC) + return c +} + +func (c *TypedCache[T]) SetType(key, typeKey string, value T) { + c.SetTypeWithExpirable(key, typeKey, value, ExpirationTime(time.Now().Add(c.ttl))) +} + +func (c *TypedCache[T]) SetTypeWithTTL(key, typeKey string, value T, ttl time.Duration) { + c.SetTypeWithExpirable(key, typeKey, value, ExpirationTime(time.Now().Add(ttl))) +} + +func (c *TypedCache[T]) SetTypeWithExpirable(key, typeKey string, value T, exp Expirable) { + c.mu.Lock() + defer c.mu.Unlock() + cache, exists := c.entries[key] + if !exists { + cache = make(map[string]*CacheEntry[T]) + c.entries[key] = cache + } + + cache[typeKey] = &CacheEntry[T]{ + data: value, + Expirable: exp, + } +} + +func (c *TypedCache[T]) GetType(key, typeKey string) (T, bool) { + c.mu.RLock() + cache, exists := c.entries[key] + if !exists { + c.mu.RUnlock() + return *new(T), false + } + entry, exists := cache[typeKey] + if !exists { + c.mu.RUnlock() + return *new(T), false + } + expired := entry.Expired() + c.mu.RUnlock() + + if !expired { + return entry.data, true + } + + c.mu.Lock() + if cache[typeKey] == entry { + delete(cache, typeKey) + if len(cache) == 0 { + delete(c.entries, key) + } + c.mu.Unlock() + return *new(T), false + } + c.mu.Unlock() + return *new(T), false +} + +func (c *TypedCache[T]) DeleteKey(key string) { + c.mu.Lock() + defer c.mu.Unlock() + delete(c.entries, key) +} + +func (c *TypedCache[T]) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + c.entries = make(map[string]map[string]*CacheEntry[T]) +} + +func (c *TypedCache[T]) GC() { + c.mu.Lock() + defer c.mu.Unlock() + expiredKeys := make(map[string][]string) + for tk, entries := range c.entries { + for key, entry := range entries { + if !entry.Expired() { + continue + } + if _, ok := expiredKeys[tk]; !ok { + expiredKeys[tk] = make([]string, 0, len(entries)) + } + expiredKeys[tk] = append(expiredKeys[tk], key) + } + } + for tk, keys := range expiredKeys { + for _, key := range keys { + delete(c.entries[tk], key) + } + if len(c.entries[tk]) == 0 { + delete(c.entries, tk) + } + } +} diff --git a/internal/cache/utils.go b/internal/cache/utils.go new file mode 100644 index 000000000..82b7cf857 --- /dev/null +++ b/internal/cache/utils.go @@ -0,0 +1,24 @@ +package cache + +import ( + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/cron" + log "github.com/sirupsen/logrus" +) + +var ( + cacheGcCron *cron.Cron + gcFuncs []func() +) + +func init() { + // TODO Move to bootstrap + cacheGcCron = cron.NewCron(time.Hour) + cacheGcCron.Do(func() { + log.Infof("Start cache GC") + for _, f := range gcFuncs { + f() + } + }) +} diff --git a/internal/conf/config.go b/internal/conf/config.go index af198e916..f347380d8 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -35,6 +35,7 @@ type Scheme struct { UnixFile string `json:"unix_file" env:"UNIX_FILE"` UnixFilePerm string `json:"unix_file_perm" env:"UNIX_FILE_PERM"` EnableH2c bool `json:"enable_h2c" env:"ENABLE_H2C"` + EnableH3 bool `json:"enable_h3" env:"ENABLE_H3"` } type LogConfig struct { @@ -130,6 +131,7 @@ type Config struct { FTP FTP `json:"ftp" envPrefix:"FTP_"` SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` LastLaunchedVersion string `json:"last_launched_version"` + ProxyAddress string `json:"proxy_address" env:"PROXY_ADDRESS"` } func DefaultConfig(dataDir string) *Config { @@ -180,7 +182,7 @@ func DefaultConfig(dataDir string) *Config { MmapThreshold: 4, MaxConnections: 0, MaxConcurrency: 64, - TlsInsecureSkipVerify: true, + TlsInsecureSkipVerify: false, Tasks: TasksConfig{ Download: TaskConfig{ Workers: 5, @@ -243,5 +245,6 @@ func DefaultConfig(dataDir string) *Config { Listen: ":5222", }, LastLaunchedVersion: "", + ProxyAddress: "", } } diff --git a/internal/conf/const.go b/internal/conf/const.go index 5543f6083..b99d8849c 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -24,17 +24,21 @@ const ( HideStorageDetailsInManagePage = "hide_storage_details_in_manage_page" // preview - TextTypes = "text_types" - AudioTypes = "audio_types" - VideoTypes = "video_types" - ImageTypes = "image_types" - ProxyTypes = "proxy_types" - ProxyIgnoreHeaders = "proxy_ignore_headers" - AudioAutoplay = "audio_autoplay" - VideoAutoplay = "video_autoplay" - PreviewArchivesByDefault = "preview_archives_by_default" - ReadMeAutoRender = "readme_autorender" - FilterReadMeScripts = "filter_readme_scripts" + TextTypes = "text_types" + AudioTypes = "audio_types" + VideoTypes = "video_types" + ImageTypes = "image_types" + ProxyTypes = "proxy_types" + ProxyIgnoreHeaders = "proxy_ignore_headers" + AudioAutoplay = "audio_autoplay" + VideoAutoplay = "video_autoplay" + PreviewDownloadByDefault = "preview_download_by_default" + PreviewArchivesByDefault = "preview_archives_by_default" + SharePreviewDownloadByDefault = "share_preview_download_by_default" + SharePreviewArchivesByDefault = "share_preview_archives_by_default" + ReadMeAutoRender = "readme_autorender" + FilterReadMeScripts = "filter_readme_scripts" + NonEFSZipEncoding = "non_efs_zip_encoding" // global HideFiles = "hide_files" @@ -52,6 +56,9 @@ const ( ShareArchivePreview = "share_archive_preview" ShareForceProxy = "share_force_proxy" ShareSummaryContent = "share_summary_content" + HandleHookAfterWriting = "handle_hook_after_writing" + HandleHookRateLimit = "handle_hook_rate_limit" + IgnoreSystemFiles = "ignore_system_files" // index SearchIndex = "search_index" @@ -70,6 +77,9 @@ const ( // 115 Pan115TempDir = "115_temp_dir" + // 123 + Pan123TempDir = "123_temp_dir" + // 115_open Pan115OpenTempDir = "115_open_temp_dir" @@ -108,6 +118,7 @@ const ( // ldap LdapLoginEnabled = "ldap_login_enabled" LdapServer = "ldap_server" + LdapSkipTlsVerify = "ldap_skip_tls_verify" LdapManagerDN = "ldap_manager_dn" LdapManagerPassword = "ldap_manager_password" LdapUserSearchBase = "ldap_user_search_base" @@ -125,6 +136,10 @@ const ( QbittorrentUrl = "qbittorrent_url" QbittorrentSeedtime = "qbittorrent_seedtime" + // 123 open offline download + Pan123OpenOfflineDownloadCallbackUrl = "123_open_callback_url" + Pan123OpenTempDir = "123_open_temp_dir" + // ftp FTPPublicHost = "ftp_public_host" FTPPasvPortMap = "ftp_pasv_port_map" @@ -159,7 +174,7 @@ const ( ) // ContextKey is the type of context keys. -type ContextKey int +type ContextKey int8 const ( _ ContextKey = iota @@ -175,4 +190,5 @@ const ( UserAgentKey PathKey SharingIDKey + SkipHookKey ) diff --git a/internal/conf/var.go b/internal/conf/var.go index de23b5c60..972f69997 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -3,6 +3,7 @@ package conf import ( "net/url" "regexp" + "sync" ) var ( @@ -14,8 +15,9 @@ var ( ) var ( - Conf *Config - URL *url.URL + Conf *Config + URL *url.URL + ConfigPath string ) var SlicesMap = make(map[string][]string) @@ -23,8 +25,6 @@ var FilenameCharMap = make(map[string]string) var PrivacyReg []*regexp.Regexp var ( - // StoragesLoaded loaded success if empty - StoragesLoaded = false // 单个Buffer最大限制 MaxBufferLimit = 16 * 1024 * 1024 // 超过该阈值的Buffer将使用 mmap 分配，可主动释放内存 @@ -35,3 +35,39 @@ var ( ManageHtml string IndexHtml string ) + +var ( + // StoragesLoaded loaded success if empty + StoragesLoaded = false + storagesLoadMu sync.RWMutex + storagesLoadSignal chan struct{} = make(chan struct{}) +) + +func StoragesLoadSignal() <-chan struct{} { + storagesLoadMu.RLock() + ch := storagesLoadSignal + storagesLoadMu.RUnlock() + return ch +} +func SendStoragesLoadedSignal() { + storagesLoadMu.Lock() + select { + case <-storagesLoadSignal: + // already closed + default: + StoragesLoaded = true + close(storagesLoadSignal) + } + storagesLoadMu.Unlock() +} +func ResetStoragesLoadSignal() { + storagesLoadMu.Lock() + select { + case <-storagesLoadSignal: + StoragesLoaded = false + storagesLoadSignal = make(chan struct{}) + default: + // not closed -> nothing to do + } + storagesLoadMu.Unlock() +} diff --git a/internal/db/sharing.go b/internal/db/sharing.go index 3748796b0..8670b15f3 100644 --- a/internal/db/sharing.go +++ b/internal/db/sharing.go @@ -38,18 +38,26 @@ func GetSharingsByCreatorId(creator uint, pageIndex, pageSize int) (sharings []m } func CreateSharing(s *model.SharingDB) (string, error) { - id := random.String(8) - for len(id) < 12 { - old := model.SharingDB{ - ID: id, + if s.ID == "" { + id := random.String(8) + for len(id) < 12 { + old := model.SharingDB{ + ID: id, + } + if err := db.Where(old).First(&old).Error; err != nil { + s.ID = id + return id, errors.WithStack(db.Create(s).Error) + } + id += random.String(1) } - if err := db.Where(old).First(&old).Error; err != nil { - s.ID = id - return id, errors.WithStack(db.Create(s).Error) + return "", errors.New("failed find valid id") + } else { + query := model.SharingDB{ID: s.ID} + if err := db.Where(query).First(&query).Error; err == nil { + return "", errors.New("sharing already exist") } - id += random.String(1) + return s.ID, errors.WithStack(db.Create(s).Error) } - return "", errors.New("failed find valid id") } func UpdateSharing(s *model.SharingDB) error { @@ -60,3 +68,7 @@ func DeleteSharingById(id string) error { s := model.SharingDB{ID: id} return errors.WithStack(db.Where(s).Delete(&s).Error) } + +func DeleteSharingsByCreatorId(creatorId uint) error { + return errors.WithStack(db.Where("creator_id = ?", creatorId).Delete(&model.SharingDB{}).Error) +} diff --git a/internal/db/tasks.go b/internal/db/tasks.go index dcb9dfeab..36054898b 100644 --- a/internal/db/tasks.go +++ b/internal/db/tasks.go @@ -1,6 +1,7 @@ package db import ( + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/pkg/errors" ) @@ -30,6 +31,7 @@ func GetTaskDataFunc(type_s string, enabled bool) func() ([]byte, error) { return nil } return func() ([]byte, error) { + <-conf.StoragesLoadSignal() return []byte(task.PersistData), nil } } diff --git a/internal/driver/config.go b/internal/driver/config.go index bec6d47bf..2063975ba 100644 --- a/internal/driver/config.go +++ b/internal/driver/config.go @@ -3,11 +3,9 @@ package driver type Config struct { Name string `json:"name"` LocalSort bool `json:"local_sort"` - // if the driver returns Link with MFile, this should be set to true - OnlyLinkMFile bool `json:"only_local"` - OnlyProxy bool `json:"only_proxy"` - NoCache bool `json:"no_cache"` - NoUpload bool `json:"no_upload"` + OnlyProxy bool `json:"only_proxy"` + NoCache bool `json:"no_cache"` + NoUpload bool `json:"no_upload"` // if need get message from user, such as validate code NeedMs bool `json:"need_ms"` DefaultRoot string `json:"default_root"` @@ -19,8 +17,32 @@ type Config struct { ProxyRangeOption bool `json:"-"` // if the driver returns Link without URL, this should be set to true NoLinkURL bool `json:"-"` + // Link cache behaviour: + // - LinkCacheAuto: let driver decide per-path (implement driver.LinkCacheModeResolver) + // - LinkCacheNone: no extra info added to cache key (default) + // - flags (OR-able) can add more attributes to cache key (IP, UA, ...) + LinkCacheMode `json:"-"` + // if the driver only store indices of files (e.g. UrlTree) + OnlyIndices bool `json:"only_indices"` + // prefer proxy download even if direct link is available + PreferProxy bool `json:"prefer_proxy"` } +type LinkCacheMode int8 + +const ( + LinkCacheAuto LinkCacheMode = -1 // Let the driver decide per-path (use driver.LinkCacheModeResolver) + LinkCacheNone LinkCacheMode = 0 // No extra info added to cache key (default) +) + +const ( + LinkCacheIP LinkCacheMode = 1 << iota // include client IP in cache key + LinkCacheUA // include User-Agent in cache key +) func (c Config) MustProxy() bool { - return c.OnlyProxy || c.OnlyLinkMFile || c.NoLinkURL + return c.OnlyProxy || c.NoLinkURL +} + +func (c Config) DefaultProxy() bool { + return c.PreferProxy } diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 4a8e88255..373bb5653 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -47,11 +47,6 @@ type Getter interface { Get(ctx context.Context, path string) (model.Obj, error) } -type GetObjInfo interface { - // GetObjInfo get file info by path - GetObjInfo(ctx context.Context, path string) (model.Obj, error) -} - //type Writer interface { // Mkdir // Move @@ -115,15 +110,6 @@ type PutURL interface { PutURL(ctx context.Context, dstDir model.Obj, name, url string) error } -//type WriteResult interface { -// MkdirResult -// MoveResult -// RenameResult -// CopyResult -// PutResult -// Remove -//} - type MkdirResult interface { MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) } @@ -218,3 +204,17 @@ type WithDetails interface { type Reference interface { InitReference(storage Driver) error } + +type LinkCacheModeResolver interface { + // ResolveLinkCacheMode returns the LinkCacheMode for the given path. + ResolveLinkCacheMode(path string) LinkCacheMode +} + +type DirectUploader interface { + // GetDirectUploadTools returns available frontend-direct upload tools + GetDirectUploadTools() []string + // GetDirectUploadInfo returns the information needed for direct upload from client to storage + // actualPath is the path relative to the storage root (after removing mount path prefix) + // return errs.NotImplement if the driver does not support the given direct upload tool + GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error) +} diff --git a/internal/driver/utils.go b/internal/driver/utils.go index 9e4b6b015..41d9c6ce3 100644 --- a/internal/driver/utils.go +++ b/internal/driver/utils.go @@ -20,7 +20,7 @@ func (p *Progress) Write(b []byte) (n int, err error) { n = len(b) p.Done += int64(n) p.up(float64(p.Done) / float64(p.Total) * 100) - return + return n, err } func NewProgress(total int64, up UpdateProgress) *Progress { diff --git a/internal/errs/errors.go b/internal/errs/errors.go index fa599124b..fdf7f2189 100644 --- a/internal/errs/errors.go +++ b/internal/errs/errors.go @@ -12,13 +12,12 @@ var ( NotSupport = errors.New("not support") RelativePath = errors.New("using relative path is not allowed") - MoveBetweenTwoStorages = errors.New("can't move files between two storages, try to copy") - UploadNotSupported = errors.New("upload not supported") - - MetaNotFound = errors.New("meta not found") - StorageNotFound = errors.New("storage not found") - StreamIncomplete = errors.New("upload/download stream incomplete, possible network issue") - StreamPeekFail = errors.New("StreamPeekFail") + UploadNotSupported = errors.New("upload not supported") + MetaNotFound = errors.New("meta not found") + StorageNotFound = errors.New("storage not found") + StorageNotInit = errors.New("storage not init") + StreamIncomplete = errors.New("upload/download stream incomplete, possible network issue") + StreamPeekFail = errors.New("StreamPeekFail") UnknownArchiveFormat = errors.New("unknown archive format") WrongArchivePassword = errors.New("wrong archive password") diff --git a/internal/errs/object.go b/internal/errs/object.go index 00e8232ff..ab78f314a 100644 --- a/internal/errs/object.go +++ b/internal/errs/object.go @@ -7,9 +7,11 @@ import ( ) var ( - ObjectNotFound = errors.New("object not found") - NotFolder = errors.New("not a folder") - NotFile = errors.New("not a file") + ObjectNotFound = errors.New("object not found") + ObjectAlreadyExists = errors.New("object already exists") + NotFolder = errors.New("not a folder") + NotFile = errors.New("not a file") + IgnoredSystemFile = errors.New("system file upload ignored") ) func IsObjectNotFound(err error) bool { diff --git a/internal/errs/unwrap.go b/internal/errs/unwrap.go index 1cec96519..f34953c76 100644 --- a/internal/errs/unwrap.go +++ b/internal/errs/unwrap.go @@ -1,12 +1,11 @@ package errs -import "errors" - func UnwrapOrSelf(err error) error { - // errors.Unwrap has no fallback mechanism - unwrapped := errors.Unwrap(err) - if unwrapped == nil { + u, ok := err.(interface { + Unwrap() error + }) + if !ok { return err } - return unwrapped + return u.Unwrap() } diff --git a/internal/fs/archive.go b/internal/fs/archive.go index e1e4c4486..784ba587a 100644 --- a/internal/fs/archive.go +++ b/internal/fs/archive.go @@ -38,8 +38,17 @@ func (t *ArchiveDownloadTask) GetName() string { } func (t *ArchiveDownloadTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err + if t.SrcStorage == nil { + if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil { + t.SrcStorage = srcStorage + } else { + return err + } + if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil { + t.DstStorage = dstStorage + } else { + return err + } } t.ClearEndTime() t.SetStartTime(time.Now()) @@ -113,6 +122,7 @@ func (t *ArchiveDownloadTask) RunWithoutPushUploadTask() (*ArchiveContentUploadT DstActualPath: t.DstActualPath, dstStorage: t.DstStorage, DstStorageMp: t.DstStorageMp, + overwrite: t.Overwrite, } return uploadTask, nil } @@ -130,6 +140,7 @@ type ArchiveContentUploadTask struct { DstStorageMp string finalized bool groupID string + overwrite bool } func (t *ArchiveContentUploadTask) GetName() string { @@ -141,24 +152,22 @@ func (t *ArchiveContentUploadTask) GetStatus() string { } func (t *ArchiveContentUploadTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err - } t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() return t.RunWithNextTaskCallback(func(nextTsk *ArchiveContentUploadTask) error { + task_group.TransferCoordinator.AddTask(t.groupID, nil) ArchiveContentUploadTaskManager.Add(nextTsk) return nil }) } func (t *ArchiveContentUploadTask) OnSucceeded() { - task_group.TransferCoordinator.Done(t.groupID, true) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, true) } func (t *ArchiveContentUploadTask) OnFailed() { - task_group.TransferCoordinator.Done(t.groupID, false) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, false) } func (t *ArchiveContentUploadTask) SetRetry(retry int, maxRetry int) { @@ -190,8 +199,8 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch if err != nil { return err } - if !t.InPlace && len(t.groupID) > 0 { - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToRefresh(nextDstActualPath)) + if !t.InPlace { + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(nextDstActualPath)) } var es error for _, entry := range entries { @@ -205,9 +214,6 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch es = stderrors.Join(es, err) continue } - if len(t.groupID) > 0 { - task_group.TransferCoordinator.AddTask(t.groupID, nil) - } err = f(&ArchiveContentUploadTask{ TaskExtension: task.TaskExtension{ Creator: t.Creator, @@ -220,6 +226,7 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch dstStorage: t.dstStorage, DstStorageMp: t.DstStorageMp, groupID: t.groupID, + overwrite: t.overwrite, }) if err != nil { es = stderrors.Join(es, err) @@ -229,6 +236,12 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch return es } } else { + if !t.overwrite { + dstPath := stdpath.Join(t.DstActualPath, t.ObjName) + if res, _ := op.Get(t.Ctx(), t.dstStorage, dstPath); res != nil { + return errs.ObjectAlreadyExists + } + } file, err := os.Open(t.FilePath) if err != nil { return err @@ -246,7 +259,7 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTask *Arch } fs.Closers.Add(file) t.status = "uploading" - err = op.Put(t.Ctx(), t.dstStorage, t.DstActualPath, fs, t.SetProgress, true) + err = op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.dstStorage, t.DstActualPath, fs, t.SetProgress) if err != nil { return err } @@ -384,14 +397,22 @@ func archiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args } defer uploadTask.deleteSrcFile() var callback func(t *ArchiveContentUploadTask) error + var hasSuccess bool callback = func(t *ArchiveContentUploadTask) error { t.Base.SetCtx(ctx) e := t.RunWithNextTaskCallback(callback) + if e == nil { + hasSuccess = true + } t.deleteSrcFile() return e } uploadTask.Base.SetCtx(ctx) - return nil, uploadTask.RunWithNextTaskCallback(callback) + uploadTask.groupID = stdpath.Join(uploadTask.DstStorageMp, uploadTask.DstActualPath) + task_group.TransferCoordinator.AddTask(uploadTask.groupID, nil) + err = uploadTask.RunWithNextTaskCallback(callback) + task_group.TransferCoordinator.Done(context.WithoutCancel(ctx), uploadTask.groupID, hasSuccess) + return nil, err } else { tsk.Creator, _ = ctx.Value(conf.UserKey).(*model.User) tsk.ApiUrl = common.GetApiUrl(ctx) diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index dbdd28358..e78fc9be8 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -22,16 +22,22 @@ import ( type taskType uint8 func (t taskType) String() string { - if t == 0 { + switch t { + case copy: return "copy" - } else { + case move: return "move" + case merge: + return "merge" + default: + return "unknown" } } const ( copy taskType = iota move + merge ) type FileTransferTask struct { @@ -45,16 +51,25 @@ func (t *FileTransferTask) GetName() string { } func (t *FileTransferTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err + if t.SrcStorage == nil { + if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil { + t.SrcStorage = srcStorage + } else { + return err + } + if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil { + t.DstStorage = dstStorage + } else { + return err + } } + t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() return t.RunWithNextTaskCallback(func(nextTask *FileTransferTask) error { - nextTask.groupID = t.groupID task_group.TransferCoordinator.AddTask(t.groupID, nil) - if t.TaskType == copy { + if t.TaskType == copy || t.TaskType == merge { CopyTaskManager.Add(nextTask) } else { MoveTaskManager.Add(nextTask) @@ -64,15 +79,15 @@ func (t *FileTransferTask) Run() error { } func (t *FileTransferTask) OnSucceeded() { - task_group.TransferCoordinator.Done(t.groupID, true) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, true) } func (t *FileTransferTask) OnFailed() { - task_group.TransferCoordinator.Done(t.groupID, false) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, false) } func (t *FileTransferTask) SetRetry(retry int, maxRetry int) { - t.TaskExtension.SetRetry(retry, maxRetry) + t.TaskData.SetRetry(retry, maxRetry) if retry == 0 && (len(t.groupID) == 0 || // 重启恢复 (t.GetErr() == nil && t.GetState() != tache.StatePending)) { // 手动重试 @@ -85,7 +100,7 @@ func (t *FileTransferTask) SetRetry(retry int, maxRetry int) { } } -func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { +func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { return nil, errors.WithMessage(err, "failed get src storage") @@ -96,13 +111,16 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str } if srcStorage.GetStorage() == dstStorage.GetStorage() { - if taskType == copy { - err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) + if utils.IsBool(skipHook...) { + ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) + } + if taskType == copy || taskType == merge { + err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath) if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { return nil, err } } else { - err = op.Move(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) + err = op.Move(ctx, srcStorage, srcObjActualPath, dstDirActualPath) if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { return nil, err } @@ -122,6 +140,8 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str TaskType: taskType, } + t.groupID = stdpath.Join(t.DstStorageMp, t.DstActualPath) + task_group.TransferCoordinator.AddTask(t.groupID, nil) if ctx.Value(conf.NoTaskKey) != nil { var callback func(nextTask *FileTransferTask) error hasSuccess := false @@ -135,24 +155,22 @@ func transfer(ctx context.Context, taskType taskType, srcObjPath, dstDirPath str } t.Base.SetCtx(ctx) err = t.RunWithNextTaskCallback(callback) - if hasSuccess || err == nil { - if taskType == move { - task_group.RefreshAndRemove(dstDirPath, task_group.SrcPathToRemove(srcObjPath)) - } else { - op.DeleteCache(t.DstStorage, dstDirActualPath) - } + if err == nil { + hasSuccess = true } + if taskType == move { + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.SrcPathToRemove(srcObjPath)) + } + task_group.TransferCoordinator.Done(context.WithoutCancel(ctx), t.groupID, hasSuccess) return nil, err } t.Creator, _ = ctx.Value(conf.UserKey).(*model.User) t.ApiUrl = common.GetApiUrl(ctx) - t.groupID = dstDirPath - if taskType == copy { - task_group.TransferCoordinator.AddTask(dstDirPath, nil) + if taskType == copy || taskType == merge { CopyTaskManager.Add(t) } else { - task_group.TransferCoordinator.AddTask(dstDirPath, task_group.SrcPathToRemove(srcObjPath)) + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.SrcPathToRemove(srcObjPath)) MoveTaskManager.Add(t) } return t, nil @@ -164,6 +182,7 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", t.SrcActualPath) } + if srcObj.IsDir() { t.Status = "src object is dir, listing objs" objs, err := op.List(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.ListArgs{}) @@ -171,17 +190,36 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcActualPath) } dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) - if t.TaskType == copy { - if t.Ctx().Value(conf.NoTaskKey) != nil { - defer op.DeleteCache(t.DstStorage, dstActualPath) - } else { - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToRefresh(dstActualPath)) + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) + + existedObjs := make(map[string]bool) + if t.TaskType == merge { + dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) + if err != nil && !errors.Is(err, errs.ObjectNotFound) { + // 目标文件夹不存在的情况不是错误，会在之后新建文件夹 + // 这种情况显然不需要统计existedObjs，dstObjs保持为nil，下面这个for将不会执行 + return errors.WithMessagef(err, "failed list dst [%s] objs", dstActualPath) + } + for _, obj := range dstObjs { + if err := t.Ctx().Err(); err != nil { + return err + } + if !obj.IsDir() { + existedObjs[obj.GetName()] = true + } } } + for _, obj := range objs { - if utils.IsCanceled(t.Ctx()) { - return nil + if err := t.Ctx().Err(); err != nil { + return err + } + + if t.TaskType == merge && !obj.IsDir() && existedObjs[obj.GetName()] { + // skip existed file + continue } + err = f(&FileTransferTask{ TaskType: t.TaskType, TaskData: TaskData{ @@ -196,6 +234,7 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer SrcStorageMp: t.SrcStorageMp, DstStorageMp: t.DstStorageMp, }, + groupID: t.groupID, }) if err != nil { return err @@ -205,7 +244,8 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return nil } - link, _, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) + t.Status = "getting src object link" + link, srcObj, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) if err != nil { return errors.WithMessagef(err, "failed get [%s] link", t.SrcActualPath) } @@ -220,7 +260,7 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer } t.SetTotalBytes(ss.GetSize()) t.Status = "uploading" - return op.Put(t.Ctx(), t.DstStorage, t.DstActualPath, ss, t.SetProgress, true) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } var ( diff --git a/internal/fs/fs.go b/internal/fs/fs.go index ca199ed40..3d459c6be 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -60,32 +60,40 @@ func Link(ctx context.Context, path string, args model.LinkArgs) (*model.Link, m return res, file, nil } -func MakeDir(ctx context.Context, path string, lazyCache ...bool) error { - err := makeDir(ctx, path, lazyCache...) +func MakeDir(ctx context.Context, path string) error { + err := makeDir(ctx, path) if err != nil { log.Errorf("failed make dir %s: %+v", path, err) } return err } -func Move(ctx context.Context, srcPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { - req, err := transfer(ctx, move, srcPath, dstDirPath, lazyCache...) +func Move(ctx context.Context, srcPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { + req, err := transfer(ctx, move, srcPath, dstDirPath, skipHook...) if err != nil { log.Errorf("failed move %s to %s: %+v", srcPath, dstDirPath, err) } return req, err } -func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { - res, err := transfer(ctx, copy, srcObjPath, dstDirPath, lazyCache...) +func Copy(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { + res, err := transfer(ctx, copy, srcObjPath, dstDirPath, skipHook...) if err != nil { log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) } return res, err } -func Rename(ctx context.Context, srcPath, dstName string, lazyCache ...bool) error { - err := rename(ctx, srcPath, dstName, lazyCache...) +func Merge(ctx context.Context, srcObjPath, dstDirPath string, skipHook ...bool) (task.TaskExtensionInfo, error) { + res, err := transfer(ctx, merge, srcObjPath, dstDirPath, skipHook...) + if err != nil { + log.Errorf("failed merge %s to %s: %+v", srcObjPath, dstDirPath, err) + } + return res, err +} + +func Rename(ctx context.Context, srcPath, dstName string, skipHook ...bool) error { + err := rename(ctx, srcPath, dstName, skipHook...) if err != nil { log.Errorf("failed rename %s to %s: %+v", srcPath, dstName, err) } @@ -100,8 +108,8 @@ func Remove(ctx context.Context, path string) error { return err } -func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, lazyCache ...bool) error { - err := putDirectly(ctx, dstDirPath, file, lazyCache...) +func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, skipHook ...bool) error { + err := putDirectly(ctx, dstDirPath, file, skipHook...) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) } @@ -190,3 +198,11 @@ func PutURL(ctx context.Context, path, dstName, urlStr string) error { } return op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) } + +func GetDirectUploadInfo(ctx context.Context, tool, path, dstName string, fileSize int64) (any, error) { + info, err := getDirectUploadInfo(ctx, tool, path, dstName, fileSize) + if err != nil { + log.Errorf("failed get %s direct upload info for %s(%d bytes): %+v", path, dstName, fileSize, err) + } + return info, err +} diff --git a/internal/fs/get.go b/internal/fs/get.go index 4e91c5bde..8a920065e 100644 --- a/internal/fs/get.go +++ b/internal/fs/get.go @@ -3,7 +3,6 @@ package fs import ( "context" stdpath "path" - "time" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" @@ -15,9 +14,10 @@ func get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) { path = utils.FixAndCleanPath(path) // maybe a virtual file if path != "/" { - virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, stdpath.Dir(path), !args.WithStorageDetails) + dir, name := stdpath.Split(path) + virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, dir, !args.WithStorageDetails, false, name) for _, f := range virtualFiles { - if f.GetName() == stdpath.Base(path) { + if f.GetName() == name { return f, nil } } @@ -28,9 +28,8 @@ func get(ctx context.Context, path string, args *GetArgs) (model.Obj, error) { if path == "/" { return &model.Object{ Name: "root", - Size: 0, - Modified: time.Time{}, IsFolder: true, + Mask: model.ReadOnly | model.Virtual, }, nil } return nil, errors.WithMessage(err, "failed get storage") diff --git a/internal/fs/list.go b/internal/fs/list.go index 5245795fc..1f92c7d46 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -15,7 +15,7 @@ import ( func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) { meta, _ := ctx.Value(conf.MetaKey).(*model.Meta) user, _ := ctx.Value(conf.UserKey).(*model.User) - virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, path, !args.WithStorageDetails) + virtualFiles := op.GetStorageVirtualFilesWithDetailsByPath(ctx, path, !args.WithStorageDetails, args.Refresh, "") storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil && len(virtualFiles) == 0 { return nil, errors.WithMessage(err, "failed get storage") diff --git a/internal/fs/other.go b/internal/fs/other.go index 8d16b0003..a23beb73b 100644 --- a/internal/fs/other.go +++ b/internal/fs/other.go @@ -3,27 +3,32 @@ package fs import ( "context" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/task" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" ) -func makeDir(ctx context.Context, path string, lazyCache ...bool) error { +func makeDir(ctx context.Context, path string) error { storage, actualPath, err := op.GetStorageAndActualPath(path) if err != nil { return errors.WithMessage(err, "failed get storage") } - return op.MakeDir(ctx, storage, actualPath, lazyCache...) + return op.MakeDir(ctx, storage, actualPath) } -func rename(ctx context.Context, srcPath, dstName string, lazyCache ...bool) error { +func rename(ctx context.Context, srcPath, dstName string, skipHook ...bool) error { storage, srcActualPath, err := op.GetStorageAndActualPath(srcPath) if err != nil { return errors.WithMessage(err, "failed get storage") } - return op.Rename(ctx, storage, srcActualPath, dstName, lazyCache...) + if utils.IsBool(skipHook...) { + ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) + } + return op.Rename(ctx, storage, srcActualPath, dstName) } func remove(ctx context.Context, path string) error { diff --git a/internal/fs/put.go b/internal/fs/put.go index 881330b04..be829ae47 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -6,6 +6,7 @@ import ( stdpath "path" "time" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -38,15 +39,15 @@ func (t *UploadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() - return op.Put(t.Ctx(), t.storage, t.dstDirActualPath, t.file, t.SetProgress, true) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.storage, t.dstDirActualPath, t.file, t.SetProgress) } func (t *UploadTask) OnSucceeded() { - task_group.TransferCoordinator.Done(stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), true) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), true) } func (t *UploadTask) OnFailed() { - task_group.TransferCoordinator.Done(stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), false) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), stdpath.Join(t.storage.GetStorage().MountPath, t.dstDirActualPath), false) } func (t *UploadTask) SetRetry(retry int, maxRetry int) { @@ -87,13 +88,13 @@ func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) file: file, } t.SetTotalBytes(file.GetSize()) - task_group.TransferCoordinator.AddTask(dstDirPath, nil) + task_group.TransferCoordinator.AddTask(stdpath.Join(storage.GetStorage().MountPath, dstDirActualPath), nil) UploadTaskManager.Add(t) return t, nil } // putDirect put the file and return after finish -func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, lazyCache ...bool) error { +func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer, skipHook ...bool) error { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { _ = file.Close() @@ -103,5 +104,16 @@ func putDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer _ = file.Close() return errors.WithStack(errs.UploadNotSupported) } - return op.Put(ctx, storage, dstDirActualPath, file, nil, lazyCache...) + if utils.IsBool(skipHook...) { + ctx = context.WithValue(ctx, conf.SkipHookKey, struct{}{}) + } + return op.Put(ctx, storage, dstDirActualPath, file, nil) +} + +func getDirectUploadInfo(ctx context.Context, tool, dstDirPath, dstName string, fileSize int64) (any, error) { + storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return nil, errors.WithMessage(err, "failed get storage") + } + return op.GetDirectUploadInfo(ctx, tool, storage, dstDirActualPath, dstName, fileSize) } diff --git a/internal/model/args.go b/internal/model/args.go index 2ec95f149..073c94a63 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -15,6 +15,7 @@ type ListArgs struct { S3ShowPlaceholder bool Refresh bool WithStorageDetails bool + SkipHook bool } type LinkArgs struct { @@ -28,16 +29,17 @@ type Link struct { URL string `json:"url"` // most common way Header http.Header `json:"header"` // needed header (for url) RangeReader RangeReaderIF `json:"-"` // recommended way if can't use URL - MFile File `json:"-"` // best for local,smb... file system, which exposes MFile Expiration *time.Duration // local cache expire Duration //for accelerating request, use multi-thread downloading Concurrency int `json:"concurrency"` PartSize int `json:"part_size"` - ContentLength int64 `json:"-"` // 转码视频、缩略图 + ContentLength int64 `json:"content_length"` // 转码视频、缩略图 utils.SyncClosers `json:"-"` + // 如果SyncClosers中的资源被关闭后Link将不可用，则此值应为 true + RequireReference bool `json:"-"` } type OtherArgs struct { @@ -76,6 +78,7 @@ type ArchiveDecompressArgs struct { ArchiveInnerArgs CacheFull bool PutIntoNewDir bool + Overwrite bool } type SharingListArgs struct { diff --git a/internal/model/direct_upload.go b/internal/model/direct_upload.go new file mode 100644 index 000000000..89bbfeb5d --- /dev/null +++ b/internal/model/direct_upload.go @@ -0,0 +1,8 @@ +package model + +type HttpDirectUploadInfo struct { + UploadURL string `json:"upload_url"` // The URL to upload the file + ChunkSize int64 `json:"chunk_size"` // The chunk size for uploading, 0 means no chunking required + Headers map[string]string `json:"headers,omitempty"` // Optional headers to include in the upload request + Method string `json:"method,omitempty"` // HTTP method, default is PUT +} diff --git a/internal/model/file.go b/internal/model/file.go index 9fd20adc0..4ca7201e1 100644 --- a/internal/model/file.go +++ b/internal/model/file.go @@ -27,6 +27,9 @@ func (f *FileCloser) Close() error { return errors.Join(errs...) } +// FileRangeReader 是对 RangeReaderIF 的轻量包装，表明由 RangeReaderIF.RangeRead +// 返回的 io.ReadCloser 同时实现了 model.File（即支持 Read/ReadAt/Seek）。 +// 只有满足这些才需要使用 FileRangeReader，否则直接使用 RangeReaderIF 即可。 type FileRangeReader struct { RangeReaderIF } diff --git a/internal/model/obj.go b/internal/model/obj.go index b3bf5ebe5..ba7467f74 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -48,7 +48,6 @@ type FileStreamer interface { // for a non-seekable Stream, if Read is called, this function won't work. // caches the full Stream and writes it to writer (if provided, even if the stream is already cached). CacheFullAndWriter(up *UpdateProgress, writer io.Writer) (File, error) - SetTmpFile(file File) // if the Stream is not a File and is not cached, returns nil. GetFile() File } @@ -138,62 +137,55 @@ func WrapObjName(objs Obj) Obj { } func WrapObjsName(objs []Obj) { - for i := 0; i < len(objs); i++ { + for i := range objs { objs[i] = &ObjWrapName{Name: utils.MappingName(objs[i].GetName()), Obj: objs[i]} } } -func UnwrapObj(obj Obj) Obj { - if unwrap, ok := obj.(ObjUnwrap); ok { - obj = unwrap.Unwrap() +func UnwrapObjName(obj Obj) Obj { + if n, ok := obj.(*ObjWrapName); ok { + return n.Obj } return obj } func GetThumb(obj Obj) (thumb string, ok bool) { - if obj, ok := obj.(Thumb); ok { - return obj.Thumb(), true - } - if unwrap, ok := obj.(ObjUnwrap); ok { - return GetThumb(unwrap.Unwrap()) + for { + switch o := obj.(type) { + case Thumb: + return o.Thumb(), true + case ObjUnwrap: + obj = o.Unwrap() + default: + return + } } - return thumb, false } func GetUrl(obj Obj) (url string, ok bool) { - if obj, ok := obj.(URL); ok { - return obj.URL(), true - } - if unwrap, ok := obj.(ObjUnwrap); ok { - return GetUrl(unwrap.Unwrap()) + for { + switch o := obj.(type) { + case URL: + return o.URL(), true + case ObjUnwrap: + obj = o.Unwrap() + default: + return + } } - return url, false } func GetProvider(obj Obj) (string, bool) { - if obj, ok := obj.(ObjWithProvider); ok { - return obj.GetProvider(), true - } - if unwrap, ok := obj.(ObjUnwrap); ok { - return GetProvider(unwrap.Unwrap()) - } - return "unknown", false -} - -func GetRawObject(obj Obj) *Object { - switch v := obj.(type) { - case *ObjThumbURL: - return &v.Object - case *ObjThumb: - return &v.Object - case *ObjectURL: - return &v.Object - case *ObjectProvider: - return &v.Object - case *Object: - return v + for { + switch o := obj.(type) { + case ObjWithProvider: + return o.GetProvider(), true + case ObjUnwrap: + obj = o.Unwrap() + default: + return "unknown", false + } } - return nil } // Merge @@ -243,3 +235,52 @@ func (om *ObjMerge) InitHideReg(hides string) { func (om *ObjMerge) Reset() { om.set.Clear() } + +type ObjMask uint8 + +func (m ObjMask) GetObjMask() ObjMask { + return m +} + +const ( + Virtual ObjMask = 1 << iota + NoRename + NoRemove + NoMove + NoCopy + NoWrite + Temp +) +const ( + Locked = NoRename | NoRemove | NoMove + ReadOnly = Locked | NoWrite // NoRename | NoDelete | NoMove | NoWrite +) + +type ObjWrapMask struct { + Obj + Mask ObjMask +} + +func (m *ObjWrapMask) Unwrap() Obj { + return m.Obj +} +func (m *ObjWrapMask) GetObjMask() ObjMask { + return m.Mask +} + +func GetObjMask(obj Obj) ObjMask { + for { + switch o := obj.(type) { + case interface{ GetObjMask() ObjMask }: + return o.GetObjMask() + case ObjUnwrap: + obj = o.Unwrap() + default: + return 0 + } + } +} + +func ObjHasMask(obj Obj, mask ObjMask) bool { + return GetObjMask(obj)&mask != 0 +} diff --git a/internal/model/object.go b/internal/model/object.go index 8e5cdf047..b6cb0d7f7 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -28,6 +28,7 @@ type Object struct { Ctime time.Time // file create time IsFolder bool HashInfo utils.HashInfo + Mask ObjMask } func (o *Object) GetName() string { @@ -68,6 +69,10 @@ func (o *Object) GetHash() utils.HashInfo { return o.HashInfo } +func (o *Object) GetObjMask() ObjMask { + return o.Mask +} + type Thumbnail struct { Thumbnail string } diff --git a/internal/model/sharing.go b/internal/model/sharing.go index dfbc81b02..c5dd95e9e 100644 --- a/internal/model/sharing.go +++ b/internal/model/sharing.go @@ -33,7 +33,7 @@ func (s *Sharing) Valid() bool { if len(s.Files) == 0 { return false } - if !s.Creator.CanShare() { + if s.Creator == nil || !s.Creator.CanShare() { return false } if s.Expires != nil && !s.Expires.IsZero() && s.Expires.Before(time.Now()) { diff --git a/internal/model/storage.go b/internal/model/storage.go index 8c754e0fe..a6b4745a6 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -1,22 +1,24 @@ package model import ( + "encoding/json" "time" ) type Storage struct { - ID uint `json:"id" gorm:"primaryKey"` // unique key - MountPath string `json:"mount_path" gorm:"unique" binding:"required"` // must be standardized - Order int `json:"order"` // use to sort - Driver string `json:"driver"` // driver used - CacheExpiration int `json:"cache_expiration"` // cache expire time - Status string `json:"status"` - Addition string `json:"addition" gorm:"type:text"` // Additional information, defined in the corresponding driver - Remark string `json:"remark"` - Modified time.Time `json:"modified"` - Disabled bool `json:"disabled"` // if disabled - DisableIndex bool `json:"disable_index"` - EnableSign bool `json:"enable_sign"` + ID uint `json:"id" gorm:"primaryKey"` // unique key + MountPath string `json:"mount_path" gorm:"unique" binding:"required"` // must be standardized + Order int `json:"order"` // use to sort + Driver string `json:"driver"` // driver used + CacheExpiration int `json:"cache_expiration"` // cache expire time + CustomCachePolicies string `json:"custom_cache_policies" gorm:"type:text"` + Status string `json:"status"` + Addition string `json:"addition" gorm:"type:text"` // Additional information, defined in the corresponding driver + Remark string `json:"remark"` + Modified time.Time `json:"modified"` + Disabled bool `json:"disabled"` // if disabled + DisableIndex bool `json:"disable_index"` + EnableSign bool `json:"enable_sign"` Sort Proxy } @@ -32,7 +34,7 @@ type Proxy struct { WebdavPolicy string `json:"webdav_policy"` ProxyRange bool `json:"proxy_range"` DownProxyURL string `json:"down_proxy_url"` - //Disable sign for DownProxyURL + // Disable sign for DownProxyURL DisableProxySign bool `json:"disable_proxy_sign"` } @@ -57,33 +59,44 @@ func (p Proxy) WebdavProxyURL() bool { } type DiskUsage struct { - TotalSpace uint64 `json:"total_space"` - FreeSpace uint64 `json:"free_space"` + TotalSpace int64 + UsedSpace int64 } -type StorageDetails struct { - DiskUsage +func (d DiskUsage) FreeSpace() int64 { + return d.TotalSpace - d.UsedSpace } -type StorageDetailsWithName struct { - *StorageDetails - DriverName string `json:"driver_name"` +func (d DiskUsage) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "total_space": d.TotalSpace, + "used_space": d.UsedSpace, + "free_space": d.FreeSpace(), + }) +} + +type StorageDetails struct { + DiskUsage } type ObjWithStorageDetails interface { - GetStorageDetails() *StorageDetailsWithName + GetStorageDetails() *StorageDetails } type ObjStorageDetails struct { Obj - StorageDetailsWithName + *StorageDetails +} + +func (o *ObjStorageDetails) Unwrap() Obj { + return o.Obj } -func (o ObjStorageDetails) GetStorageDetails() *StorageDetailsWithName { - return &o.StorageDetailsWithName +func (o *ObjStorageDetails) GetStorageDetails() *StorageDetails { + return o.StorageDetails } -func GetStorageDetails(obj Obj) (*StorageDetailsWithName, bool) { +func GetStorageDetails(obj Obj) (*StorageDetails, bool) { if obj, ok := obj.(ObjWithStorageDetails); ok { return obj.GetStorageDetails(), true } diff --git a/internal/model/user.go b/internal/model/user.go index ae2d7d58c..3bad4ebb9 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -20,7 +20,15 @@ const ( ADMIN ) -const StaticHashSalt = "https://github.com/alist-org/alist" +const ( + StaticHashSalt = "https://github.com/alist-org/alist" + + InvalidUsernameOrPassword = "Invalid username or password" + Invalid2FACode = "Invalid 2FA code" + TooManyAttempts = "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later." + GuestCannotUpdateProfile = "Guest user can not update profile" + GuestCannotGenerate2FA = "Guest user can not generate 2FA code" +) var LoginCache = cache.NewMemCache[int]() @@ -59,6 +67,7 @@ type User struct { OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform Authn string `gorm:"type:text" json:"-"` + AllowLdap bool `json:"allow_ldap" gorm:"default:true"` } func (u *User) IsGuest() bool { @@ -90,64 +99,124 @@ func (u *User) SetPassword(pwd string) *User { return u } +func CanSeeHides(permission int32) bool { + return permission&1 == 1 +} + func (u *User) CanSeeHides() bool { - return u.Permission&1 == 1 + return CanSeeHides(u.Permission) +} + +func CanAccessWithoutPassword(permission int32) bool { + return (permission>>1)&1 == 1 } func (u *User) CanAccessWithoutPassword() bool { - return (u.Permission>>1)&1 == 1 + return CanAccessWithoutPassword(u.Permission) +} + +func CanAddOfflineDownloadTasks(permission int32) bool { + return (permission>>2)&1 == 1 } func (u *User) CanAddOfflineDownloadTasks() bool { - return (u.Permission>>2)&1 == 1 + return CanAddOfflineDownloadTasks(u.Permission) +} + +func CanWrite(permission int32) bool { + return (permission>>3)&1 == 1 } func (u *User) CanWrite() bool { - return (u.Permission>>3)&1 == 1 + return CanWrite(u.Permission) +} + +func CanRename(permission int32) bool { + return (permission>>4)&1 == 1 } func (u *User) CanRename() bool { - return (u.Permission>>4)&1 == 1 + return CanRename(u.Permission) +} + +func CanMove(permission int32) bool { + return (permission>>5)&1 == 1 } func (u *User) CanMove() bool { - return (u.Permission>>5)&1 == 1 + return CanMove(u.Permission) +} + +func CanCopy(permission int32) bool { + return (permission>>6)&1 == 1 } func (u *User) CanCopy() bool { - return (u.Permission>>6)&1 == 1 + return CanCopy(u.Permission) +} + +func CanRemove(permission int32) bool { + return (permission>>7)&1 == 1 } func (u *User) CanRemove() bool { - return (u.Permission>>7)&1 == 1 + return CanRemove(u.Permission) +} + +func CanWebdavRead(permission int32) bool { + return (permission>>8)&1 == 1 } func (u *User) CanWebdavRead() bool { - return (u.Permission>>8)&1 == 1 + return CanWebdavRead(u.Permission) +} + +func CanWebdavManage(permission int32) bool { + return (permission>>9)&1 == 1 } func (u *User) CanWebdavManage() bool { - return (u.Permission>>9)&1 == 1 + return CanWebdavManage(u.Permission) +} + +func CanFTPAccess(permission int32) bool { + return (permission>>10)&1 == 1 } func (u *User) CanFTPAccess() bool { - return (u.Permission>>10)&1 == 1 + return CanFTPAccess(u.Permission) +} + +func CanFTPManage(permission int32) bool { + return (permission>>11)&1 == 1 } func (u *User) CanFTPManage() bool { - return (u.Permission>>11)&1 == 1 + return CanFTPManage(u.Permission) +} + +func CanReadArchives(permission int32) bool { + return (permission>>12)&1 == 1 } func (u *User) CanReadArchives() bool { - return (u.Permission>>12)&1 == 1 + return CanReadArchives(u.Permission) +} + +func CanDecompress(permission int32) bool { + return (permission>>13)&1 == 1 } func (u *User) CanDecompress() bool { - return (u.Permission>>13)&1 == 1 + return CanDecompress(u.Permission) +} + +func CanShare(permission int32) bool { + return (permission>>14)&1 == 1 } func (u *User) CanShare() bool { - return (u.Permission>>14)&1 == 1 + return CanShare(u.Permission) } func (u *User) JoinPath(reqPath string) (string, error) { diff --git a/internal/net/request.go b/internal/net/request.go index 1306bc549..8d380ea4e 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -12,6 +12,7 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/rclone/rclone/lib/mmap" @@ -403,7 +404,7 @@ var errInfiniteRetry = errors.New("infinite retry") func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { resp, err := d.cfg.HttpClient(d.ctx, params) if err != nil { - statusCode, ok := errors.Unwrap(err).(HttpStatusCodeError) + statusCode, ok := errs.UnwrapOrSelf(err).(HttpStatusCodeError) if !ok { return 0, err } diff --git a/internal/net/serve.go b/internal/net/serve.go index f8d2e3ca0..6a20460b1 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -283,11 +283,15 @@ func HttpClient() *http.Client { } func NewHttpClient() *http.Client { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + } + + SetProxyIfConfigured(transport) + return &http.Client{ - Timeout: time.Hour * 48, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, - }, + Timeout: time.Hour * 48, + Transport: transport, } } diff --git a/internal/net/util.go b/internal/net/util.go index 40b5e145e..4bf2ba6cf 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -1,18 +1,20 @@ package net import ( - "fmt" "io" - "math" "mime/multipart" "net/http" "net/textproto" + "net/url" "strings" "time" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/rclone/rclone/lib/readers" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -305,39 +307,9 @@ func rangesMIMESize(ranges []http_range.Range, contentType string, contentSize i return encSize, nil } -// LimitedReadCloser wraps a io.ReadCloser and limits the number of bytes that can be read from it. -type LimitedReadCloser struct { - rc io.ReadCloser - remaining int -} - -func (l *LimitedReadCloser) Read(buf []byte) (int, error) { - if l.remaining <= 0 { - return 0, io.EOF - } - - if len(buf) > l.remaining { - buf = buf[0:l.remaining] - } - - n, err := l.rc.Read(buf) - l.remaining -= n - - return n, err -} - -func (l *LimitedReadCloser) Close() error { - return l.rc.Close() -} - // GetRangedHttpReader some http server doesn't support "Range" header, // so this function read readCloser with whole data, skip offset, then return ReaderCloser. func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.ReadCloser, error) { - var length_int int - if length > math.MaxInt { - return nil, fmt.Errorf("doesnot support length bigger than int32 max ") - } - length_int = int(length) if offset > 100*1024*1024 { log.Warnf("offset is more than 100MB, if loading data from internet, high-latency and wasting of bandwidth is expected") @@ -348,5 +320,25 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea } // return an io.ReadCloser that is limited to `length` bytes. - return &LimitedReadCloser{readCloser, length_int}, nil + return readers.NewLimitedReadCloser(readCloser, length), nil +} + +// SetProxyIfConfigured sets proxy for HTTP Transport if configured +func SetProxyIfConfigured(transport *http.Transport) { + // If proxy address is configured, override environment variable settings + if conf.Conf.ProxyAddress != "" { + if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + } +} + +// SetRestyProxyIfConfigured sets proxy for Resty client if configured +func SetRestyProxyIfConfigured(client *resty.Client) { + // If proxy address is configured, override environment variable settings + if conf.Conf.ProxyAddress != "" { + if proxyURL, err := url.Parse(conf.Conf.ProxyAddress); err == nil { + client.SetProxy(proxyURL.String()) + } + } } diff --git a/internal/offline_download/123/client.go b/internal/offline_download/123/client.go new file mode 100644 index 000000000..2c4f47048 --- /dev/null +++ b/internal/offline_download/123/client.go @@ -0,0 +1,142 @@ +package _123_pan + +import ( + "context" + "fmt" + "strconv" + + _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" +) + +type Pan123 struct{} + +func (*Pan123) Name() string { + return "123Pan" +} + +func (*Pan123) Items() []model.SettingItem { + return []model.SettingItem{ + {Key: conf.Pan123TempDir, Value: "", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (*Pan123) Run(_ *tool.DownloadTask) error { + return errs.NotSupport +} + +func (*Pan123) Init() (string, error) { + return "ok", nil +} + +func (*Pan123) IsReady() bool { + tempDir := setting.GetStr(conf.Pan123TempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*_123.Pan123); !ok { + return false + } + return true +} + +func (*Pan123) AddURL(args *tool.AddUrlArgs) (string, error) { + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver123, ok := storage.(*_123.Pan123) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") + } + ctx := context.Background() + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + taskID, err := driver123.OfflineDownload(ctx, args.Url, parentDir) + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + return strconv.FormatInt(taskID, 10), nil +} + +func (*Pan123) Remove(task *tool.DownloadTask) error { + taskID, err := strconv.ParseInt(task.GID, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse task ID: %s", task.GID) + } + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + driver123, ok := storage.(*_123.Pan123) + if !ok { + return fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") + } + return driver123.DeleteOfflineTasks(context.Background(), []int64{taskID}) +} + +func (*Pan123) Status(task *tool.DownloadTask) (*tool.Status, error) { + taskID, err := strconv.ParseInt(task.GID, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse task ID: %s", task.GID) + } + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver123, ok := storage.(*_123.Pan123) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only 123Pan is supported") + } + + t, err := driver123.GetOfflineTask(context.Background(), taskID) + if err != nil { + return nil, err + } + + var statusStr string + completed := false + var taskErr error + switch t.Status { + case 0: + statusStr = "downloading" + case 2: + statusStr = "succeed" + completed = true + case 1: + statusStr = "failed" + taskErr = fmt.Errorf("offline download failed") + case 3: + statusStr = "retrying" + default: + statusStr = fmt.Sprintf("status_%d", t.Status) + } + + return &tool.Status{ + TotalBytes: t.Size, + Progress: t.Progress, + Completed: completed, + Status: statusStr, + Err: taskErr, + }, nil +} + +var _ tool.Tool = (*Pan123)(nil) + +func init() { + tool.Tools.Add(&Pan123{}) +} diff --git a/internal/offline_download/123_open/client.go b/internal/offline_download/123_open/client.go new file mode 100644 index 000000000..ce1453c32 --- /dev/null +++ b/internal/offline_download/123_open/client.go @@ -0,0 +1,119 @@ +package _123_open + +import ( + "context" + "fmt" + "strconv" + + _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" +) + +type Open123 struct{} + +func (*Open123) Name() string { + return "123 Open" +} + +func (*Open123) Items() []model.SettingItem { + return nil +} + +func (*Open123) Run(_ *tool.DownloadTask) error { + return errs.NotSupport +} + +func (*Open123) Init() (string, error) { + return "ok", nil +} + +func (*Open123) IsReady() bool { + tempDir := setting.GetStr(conf.Pan123OpenTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*_123_open.Open123); !ok { + return false + } + return true +} + +func (*Open123) AddURL(args *tool.AddUrlArgs) (string, error) { + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver123Open, ok := storage.(*_123_open.Open123) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only 123 Open is supported") + } + ctx := context.Background() + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + cb := setting.GetStr(conf.Pan123OpenOfflineDownloadCallbackUrl) + taskID, err := driver123Open.OfflineDownload(ctx, args.Url, parentDir, cb) + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + return strconv.Itoa(taskID), nil +} + +func (*Open123) Remove(_ *tool.DownloadTask) error { + return errs.NotSupport +} + +func (*Open123) Status(task *tool.DownloadTask) (*tool.Status, error) { + taskID, err := strconv.Atoi(task.GID) + if err != nil { + return nil, fmt.Errorf("failed to parse task ID: %s", task.GID) + } + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver123Open, ok := storage.(*_123_open.Open123) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only 123 Open is supported") + } + process, status, err := driver123Open.OfflineDownloadProcess(context.Background(), taskID) + if err != nil { + return nil, err + } + var statusStr string + switch status { + case 0: + statusStr = "downloading" + case 1: + err = fmt.Errorf("offline download failed") + case 2: + statusStr = "succeed" + case 3: + statusStr = "retrying" + } + return &tool.Status{ + Progress: process, + Completed: status == 2, + Status: statusStr, + Err: err, + }, nil +} + +var _ tool.Tool = (*Open123)(nil) + +func init() { + tool.Tools.Add(&Open123{}) +} diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index f79a8312d..4fcbdb9c4 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -3,6 +3,8 @@ package offline_download import ( _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/115" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/115_open" + _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123" + _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123_open" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/aria2" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/http" _ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/pikpak" diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index aea88e2a4..0f574571e 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,18 +2,17 @@ package tool import ( "context" - "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" - - _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" - "github.com/OpenListTeam/OpenList/v4/server/common" - "net/url" stdpath "path" "path/filepath" _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" + _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" + _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" + "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -22,6 +21,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/task" + "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/google/uuid" "github.com/pkg/errors" ) @@ -104,6 +104,19 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } else { tempDir = filepath.Join(setting.GetStr(conf.Pan115OpenTempDir), uid) } + case "123 Open": + if _, ok := storage.(*_123_open.Open123); ok && dstDirActualPath != "/" { + // directly offline downloading to the root path is not allowed via 123 open platform + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.Pan123OpenTempDir), uid) + } + case "123Pan": + if _, ok := storage.(*_123.Pan123); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.Pan123TempDir), uid) + } case "PikPak": if _, ok := storage.(*pikpak.PikPak); ok { tempDir = args.DstDirPath diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 2d402dcf9..50a4f6343 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -2,6 +2,7 @@ package tool import ( "fmt" + "path" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -32,9 +33,6 @@ type DownloadTask struct { } func (t *DownloadTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err - } t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() @@ -111,6 +109,9 @@ outer: if t.tool.Name() == "115 Open" { return nil } + if t.tool.Name() == "123 Open" { + return nil + } t.Status = "offline download completed, maybe transferring" // hack for qBittorrent if t.tool.Name() == "qBittorrent" { @@ -174,7 +175,7 @@ func (t *DownloadTask) Update() (bool, error) { func (t *DownloadTask) Transfer() error { toolName := t.tool.Name() - if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" { + if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "123 Open" || toolName == "123Pan" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" { // 如果不是直接下载到目标路径，则进行转存 if t.TempDir != t.DstDirPath { return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) @@ -198,11 +199,11 @@ func (t *DownloadTask) Transfer() error { DstStorage: dstStorage, DstStorageMp: dstStorage.GetStorage().MountPath, }, - groupID: t.DstDirPath, DeletePolicy: t.DeletePolicy, Url: t.Url, } tsk.SetTotalBytes(t.GetTotalBytes()) + tsk.groupID = path.Join(tsk.DstStorageMp, tsk.DstActualPath) task_group.TransferCoordinator.AddTask(tsk.groupID, nil) TransferTaskManager.Add(tsk) return nil diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index 1c1284a02..fd6b8f464 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path" stdpath "path" "path/filepath" "time" @@ -31,8 +32,19 @@ type TransferTask struct { } func (t *TransferTask) Run() error { - if err := t.ReinitCtx(); err != nil { - return err + if t.SrcStorage == nil && t.SrcStorageMp != "" { + if srcStorage, _, err := op.GetStorageAndActualPath(t.SrcStorageMp); err == nil { + t.SrcStorage = srcStorage + } else { + return err + } + if t.DstStorage == nil { + if dstStorage, _, err := op.GetStorageAndActualPath(t.DstStorageMp); err == nil { + t.DstStorage = dstStorage + } else { + return err + } + } } t.ClearEndTime() t.SetStartTime(time.Now()) @@ -61,12 +73,11 @@ func (t *TransferTask) Run() error { Mimetype: mimetype, Closers: utils.NewClosers(r), } - return op.Put(t.Ctx(), t.DstStorage, t.DstActualPath, s, t.SetProgress) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) } return transferStdPath(t) - } else { - return transferObjPath(t) } + return transferObjPath(t) } func (t *TransferTask) GetName() string { @@ -84,7 +95,7 @@ func (t *TransferTask) OnSucceeded() { removeObjTemp(t) } } - task_group.TransferCoordinator.Done(t.groupID, true) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, true) } func (t *TransferTask) OnFailed() { @@ -95,7 +106,7 @@ func (t *TransferTask) OnFailed() { removeObjTemp(t) } } - task_group.TransferCoordinator.Done(t.groupID, false) + task_group.TransferCoordinator.Done(context.WithoutCancel(t.Ctx()), t.groupID, false) } func (t *TransferTask) SetRetry(retry int, maxRetry int) { @@ -105,7 +116,7 @@ func (t *TransferTask) SetRetry(retry int, maxRetry int) { t.groupID = stdpath.Join(t.DstStorageMp, t.DstActualPath) task_group.TransferCoordinator.AddTask(t.groupID, nil) } - t.TaskExtension.SetRetry(retry, maxRetry) + t.TaskData.SetRetry(retry, maxRetry) } var ( @@ -134,10 +145,10 @@ func transferStd(ctx context.Context, tempDir, dstDirPath string, deletePolicy D DstStorage: dstStorage, DstStorageMp: dstStorage.GetStorage().MountPath, }, - groupID: dstDirPath, DeletePolicy: deletePolicy, } - task_group.TransferCoordinator.AddTask(dstDirPath, nil) + t.groupID = path.Join(t.DstStorageMp, t.DstActualPath) + task_group.TransferCoordinator.AddTask(t.groupID, nil) TransferTaskManager.Add(t) } return nil @@ -156,7 +167,7 @@ func transferStdPath(t *TransferTask) error { return err } dstDirActualPath := stdpath.Join(t.DstActualPath, info.Name()) - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToRefresh(dstDirActualPath)) + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstDirActualPath)) for _, entry := range entries { srcRawPath := stdpath.Join(t.SrcActualPath, entry.Name()) task := &TransferTask{ @@ -206,7 +217,7 @@ func transferStdFile(t *TransferTask) error { Closers: utils.NewClosers(rc), } t.SetTotalBytes(info.Size()) - return op.Put(t.Ctx(), t.DstStorage, t.DstActualPath, s, t.SetProgress) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, s, t.SetProgress) } func removeStdTemp(t *TransferTask) { @@ -247,10 +258,10 @@ func transferObj(ctx context.Context, tempDir, dstDirPath string, deletePolicy D SrcStorageMp: srcStorage.GetStorage().MountPath, DstStorageMp: dstStorage.GetStorage().MountPath, }, - groupID: dstDirPath, DeletePolicy: deletePolicy, } - task_group.TransferCoordinator.AddTask(dstDirPath, nil) + t.groupID = path.Join(t.DstStorageMp, t.DstActualPath) + task_group.TransferCoordinator.AddTask(t.groupID, nil) TransferTaskManager.Add(t) } return nil @@ -269,7 +280,7 @@ func transferObjPath(t *TransferTask) error { return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcActualPath) } dstDirActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) - task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToRefresh(dstDirActualPath)) + task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstDirActualPath)) for _, obj := range objs { if utils.IsCanceled(t.Ctx()) { return nil @@ -300,11 +311,11 @@ func transferObjPath(t *TransferTask) error { } func transferObjFile(t *TransferTask) error { - srcFile, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcActualPath) + _, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcActualPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", t.SrcActualPath) } - link, _, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) + link, srcFile, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcActualPath, model.LinkArgs{}) if err != nil { return errors.WithMessagef(err, "failed get [%s] link", t.SrcActualPath) } @@ -318,7 +329,7 @@ func transferObjFile(t *TransferTask) error { return errors.WithMessagef(err, "failed get [%s] stream", t.SrcActualPath) } t.SetTotalBytes(ss.GetSize()) - return op.Put(t.Ctx(), t.DstStorage, t.DstActualPath, ss, t.SetProgress) + return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } func removeObjTemp(t *TransferTask) { diff --git a/internal/op/archive.go b/internal/op/archive.go index 4d85d2064..d0a53a919 100644 --- a/internal/op/archive.go +++ b/internal/op/archive.go @@ -3,31 +3,35 @@ package op import ( "context" stderrors "errors" - "fmt" "io" stdpath "path" + "strconv" "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/archive/tool" - "github.com/OpenListTeam/OpenList/v4/internal/stream" - + "github.com/OpenListTeam/OpenList/v4/internal/cache" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/go-cache" + gocache "github.com/OpenListTeam/go-cache" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) -var archiveMetaCache = cache.NewMemCache(cache.WithShards[*model.ArchiveMetaProvider](64)) -var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider] +var ( + archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64)) + archiveMetaG singleflight.Group[*model.ArchiveMetaProvider] +) func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) key := Key(storage, path) @@ -37,14 +41,14 @@ func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg return nil, errors.Wrapf(err, "failed to get %s archive met: %+v", path, err) } if m.Expiration != nil { - archiveMetaCache.Set(key, m, cache.WithEx[*model.ArchiveMetaProvider](*m.Expiration)) + archiveMetaCache.Set(key, m, gocache.WithEx[*model.ArchiveMetaProvider](*m.Expiration)) } return m, nil } - if storage.Config().OnlyLinkMFile { - meta, err := fn() - return meta, err - } + // if storage.Config().NoLinkSingleflight { + // meta, err := fn() + // return meta, err + // } if !args.Refresh { if meta, ok := archiveMetaCache.Get(key); ok { log.Debugf("use cache when get %s archive meta", path) @@ -60,20 +64,25 @@ func GetArchiveToolAndStream(ctx context.Context, storage driver.Driver, path st if err != nil { return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] link", path) } - baseName, ext, found := strings.Cut(obj.GetName(), ".") - if !found { - _ = l.Close() - return nil, nil, nil, errors.Errorf("failed get archive tool: the obj does not have an extension.") - } - partExt, t, err := tool.GetArchiveTool("." + ext) - if err != nil { - var e error - partExt, t, e = tool.GetArchiveTool(stdpath.Ext(obj.GetName())) - if e != nil { + + // Get archive tool + var partExt *tool.MultipartExtension + var t tool.Tool + ext := obj.GetName() + for { + var found bool + _, ext, found = strings.Cut(ext, ".") + if !found { _ = l.Close() - return nil, nil, nil, errors.WithMessagef(stderrors.Join(err, e), "failed get archive tool: %s", ext) + return nil, nil, nil, errors.Errorf("failed get archive tool: the obj does not have an extension.") + } + partExt, t, err = tool.GetArchiveTool("." + ext) + if err == nil { + break } } + + // Get first part stream ss, err := stream.NewSeekableStream(&stream.FileStream{Ctx: ctx, Obj: obj}, l) if err != nil { _ = l.Close() @@ -82,29 +91,62 @@ func GetArchiveToolAndStream(ctx context.Context, storage driver.Driver, path st ret := []*stream.SeekableStream{ss} if partExt == nil { return obj, t, ret, nil - } else { - index := partExt.SecondPartIndex - dir := stdpath.Dir(path) - for { - p := stdpath.Join(dir, baseName+fmt.Sprintf(partExt.PartFileFormat, index)) - var o model.Obj - l, o, err = Link(ctx, storage, p, args) - if err != nil { - break - } - ss, err = stream.NewSeekableStream(&stream.FileStream{Ctx: ctx, Obj: o}, l) - if err != nil { - _ = l.Close() - for _, s := range ret { - _ = s.Close() - } - return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path) + } + + // Merge multi-part archive + dir := stdpath.Dir(path) + objs, err := List(ctx, storage, dir, model.ListArgs{}) + if err != nil { + return obj, t, ret, nil + } + for _, o := range objs { + submatch := partExt.PartFileFormat.FindStringSubmatch(o.GetName()) + if submatch == nil { + continue + } + partIdx, e := strconv.Atoi(submatch[1]) + if e != nil { + continue + } + partIdx = partIdx - partExt.SecondPartIndex + 1 + if partIdx < 1 { + continue + } + p := stdpath.Join(dir, o.GetName()) + l1, o1, e := Link(ctx, storage, p, args) + if e != nil { + err = errors.WithMessagef(e, "failed get [%s] link", p) + break + } + ss1, e := stream.NewSeekableStream(&stream.FileStream{Ctx: ctx, Obj: o1}, l1) + if e != nil { + _ = l1.Close() + err = errors.WithMessagef(e, "failed get [%s] stream", p) + break + } + for partIdx >= len(ret) { + ret = append(ret, nil) + } + ret[partIdx] = ss1 + } + closeAll := func(r []*stream.SeekableStream) { + for _, s := range r { + if s != nil { + _ = s.Close() } - ret = append(ret, ss) - index++ } - return obj, t, ret, nil } + if err != nil { + closeAll(ret) + return nil, nil, nil, err + } + for i, ss1 := range ret { + if ss1 == nil { + closeAll(ret) + return nil, nil, nil, errors.Errorf("failed merge [%s] parts, missing part %d", path, i) + } + } + return obj, t, ret, nil } func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (model.Obj, *model.ArchiveMetaProvider, error) { @@ -158,12 +200,14 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg return obj, archiveMetaProvider, err } -var archiveListCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) -var archiveListG singleflight.Group[[]model.Obj] +var ( + archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64)) + archiveListG singleflight.Group[[]model.Obj] +) func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) metaKey := Key(storage, path) @@ -179,16 +223,10 @@ func ListArchive(ctx context.Context, storage driver.Driver, path string, args m // } } objs, err, _ := archiveListG.Do(key, func() ([]model.Obj, error) { - obj, files, err := listArchive(ctx, storage, path, args) + files, err := listArchive(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed to list archive [%s]%s: %+v", path, args.InnerPath, err) } - // set path - for _, f := range files { - if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && obj.GetPath() != "" { - s.SetPath(stdpath.Join(obj.GetPath(), args.InnerPath, f.GetName())) - } - } // warp obj name model.WrapObjsName(files) // sort objs @@ -199,7 +237,7 @@ func ListArchive(ctx context.Context, storage driver.Driver, path string, args m if !storage.Config().NoCache { if len(files) > 0 { log.Debugf("set cache: %s => %+v", key, files) - archiveListCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + archiveListCache.Set(key, files, gocache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) } else { log.Debugf("del cache: %s", key) archiveListCache.Del(key) @@ -210,24 +248,24 @@ func ListArchive(ctx context.Context, storage driver.Driver, path string, args m return objs, err } -func _listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { +func _listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { storageAr, ok := storage.(driver.ArchiveReader) if ok { obj, err := GetUnwrap(ctx, storage, path) if err != nil { - return nil, nil, errors.WithMessage(err, "failed to get file") + return nil, errors.WithMessage(err, "failed to get file") } if obj.IsDir() { - return nil, nil, errors.WithStack(errs.NotFile) + return nil, errors.WithStack(errs.NotFile) } files, err := storageAr.ListArchive(ctx, obj, args.ArchiveInnerArgs) if !errors.Is(err, errs.NotImplement) { - return obj, files, err + return files, err } } - obj, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + _, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { - return nil, nil, err + return nil, err } defer func() { var e error @@ -239,11 +277,11 @@ func _listArchive(ctx context.Context, storage driver.Driver, path string, args } }() files, err := t.List(ss, args.ArchiveInnerArgs) - return obj, files, err + return files, err } -func listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { - obj, files, err := _listArchive(ctx, storage, path, args) +func listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { + files, err := _listArchive(ctx, storage, path, args) if errors.Is(err, errs.NotSupport) { var meta model.ArchiveMeta meta, err = GetArchiveMeta(ctx, storage, path, model.ArchiveMetaArgs{ @@ -251,20 +289,17 @@ func listArchive(ctx context.Context, storage driver.Driver, path string, args m Refresh: args.Refresh, }) if err != nil { - return nil, nil, err + return nil, err } files, err = getChildrenFromArchiveMeta(meta, args.InnerPath) if err != nil { - return nil, nil, err + return nil, err } } - if err == nil && obj == nil { - obj, err = GetUnwrap(ctx, storage, path) - } if err != nil { - return nil, nil, err + return nil, err } - return obj, files, err + return files, err } func getChildrenFromArchiveMeta(meta model.ArchiveMeta, innerPath string) ([]model.Obj, error) { @@ -309,7 +344,7 @@ func splitPath(path string) []string { func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) af, err := GetUnwrap(ctx, storage, path) @@ -354,75 +389,52 @@ func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args mo return nil, nil, errors.WithStack(errs.ObjectNotFound) } -type extractLink struct { - *model.Link - Obj model.Obj +type objWithLink struct { + link *model.Link + obj model.Obj } -var extractCache = cache.NewMemCache(cache.WithShards[*extractLink](16)) -var extractG = singleflight.Group[*extractLink]{Remember: true} +var ( + extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute) + extractG = singleflight.Group[*objWithLink]{} +) func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } key := stdpath.Join(Key(storage, path), args.InnerPath) - if link, ok := extractCache.Get(key); ok { - return link.Link, link.Obj, nil + if ol, ok := extractCache.Get(key); ok { + if ol.link.Expiration != nil || ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil + } } - var forget any - var linkM *extractLink - fn := func() (*extractLink, error) { - link, err := driverExtract(ctx, storage, path, args) + fn := func() (*objWithLink, error) { + ol, err := driverExtract(ctx, storage, path, args) if err != nil { return nil, errors.Wrapf(err, "failed extract archive") } - if link.MFile != nil && forget != nil { - linkM = link - return nil, errLinkMFileCache - } - if link.Link.Expiration != nil { - extractCache.Set(key, link, cache.WithEx[*extractLink](*link.Link.Expiration)) + if ol.link.Expiration != nil { + extractCache.SetWithTTL(key, ol, *ol.link.Expiration) + } else { + extractCache.SetWithExpirable(key, ol, &ol.link.SyncClosers) } - link.AddIfCloser(forget) - return link, nil + return ol, nil } - if storage.Config().OnlyLinkMFile { - link, err := fn() + for { + ol, err, _ := extractG.Do(key, fn) if err != nil { return nil, nil, err } - return link.Link, link.Obj, nil - } - - forget = utils.CloseFunc(func() error { - if forget != nil { - forget = nil - linkG.Forget(key) - } - return nil - }) - link, err, _ := extractG.Do(key, fn) - for err == nil && !link.AcquireReference() { - link, err, _ = extractG.Do(key, fn) - } - if err == errLinkMFileCache { - if linkM != nil { - return linkM.Link, linkM.Obj, nil + if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil } - forget = nil - link, err = fn() } - - if err != nil { - return nil, nil, err - } - return link.Link, link.Obj, nil } -func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*extractLink, error) { +func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*objWithLink, error) { storageAr, ok := storage.(driver.ArchiveReader) if !ok { return nil, errs.DriverExtractNotSupported @@ -438,7 +450,7 @@ func driverExtract(ctx context.Context, storage driver.Driver, path string, args return nil, errors.WithStack(errs.NotFile) } link, err := storageAr.Extract(ctx, archiveFile, args) - return &extractLink{Link: link, Obj: extracted}, err + return &objWithLink{link: link, obj: extracted}, err } type streamWithParent struct { @@ -480,7 +492,7 @@ func InternalExtract(ctx context.Context, storage driver.Driver, path string, ar func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) @@ -493,26 +505,56 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD return errors.WithMessage(err, "failed to get dst dir") } + var newObjs []model.Obj switch s := storage.(type) { case driver.ArchiveDecompressResult: - var newObjs []model.Obj newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) if err == nil { if len(newObjs) > 0 { - for _, newObj := range newObjs { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { + for _, newObj := range newObjs { + cache.UpdateObject(newObj.GetName(), newObj) + } + } } } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) + Cache.DeleteDirectory(storage, dstDirPath) } } case driver.ArchiveDecompress: err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) + Cache.DeleteDirectory(storage, dstDirPath) } default: return errs.NotImplement } + if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() { + onlyList := false + targetPath := dstDirPath + if len(newObjs) == 1 && newObjs[0].IsDir() { + targetPath = stdpath.Join(dstDirPath, newObjs[0].GetName()) + } else if len(newObjs) == 1 && !newObjs[0].IsDir() { + onlyList = true + } else if args.PutIntoNewDir { + targetPath = stdpath.Join(dstDirPath, strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName()))) + } else if innerBase := stdpath.Base(args.InnerPath); innerBase != "." && innerBase != "/" { + targetPath = stdpath.Join(dstDirPath, innerBase) + dstObj, e := Get(ctx, storage, targetPath) + onlyList = e != nil || !dstObj.IsDir() + } + if onlyList { + go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true}) + } else { + var limiter *rate.Limiter + if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil { + if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 { + limiter = rate.NewLimiter(rate.Limit(f), 1) + } + } + go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil) + } + } return errors.WithStack(err) } diff --git a/internal/op/cache.go b/internal/op/cache.go new file mode 100644 index 000000000..d8d32a74b --- /dev/null +++ b/internal/op/cache.go @@ -0,0 +1,228 @@ +package op + +import ( + stdpath "path" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/cache" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type CacheManager struct { + dirCache *cache.KeyedCache[*directoryCache] // Cache for directory listings + linkCache *cache.TypedCache[*objWithLink] // Cache for file links + userCache *cache.KeyedCache[*model.User] // Cache for user data + settingCache *cache.KeyedCache[any] // Cache for settings + detailCache *cache.KeyedCache[*model.StorageDetails] // Cache for storage details +} + +func NewCacheManager() *CacheManager { + return &CacheManager{ + dirCache: cache.NewKeyedCache[*directoryCache](time.Minute * 5), + linkCache: cache.NewTypedCache[*objWithLink](time.Minute * 30), + userCache: cache.NewKeyedCache[*model.User](time.Hour), + settingCache: cache.NewKeyedCache[any](time.Hour), + detailCache: cache.NewKeyedCache[*model.StorageDetails](time.Minute * 30), + } +} + +// global instance +var Cache = NewCacheManager() + +func Key(storage driver.Driver, path string) string { + return utils.GetFullPath(storage.GetStorage().MountPath, path) +} + +// recursively delete directory and its children from dirCache +func (cm *CacheManager) DeleteDirectoryTree(storage driver.Driver, dirPath string) { + if storage.Config().NoCache { + return + } + cm.deleteDirectoryTree(Key(storage, dirPath)) +} +func (cm *CacheManager) deleteDirectoryTree(key string) { + if dirCache, exists := cm.dirCache.Pop(key); exists { + for _, obj := range dirCache.objs { + if obj.IsDir() { + cm.deleteDirectoryTree(stdpath.Join(key, obj.GetName())) + } else { + cm.linkCache.DeleteKey(stdpath.Join(key, obj.GetName())) + } + } + } +} + +// remove directory from dirCache +func (cm *CacheManager) DeleteDirectory(storage driver.Driver, dirPath string) { + if storage.Config().NoCache { + return + } + cm.dirCache.Delete(Key(storage, dirPath)) +} + +// remove object from dirCache. +// if it's a directory, remove all its children from dirCache too. +// if it's a file, remove its link from linkCache. +func (cm *CacheManager) removeDirectoryObject(storage driver.Driver, dirPath string, obj model.Obj) { + key := Key(storage, dirPath) + if !obj.IsDir() { + cm.linkCache.DeleteKey(stdpath.Join(key, obj.GetName())) + } + + if storage.Config().NoCache { + return + } + if cache, exist := cm.dirCache.Get(key); exist { + if obj.IsDir() { + cm.deleteDirectoryTree(stdpath.Join(key, obj.GetName())) + } + cache.RemoveObject(obj.GetName()) + } +} + +// cache user data +func (cm *CacheManager) SetUser(username string, user *model.User) { + cm.userCache.Set(username, user) +} + +// cached user data +func (cm *CacheManager) GetUser(username string) (*model.User, bool) { + return cm.userCache.Get(username) +} + +// remove user data from cache +func (cm *CacheManager) DeleteUser(username string) { + cm.userCache.Delete(username) +} + +// caches setting +func (cm *CacheManager) SetSetting(key string, setting *model.SettingItem) { + cm.settingCache.Set(key, setting) +} + +// cached setting +func (cm *CacheManager) GetSetting(key string) (*model.SettingItem, bool) { + if data, exists := cm.settingCache.Get(key); exists { + if setting, ok := data.(*model.SettingItem); ok { + return setting, true + } + } + return nil, false +} + +// cache setting groups +func (cm *CacheManager) SetSettingGroup(key string, settings []model.SettingItem) { + cm.settingCache.Set(key, settings) +} + +// cached setting group +func (cm *CacheManager) GetSettingGroup(key string) ([]model.SettingItem, bool) { + if data, exists := cm.settingCache.Get(key); exists { + if settings, ok := data.([]model.SettingItem); ok { + return settings, true + } + } + return nil, false +} + +func (cm *CacheManager) SetStorageDetails(storage driver.Driver, details *model.StorageDetails) { + if storage.Config().NoCache { + return + } + expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) + cm.detailCache.SetWithTTL(utils.GetActualMountPath(storage.GetStorage().MountPath), details, expiration) +} + +func (cm *CacheManager) GetStorageDetails(storage driver.Driver) (*model.StorageDetails, bool) { + return cm.detailCache.Get(utils.GetActualMountPath(storage.GetStorage().MountPath)) +} + +func (cm *CacheManager) InvalidateStorageDetails(storage driver.Driver) { + cm.detailCache.Delete(utils.GetActualMountPath(storage.GetStorage().MountPath)) +} + +// clears all caches +func (cm *CacheManager) ClearAll() { + cm.dirCache.Clear() + cm.linkCache.Clear() + cm.userCache.Clear() + cm.settingCache.Clear() + cm.detailCache.Clear() +} + +type directoryCache struct { + objs []model.Obj + sorted []model.Obj + mu sync.RWMutex + + dirtyFlags uint8 +} + +const ( + dirtyRemove uint8 = 1 << iota // 对象删除：刷新 sorted 副本，但不需要 full sort/extract + dirtyUpdate // 对象更新：需要执行 full sort + extract +) + +func newDirectoryCache(objs []model.Obj) *directoryCache { + sorted := make([]model.Obj, len(objs)) + copy(sorted, objs) + return &directoryCache{ + objs: objs, + sorted: sorted, + } +} + +func (dc *directoryCache) RemoveObject(name string) { + dc.mu.Lock() + defer dc.mu.Unlock() + for i, obj := range dc.objs { + if obj.GetName() == name { + dc.objs = append(dc.objs[:i], dc.objs[i+1:]...) + dc.dirtyFlags |= dirtyRemove + break + } + } +} + +func (dc *directoryCache) UpdateObject(oldName string, newObj model.Obj) { + dc.mu.Lock() + defer dc.mu.Unlock() + if oldName != "" { + for i, obj := range dc.objs { + if obj.GetName() == oldName { + dc.objs[i] = newObj + dc.dirtyFlags |= dirtyUpdate + return + } + } + } + dc.objs = append(dc.objs, newObj) + dc.dirtyFlags |= dirtyUpdate +} + +func (dc *directoryCache) GetSortedObjects(meta driver.Meta) []model.Obj { + dc.mu.RLock() + if dc.dirtyFlags == 0 { + dc.mu.RUnlock() + return dc.sorted + } + dc.mu.RUnlock() + dc.mu.Lock() + defer dc.mu.Unlock() + + sorted := make([]model.Obj, len(dc.objs)) + copy(sorted, dc.objs) + dc.sorted = sorted + if dc.dirtyFlags&dirtyUpdate != 0 { + storage := meta.GetStorage() + if meta.Config().LocalSort { + model.SortFiles(sorted, storage.OrderBy, storage.OrderDirection) + } + model.ExtractFolder(sorted, storage.ExtractFolder) + } + dc.dirtyFlags = 0 + return sorted +} diff --git a/internal/op/driver.go b/internal/op/driver.go index f25b3a6b1..5b79b0aed 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -80,6 +80,13 @@ func getMainItems(config driver.Config) []driver.Item { Required: true, Help: "The cache expiration time for this storage", }) + items = append(items, driver.Item{ + Name: "custom_cache_policies", + Type: conf.TypeText, + Default: "", + Required: false, + Help: "The cache expiration rules for this storage", + }) } if config.MustProxy() { items = append(items, driver.Item{ @@ -90,17 +97,32 @@ func getMainItems(config driver.Config) []driver.Item { Required: true, }) } else { - items = append(items, []driver.Item{{ - Name: "web_proxy", - Type: conf.TypeBool, - }, { - Name: "webdav_policy", - Type: conf.TypeSelect, - Options: "302_redirect,use_proxy_url,native_proxy", - Default: "302_redirect", - Required: true, - }, - }...) + if config.DefaultProxy() { + items = append(items, []driver.Item{{ + Name: "web_proxy", + Type: conf.TypeBool, + Default: "true", + }, { + Name: "webdav_policy", + Type: conf.TypeSelect, + Options: "302_redirect,use_proxy_url,native_proxy", + Default: "native_proxy", + Required: true, + }, + }...) + } else { + items = append(items, []driver.Item{{ + Name: "web_proxy", + Type: conf.TypeBool, + }, { + Name: "webdav_policy", + Type: conf.TypeSelect, + Options: "302_redirect,use_proxy_url,native_proxy", + Default: "302_redirect", + Required: true, + }, + }...) + } if config.ProxyRangeOption { item := driver.Item{ Name: "proxy_range", diff --git a/internal/op/fs.go b/internal/op/fs.go index 99f6c84af..5116bbef5 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -2,242 +2,212 @@ package op import ( "context" - stderrors "errors" stdpath "path" - "slices" + "strconv" "strings" "time" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/stream" - "github.com/OpenListTeam/OpenList/v4/pkg/generic_sync" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/go-cache" + "github.com/bmatcuk/doublestar/v4" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) -// In order to facilitate adding some other things before and after file op - -var listCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) var listG singleflight.Group[[]model.Obj] -func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj model.Obj) { - key := Key(storage, path) - objs, ok := listCache.Get(key) - if ok { - for i, obj := range objs { - if obj.GetName() == newObj.GetName() { - objs = slices.Delete(objs, i, i+1) - break - } - } - for i, obj := range objs { - if obj.GetName() == oldObj.GetName() { - objs[i] = newObj - break - } - } - listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } -} - -func delCacheObj(storage driver.Driver, path string, obj model.Obj) { - key := Key(storage, path) - objs, ok := listCache.Get(key) - if ok { - for i, oldObj := range objs { - if oldObj.GetName() == obj.GetName() { - objs = append(objs[:i], objs[i+1:]...) - break - } - } - listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } -} - -var addSortDebounceMap generic_sync.MapOf[string, func(func())] - -func addCacheObj(storage driver.Driver, path string, newObj model.Obj) { - key := Key(storage, path) - objs, ok := listCache.Get(key) - if ok { - for i, obj := range objs { - if obj.GetName() == newObj.GetName() { - objs[i] = newObj - return - } - } - - // Simple separation of files and folders - if len(objs) > 0 && objs[len(objs)-1].IsDir() == newObj.IsDir() { - objs = append(objs, newObj) - } else { - objs = append([]model.Obj{newObj}, objs...) - } - - if storage.Config().LocalSort { - debounce, _ := addSortDebounceMap.LoadOrStore(key, utils.NewDebounce(time.Minute)) - log.Debug("addCacheObj: wait start sort") - debounce(func() { - log.Debug("addCacheObj: start sort") - model.SortFiles(objs, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) - addSortDebounceMap.Delete(key) - }) - } - - listCache.Set(key, objs, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) - } -} - -func ClearCache(storage driver.Driver, path string) { - objs, ok := listCache.Get(Key(storage, path)) - if ok { - for _, obj := range objs { - if obj.IsDir() { - ClearCache(storage, stdpath.Join(path, obj.GetName())) - } - } - } - listCache.Del(Key(storage, path)) -} - -func DeleteCache(storage driver.Driver, path string) { - listCache.Del(Key(storage, path)) -} - -func Key(storage driver.Driver, path string) string { - return stdpath.Join(storage.GetStorage().MountPath, utils.FixAndCleanPath(path)) -} - // List files in storage, not contains virtual file func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs) ([]model.Obj, error) { + return list(ctx, storage, path, args, nil) +} + +func list(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, resultValidator func([]model.Obj) error) ([]model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) log.Debugf("op.List %s", path) key := Key(storage, path) if !args.Refresh { - if files, ok := listCache.Get(key); ok { + if dirCache, exists := Cache.dirCache.Get(key); exists { log.Debugf("use cache when list %s", path) - return files, nil + objs := dirCache.GetSortedObjects(storage) + if resultValidator != nil { + if err := resultValidator(objs); err == nil { + return objs, nil + } + } else { + return objs, nil + } } } - dir, err := GetUnwrap(ctx, storage, path) - if err != nil { - return nil, errors.WithMessage(err, "failed get dir") - } - log.Debugf("list dir: %+v", dir) - if !dir.IsDir() { - return nil, errors.WithStack(errs.NotFolder) - } + objs, err, _ := listG.Do(key, func() ([]model.Obj, error) { + dir, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, errors.WithMessage(err, "failed get dir") + } + log.Debugf("list dir: %+v", dir) + if !dir.IsDir() { + return nil, errors.WithStack(errs.NotFolder) + } files, err := storage.List(ctx, dir, args) if err != nil { return nil, errors.Wrapf(err, "failed to list objs") } - // set path - for _, f := range files { - if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && dir.GetPath() != "" { - s.SetPath(stdpath.Join(dir.GetPath(), f.GetName())) - } - } // warp obj name - model.WrapObjsName(files) - // call hooks - go func(reqPath string, files []model.Obj) { - HandleObjsUpdateHook(reqPath, files) - }(utils.GetFullPath(storage.GetStorage().MountPath, path), files) - + wrapObjsName(storage, files) // sort objs if storage.Config().LocalSort { model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) } model.ExtractFolder(files, storage.GetStorage().ExtractFolder) + if !args.SkipHook { + // call hooks + go func(reqPath string, files []model.Obj) { + HandleObjsUpdateHook(context.WithoutCancel(ctx), reqPath, files) + }(utils.GetFullPath(storage.GetStorage().MountPath, path), files) + } + if !storage.Config().NoCache { if len(files) > 0 { log.Debugf("set cache: %s => %+v", key, files) - listCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + + ttl := storage.GetStorage().CacheExpiration + + customCachePolicies := storage.GetStorage().CustomCachePolicies + if len(customCachePolicies) > 0 { + configPolicies := strings.Split(customCachePolicies, "\n") + for _, configPolicy := range configPolicies { + pattern, ttlstr, ok := strings.Cut(strings.TrimSpace(configPolicy), ":") + if !ok { + log.Warnf("Malformed custom cache policy entry: %s in storage %s for path %s. Expected format: pattern:ttl", configPolicy, storage.GetStorage().MountPath, path) + continue + } + if match, err1 := doublestar.Match(pattern, path); err1 != nil { + log.Warnf("Invalid glob pattern in custom cache policy: %s, error: %v", pattern, err1) + continue + } else if !match { + continue + } + + if configTtl, err1 := strconv.ParseInt(ttlstr, 10, 64); err1 == nil { + ttl = int(configTtl) + break + } + } + } + + duration := time.Minute * time.Duration(ttl) + Cache.dirCache.SetWithTTL(key, newDirectoryCache(files), duration) } else { log.Debugf("del cache: %s", key) - listCache.Del(key) + Cache.deleteDirectoryTree(key) } } return files, nil }) - return objs, err + if err != nil { + return nil, err + } + if resultValidator != nil { + if err := resultValidator(objs); err != nil { + return nil, err + } + } + return objs, nil } // Get object from list of files -func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { +func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj ...bool) (model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) + } path = utils.FixAndCleanPath(path) log.Debugf("op.Get %s", path) + // is root folder + if path == "/" { + if getRooter, ok := storage.(driver.GetRooter); ok { + rootObj, err := getRooter.GetRoot(ctx) + if err != nil { + return nil, errors.WithMessage(err, "failed get root obj") + } + return rootObj, nil + } + switch r := storage.(type) { + case driver.IRootId: + return &model.Object{ + ID: r.GetRootId(), + Name: RootName, + Modified: storage.GetStorage().Modified, + IsFolder: true, + Mask: model.Locked, + }, nil + case driver.IRootPath: + return &model.Object{ + Path: r.GetRootPath(), + Name: RootName, + Modified: storage.GetStorage().Modified, + Mask: model.Locked, + IsFolder: true, + }, nil + } + return nil, errors.New("please implement GetRooter or IRootPath or IRootId interface") + } + + // try get from cache first + dir, name := stdpath.Split(path) + dirCache, dirCacheExists := Cache.dirCache.Get(Key(storage, dir)) + refreshList := false + excludeTemp := utils.IsBool(excludeTempObj...) + if dirCacheExists { + files := dirCache.GetSortedObjects(storage) + for _, f := range files { + if f.GetName() == name { + if excludeTemp && model.ObjHasMask(f, model.Temp) { + refreshList = true + break + } + return f, nil + } + } + } + // get the obj directly without list so that we can reduce the io if g, ok := storage.(driver.Getter); ok { obj, err := g.Get(ctx, path) if err == nil { - return model.WrapObjName(obj), nil + return obj, nil } if !errs.IsNotImplementError(err) && !errs.IsNotSupportError(err) { return nil, errors.WithMessage(err, "failed to get obj") } } - // is root folder - if utils.PathEqual(path, "/") { - var rootObj model.Obj - if getRooter, ok := storage.(driver.GetRooter); ok { - obj, err := getRooter.GetRoot(ctx) - if err != nil { - return nil, errors.WithMessage(err, "failed get root obj") - } - rootObj = obj - } else { - switch r := storage.GetAddition().(type) { - case driver.IRootId: - rootObj = &model.Object{ - ID: r.GetRootId(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, - } - case driver.IRootPath: - rootObj = &model.Object{ - Path: r.GetRootPath(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, + if !dirCacheExists || refreshList { + var obj model.Obj + list(ctx, storage, dir, model.ListArgs{Refresh: refreshList}, func(objs []model.Obj) error { + for _, f := range objs { + if f.GetName() == name { + if excludeTemp && model.ObjHasMask(f, model.Temp) { + return errs.ObjectNotFound + } + obj = f + return nil } - default: - return nil, errors.Errorf("please implement IRootPath or IRootId or GetRooter method") } - } - if rootObj == nil { - return nil, errors.Errorf("please implement IRootPath or IRootId or GetRooter method") - } - return &model.ObjWrapName{ - Name: RootName, - Obj: rootObj, - }, nil - } - - // not root folder - dir, name := stdpath.Split(path) - files, err := List(ctx, storage, dir, model.ListArgs{}) - if err != nil { - return nil, errors.WithMessage(err, "failed get parent list") - } - for _, f := range files { - if f.GetName() == name { - return f, nil + return nil + }) + if obj != nil { + return obj, nil } } log.Debugf("cant find obj with name: %s", name) @@ -245,313 +215,368 @@ func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, er } func GetUnwrap(ctx context.Context, storage driver.Driver, path string) (model.Obj, error) { - obj, err := Get(ctx, storage, path) + obj, err := Get(ctx, storage, path, true) if err != nil { return nil, err } - return model.UnwrapObj(obj), err + return model.UnwrapObjName(obj), err } -var linkCache = cache.NewMemCache(cache.WithShards[*model.Link](16)) -var linkG = singleflight.Group[*model.Link]{Remember: true} -var errLinkMFileCache = stderrors.New("ErrLinkMFileCache") +var linkG = singleflight.Group[*objWithLink]{} // Link get link, if is an url. should have an expiry time func Link(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (*model.Link, model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) - } - var ( - file model.Obj - err error - ) - // use cache directly - dir, name := stdpath.Split(stdpath.Join(storage.GetStorage().MountPath, path)) - if cacheFiles, ok := listCache.Get(strings.TrimSuffix(dir, "/")); ok { - for _, f := range cacheFiles { - if f.GetName() == name { - file = model.UnwrapObj(f) - break - } - } - } else { - if g, ok := storage.(driver.GetObjInfo); ok { - file, err = g.GetObjInfo(ctx, path) - } else { - file, err = GetUnwrap(ctx, storage, path) - } + return nil, nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } - if file == nil { - if err != nil { - return nil, nil, errors.WithMessage(err, "failed to get file") - } - return nil, nil, errors.WithStack(errs.ObjectNotFound) + + mode := storage.Config().LinkCacheMode + if mode == -1 { + mode = storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(path) } - if file.IsDir() { - return nil, nil, errors.WithStack(errs.NotFile) + typeKey := args.Type + if mode&driver.LinkCacheIP != 0 { + typeKey += "/" + args.IP } - - key := stdpath.Join(Key(storage, path), args.Type) - if link, ok := linkCache.Get(key); ok { - return link, file, nil + if mode&driver.LinkCacheUA != 0 { + typeKey += "/" + args.Header.Get("User-Agent") } + key := Key(storage, path) + if ol, exists := Cache.linkCache.GetType(key, typeKey); exists { + if ol.link.Expiration != nil || + ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil + } + } + + fn := func() (*objWithLink, error) { + file, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, errors.WithMessage(err, "failed to get file") + } + if file.IsDir() { + return nil, errors.WithStack(errs.NotFile) + } - var forget any - var linkM *model.Link - fn := func() (*model.Link, error) { link, err := storage.Link(ctx, file, args) if err != nil { return nil, errors.Wrapf(err, "failed get link") } - if link.MFile != nil && forget != nil { - linkM = link - return nil, errLinkMFileCache - } + ol := &objWithLink{link: link, obj: file} if link.Expiration != nil { - linkCache.Set(key, link, cache.WithEx[*model.Link](*link.Expiration)) + Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) + } else { + Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) } - link.AddIfCloser(forget) - return link, nil + return ol, nil } - - if storage.Config().OnlyLinkMFile { - link, err := fn() + for { + ol, err, _ := linkG.Do(key+"/"+typeKey, fn) if err != nil { return nil, nil, err } - return link, file, err - } - - forget = utils.CloseFunc(func() error { - if forget != nil { - forget = nil - linkG.Forget(key) + if ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { + return ol.link, ol.obj, nil } - return nil - }) - link, err, _ := linkG.Do(key, fn) - for err == nil && !link.AcquireReference() { - link, err, _ = linkG.Do(key, fn) } - - if err == errLinkMFileCache { - if linkM != nil { - return linkM, file, nil - } - forget = nil - link, err = fn() - } - - if err != nil { - return nil, nil, err - } - return link, file, nil } // Other api -func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (interface{}, error) { +func Other(ctx context.Context, storage driver.Driver, args model.FsOtherArgs) (any, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) + } + o, ok := storage.(driver.Other) + if !ok { + return nil, errs.NotImplement + } obj, err := GetUnwrap(ctx, storage, args.Path) if err != nil { return nil, errors.WithMessagef(err, "failed to get obj") } - if o, ok := storage.(driver.Other); ok { - return o.Other(ctx, model.OtherArgs{ - Obj: obj, - Method: args.Method, - Data: args.Data, - }) - } else { - return nil, errs.NotImplement - } + return o.Other(ctx, model.OtherArgs{ + Obj: obj, + Method: args.Method, + Data: args.Data, + }) } -var mkdirG singleflight.Group[interface{}] +var mkdirG singleflight.Group[any] -func MakeDir(ctx context.Context, storage driver.Driver, path string, lazyCache ...bool) error { +func MakeDir(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) key := Key(storage, path) - _, err, _ := mkdirG.Do(key, func() (interface{}, error) { + _, err, _ := mkdirG.Do(key, func() (any, error) { // check if dir exists - f, err := GetUnwrap(ctx, storage, path) - if err != nil { - if errs.IsObjectNotFound(err) { - parentPath, dirName := stdpath.Split(path) - err = MakeDir(ctx, storage, parentPath) - if err != nil { - return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) - } - parentDir, err := GetUnwrap(ctx, storage, parentPath) - // this should not happen - if err != nil { - return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) - } - - switch s := storage.(type) { - case driver.MkdirResult: - var newObj model.Obj - newObj, err = s.MakeDir(ctx, parentDir, dirName) - if err == nil { - if newObj != nil { - addCacheObj(storage, parentPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, parentPath) - } - } - case driver.Mkdir: - err = s.MakeDir(ctx, parentDir, dirName) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, parentPath) - } - default: - return nil, errs.NotImplement - } - return nil, errors.WithStack(err) + f, err := Get(ctx, storage, path) + if err == nil { + if f.IsDir() { + return nil, nil } + return nil, errors.New("file exists") + } + if !errs.IsObjectNotFound(err) { return nil, errors.WithMessage(err, "failed to check if dir exists") } - // dir exists - if f.IsDir() { + parentPath, dirName := stdpath.Split(path) + if err = MakeDir(ctx, storage, parentPath); err != nil { + return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) + } + parentDir, err := GetUnwrap(ctx, storage, parentPath) + // this should not happen + if err != nil { + return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } + if model.ObjHasMask(parentDir, model.NoWrite) { + return nil, errors.WithStack(errs.PermissionDenied) + } + + var newObj model.Obj + switch s := storage.(type) { + case driver.MkdirResult: + newObj, err = s.MakeDir(ctx, parentDir, dirName) + case driver.Mkdir: + err = s.MakeDir(ctx, parentDir, dirName) + default: + return nil, errs.NotImplement + } + if err != nil { + return nil, errors.WithStack(err) + } + if storage.Config().NoCache { return nil, nil } - // dir to make is a file - return nil, errors.New("file exists") + if dirCache, exist := Cache.dirCache.Get(Key(storage, parentPath)); exist { + if newObj == nil { + t := time.Now() + newObj = &model.Object{ + Name: dirName, + IsFolder: true, + Modified: t, + Ctime: t, + Mask: model.Temp, + } + } + dirCache.UpdateObject("", wrapObjName(storage, newObj)) + } + return nil, nil }) return err } -func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { +func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) + if utils.PathEqual(srcPath, "/") { + return errors.New("move root folder is not allowed") + } + srcDirPath := stdpath.Dir(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) - srcRawObj, err := Get(ctx, storage, srcPath) + if dstDirPath == srcDirPath { + return errors.New("move in place") + } + srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { return errors.WithMessage(err, "failed to get src object") } - srcObj := model.UnwrapObj(srcRawObj) + if model.ObjHasMask(srcRawObj, model.NoMove) { + return errors.WithStack(errs.PermissionDenied) + } + srcObj := model.UnwrapObjName(srcRawObj) dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } - srcDirPath := stdpath.Dir(srcPath) + if model.ObjHasMask(dstDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } + var newObj model.Obj switch s := storage.(type) { case driver.MoveResult: - var newObj model.Obj newObj, err = s.Move(ctx, srcObj, dstDir) - if err == nil { - delCacheObj(storage, srcDirPath, srcRawObj) - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } - } case driver.Move: err = s.Move(ctx, srcObj, dstDir) - if err == nil { - delCacheObj(storage, srcDirPath, srcRawObj) - if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) + default: + err = errs.NotImplement + } + if err != nil { + return errors.WithStack(err) + } + + srcKey := Key(storage, srcDirPath) + dstKey := Key(storage, dstDirPath) + if !srcRawObj.IsDir() { + Cache.linkCache.DeleteKey(stdpath.Join(srcKey, srcRawObj.GetName())) + Cache.linkCache.DeleteKey(stdpath.Join(dstKey, srcRawObj.GetName())) + } + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(srcKey); exist { + if srcRawObj.IsDir() { + Cache.deleteDirectoryTree(stdpath.Join(srcKey, srcRawObj.GetName())) } + cache.RemoveObject(srcRawObj.GetName()) + } + if cache, exist := Cache.dirCache.Get(dstKey); exist { + if newObj == nil { + newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp} + } else { + newObj = wrapObjName(storage, newObj) + } + cache.UpdateObject(srcRawObj.GetName(), newObj) } - default: - return errs.NotImplement } - return errors.WithStack(err) + + if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { + return nil + } + if !srcObj.IsDir() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } else { + go objsUpdateHook(context.WithoutCancel(ctx), storage, stdpath.Join(dstDirPath, srcObj.GetName()), true) + } + return nil } -func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string, lazyCache ...bool) error { +func Rename(ctx context.Context, storage driver.Driver, srcPath, dstName string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) - srcRawObj, err := Get(ctx, storage, srcPath) + if utils.PathEqual(srcPath, "/") { + return errors.New("rename root folder is not allowed") + } + srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { return errors.WithMessage(err, "failed to get src object") } - srcObj := model.UnwrapObj(srcRawObj) - srcDirPath := stdpath.Dir(srcPath) + if model.ObjHasMask(srcRawObj, model.NoRename) { + return errors.WithStack(errs.PermissionDenied) + } + srcObj := model.UnwrapObjName(srcRawObj) + var newObj model.Obj switch s := storage.(type) { case driver.RenameResult: - var newObj model.Obj newObj, err = s.Rename(ctx, srcObj, dstName) - if err == nil { - if newObj != nil { - updateCacheObj(storage, srcDirPath, srcRawObj, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, srcDirPath) - if srcRawObj.IsDir() { - ClearCache(storage, srcPath) - } - } - } case driver.Rename: err = s.Rename(ctx, srcObj, dstName) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, srcDirPath) + default: + return errs.NotImplement + } + if err != nil { + return errors.WithStack(err) + } + + dirKey := Key(storage, stdpath.Dir(srcPath)) + if !srcRawObj.IsDir() { + Cache.linkCache.DeleteKey(stdpath.Join(dirKey, srcRawObj.GetName())) + Cache.linkCache.DeleteKey(stdpath.Join(dirKey, dstName)) + } + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(dirKey); exist { if srcRawObj.IsDir() { - ClearCache(storage, srcPath) + Cache.deleteDirectoryTree(stdpath.Join(dirKey, srcRawObj.GetName())) + } + if newObj == nil { + newObj = &model.ObjWrapMask{Obj: &model.ObjWrapName{Name: dstName, Obj: srcObj}, Mask: model.Temp} } + newObj = wrapObjName(storage, newObj) + cache.UpdateObject(srcRawObj.GetName(), newObj) } - default: - return errs.NotImplement } - return errors.WithStack(err) + + if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { + return nil + } + dstDirPath := stdpath.Dir(srcPath) + if !srcObj.IsDir() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } else { + go objsUpdateHook(context.WithoutCancel(ctx), storage, stdpath.Join(dstDirPath, srcObj.GetName()), true) + } + return nil } // Copy Just copy file[s] in a storage -func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, lazyCache ...bool) error { +func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } srcPath = utils.FixAndCleanPath(srcPath) dstDirPath = utils.FixAndCleanPath(dstDirPath) - srcObj, err := GetUnwrap(ctx, storage, srcPath) + if dstDirPath == stdpath.Dir(srcPath) { + return errors.New("copy in place") + } + srcRawObj, err := Get(ctx, storage, srcPath, true) if err != nil { return errors.WithMessage(err, "failed to get src object") } + // if model.ObjHasMask(srcRawObj, model.NoCopy) { + // return errors.WithStack(errs.PermissionDenied) + // } + srcObj := model.UnwrapObjName(srcRawObj) dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { return errors.WithMessage(err, "failed to get dst dir") } + if model.ObjHasMask(dstDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } + var newObj model.Obj switch s := storage.(type) { case driver.CopyResult: - var newObj model.Obj newObj, err = s.Copy(ctx, srcObj, dstDir) - if err == nil { - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } - } case driver.Copy: err = s.Copy(ctx, srcObj, dstDir) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } default: - return errs.NotImplement + err = errs.NotImplement } - return errors.WithStack(err) + if err != nil { + return errors.WithStack(err) + } + + dstKey := Key(storage, dstDirPath) + if !srcRawObj.IsDir() { + Cache.linkCache.DeleteKey(stdpath.Join(dstKey, srcRawObj.GetName())) + } + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(dstKey); exist { + if newObj == nil { + newObj = &model.ObjWrapMask{Obj: srcRawObj, Mask: model.Temp} + } else { + newObj = wrapObjName(storage, newObj) + } + cache.UpdateObject(srcRawObj.GetName(), newObj) + } + } + + if ctx.Value(conf.SkipHookKey) != nil || !needHandleObjsUpdateHook() { + return nil + } + if !srcObj.IsDir() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } else { + go objsUpdateHook(context.WithoutCancel(ctx), storage, stdpath.Join(dstDirPath, srcObj.GetName()), true) + } + return nil } func Remove(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } + path = utils.FixAndCleanPath(path) if utils.PathEqual(path, "/") { - return errors.New("delete root folder is not allowed, please goto the manage page to delete the storage instead") + return errors.New("delete root folder is not allowed") } - path = utils.FixAndCleanPath(path) - rawObj, err := Get(ctx, storage, path) + rawObj, err := Get(ctx, storage, path, true) if err != nil { // if object not found, it's ok if errs.IsObjectNotFound(err) { @@ -560,17 +585,16 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { } return errors.WithMessage(err, "failed to get object") } + if model.ObjHasMask(rawObj, model.NoRemove) { + return errors.WithStack(errs.PermissionDenied) + } dirPath := stdpath.Dir(path) switch s := storage.(type) { case driver.Remove: - err = s.Remove(ctx, model.UnwrapObj(rawObj)) + err = s.Remove(ctx, model.UnwrapObjName(rawObj)) if err == nil { - delCacheObj(storage, dirPath, rawObj) - // clear folder cache recursively - if rawObj.IsDir() { - ClearCache(storage, path) - } + Cache.removeDirectoryObject(storage, dirPath, rawObj) } default: return errs.NotImplement @@ -578,21 +602,20 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { return errors.WithStack(err) } -func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress, lazyCache ...bool) error { - close := file.Close +func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file model.FileStreamer, up driver.UpdateProgress) error { defer func() { - if err := close(); err != nil { + if err := file.Close(); err != nil { log.Errorf("failed to close file streamer, %v", err) } }() if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } // UrlTree PUT - if storage.GetStorage().Driver == "UrlTree" { + if storage.Config().OnlyIndices { var link string dstDirPath, link = urlTreeSplitLineFormPath(stdpath.Join(dstDirPath, file.GetName())) - file = &stream.FileStream{Obj: &model.Object{Name: link}} + file = &stream.FileStream{Obj: &model.Object{Name: link}, Closers: utils.Closers{file}} } // if file exist and size = 0, delete it dstDirPath = utils.FixAndCleanPath(dstDirPath) @@ -625,6 +648,9 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod if err != nil { return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) } + if model.ObjHasMask(parentDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) + } // if up is nil, set a default to prevent panic if up == nil { up = func(p float64) {} @@ -635,25 +661,38 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod log.Warnf("file size < 0, try to get full size from cache") file.CacheFullAndWriter(nil, nil) } + + var newObj model.Obj switch s := storage.(type) { case driver.PutResult: - var newObj model.Obj newObj, err = s.Put(ctx, parentDir, file, up) - if err == nil { - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } - } case driver.Put: err = s.Put(ctx, parentDir, file, up) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } default: return errs.NotImplement } + if err == nil { + Cache.linkCache.DeleteKey(Key(storage, dstPath)) + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { + if newObj == nil { + newObj = &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + Modified: file.ModTime(), + Ctime: file.CreateTime(), + Mask: model.Temp, + } + } + newObj = wrapObjName(storage, newObj) + cache.UpdateObject(newObj.GetName(), newObj) + } + } + + if ctx.Value(conf.SkipHookKey) == nil && needHandleObjsUpdateHook() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } + } log.Debugf("put file [%s] done", file.GetName()) if storage.Config().NoOverwriteUpload && fi != nil && fi.GetSize() > 0 { if err != nil { @@ -664,54 +703,163 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod } } else { // upload success, remove old obj - err := Remove(ctx, storage, tempPath) - if err != nil { - return err - } else { - key := Key(storage, stdpath.Join(dstDirPath, file.GetName())) - linkCache.Del(key) - } + err = Remove(ctx, storage, tempPath) } } return errors.WithStack(err) } -func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string, lazyCache ...bool) error { +func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { - return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + return errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) } dstDirPath = utils.FixAndCleanPath(dstDirPath) - _, err := GetUnwrap(ctx, storage, stdpath.Join(dstDirPath, dstName)) - if err == nil { - return errors.New("obj already exists") + dstPath := stdpath.Join(dstDirPath, dstName) + + if _, err := Get(ctx, storage, dstPath); err == nil { + return errors.WithStack(errs.ObjectAlreadyExists) } - err = MakeDir(ctx, storage, dstDirPath) + err := MakeDir(ctx, storage, dstDirPath) if err != nil { - return errors.WithMessagef(err, "failed to put url") + return errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) } dstDir, err := GetUnwrap(ctx, storage, dstDirPath) if err != nil { - return errors.WithMessagef(err, "failed to put url") + return errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) + } + if model.ObjHasMask(dstDir, model.NoWrite) { + return errors.WithStack(errs.PermissionDenied) } + var newObj model.Obj switch s := storage.(type) { case driver.PutURLResult: - var newObj model.Obj newObj, err = s.PutURL(ctx, dstDir, dstName, url) - if err == nil { - if newObj != nil { - addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) - } else if !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } - } case driver.PutURL: err = s.PutURL(ctx, dstDir, dstName, url) - if err == nil && !utils.IsBool(lazyCache...) { - DeleteCache(storage, dstDirPath) - } default: - return errs.NotImplement + return errors.WithStack(errs.NotImplement) + } + if err == nil { + Cache.linkCache.DeleteKey(Key(storage, dstPath)) + if !storage.Config().NoCache { + if cache, exist := Cache.dirCache.Get(Key(storage, dstDirPath)); exist { + if newObj == nil { + t := time.Now() + newObj = &model.Object{ + Name: dstName, + Modified: t, + Ctime: t, + Mask: model.Temp, + } + } + newObj = wrapObjName(storage, newObj) + cache.UpdateObject(newObj.GetName(), newObj) + } + + if ctx.Value(conf.SkipHookKey) == nil && needHandleObjsUpdateHook() { + go objsUpdateHook(context.WithoutCancel(ctx), storage, dstDirPath, false) + } + } } log.Debugf("put url [%s](%s) done", dstName, url) return errors.WithStack(err) } + +func GetDirectUploadTools(storage driver.Driver) []string { + du, ok := storage.(driver.DirectUploader) + if !ok { + return nil + } + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil + } + return du.GetDirectUploadTools() +} + +func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver, dstDirPath, dstName string, fileSize int64) (any, error) { + du, ok := storage.(driver.DirectUploader) + if !ok { + return nil, errors.WithStack(errs.NotImplement) + } + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) + } + dstDirPath = utils.FixAndCleanPath(dstDirPath) + dstPath := stdpath.Join(dstDirPath, dstName) + _, err := Get(ctx, storage, dstPath) + if err == nil { + return nil, errors.WithStack(errs.ObjectAlreadyExists) + } + err = MakeDir(ctx, storage, dstDirPath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to make dir [%s]", dstDirPath) + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return nil, errors.WithMessagef(err, "failed to get dir [%s]", dstDirPath) + } + info, err := du.GetDirectUploadInfo(ctx, tool, dstDir, dstName, fileSize) + if err != nil { + return nil, errors.WithStack(err) + } + return info, nil +} + +func objsUpdateHook(ctx context.Context, storage driver.Driver, dirPath string, recursive bool) { + files, err := List(ctx, storage, dirPath, model.ListArgs{SkipHook: true}) + if err != nil { + return + } + if !recursive { + HandleObjsUpdateHook(ctx, utils.GetFullPath(storage.GetStorage().MountPath, dirPath), files) + return + } + var limiter *rate.Limiter + if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil { + if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 { + limiter = rate.NewLimiter(rate.Limit(f), 1) + } + } + recursivelyObjsUpdateHook(ctx, storage, dirPath, files, limiter) +} +func recursivelyObjsUpdateHook(ctx context.Context, storage driver.Driver, dirPath string, files []model.Obj, limiter *rate.Limiter) { + HandleObjsUpdateHook(ctx, utils.GetFullPath(storage.GetStorage().MountPath, dirPath), files) + for _, f := range files { + if utils.IsCanceled(ctx) { + return + } + if !f.IsDir() { + continue + } + dstPath := stdpath.Join(dirPath, f.GetName()) + if limiter != nil { + if err := limiter.Wait(ctx); err != nil { + return + } + } + files, err := List(ctx, storage, dstPath, model.ListArgs{SkipHook: true}) + if err == nil { + recursivelyObjsUpdateHook(ctx, storage, dstPath, files, limiter) + } + } +} + +func needHandleObjsUpdateHook() bool { + if len(objsUpdateHooks) < 1 { + return false + } + needHandle, _ := GetSettingItemByKey(conf.HandleHookAfterWriting) + return needHandle != nil && (needHandle.Value == "true" || needHandle.Value == "1") +} + +func wrapObjsName(storage driver.Driver, objs []model.Obj) { + if _, ok := storage.(driver.Getter); !ok { + model.WrapObjsName(objs) + } +} +func wrapObjName(storage driver.Driver, obj model.Obj) model.Obj { + if _, ok := storage.(driver.Getter); !ok { + return model.WrapObjName(obj) + } + return obj +} diff --git a/internal/op/hook.go b/internal/op/hook.go index 22a069985..5cf01730d 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -1,6 +1,7 @@ package op import ( + "context" "regexp" "strings" @@ -13,7 +14,7 @@ import ( ) // Obj -type ObjsUpdateHook = func(parent string, objs []model.Obj) +type ObjsUpdateHook = func(ctx context.Context, parent string, objs []model.Obj) var ( objsUpdateHooks = make([]ObjsUpdateHook, 0) @@ -23,9 +24,9 @@ func RegisterObjsUpdateHook(hook ObjsUpdateHook) { objsUpdateHooks = append(objsUpdateHooks, hook) } -func HandleObjsUpdateHook(parent string, objs []model.Obj) { +func HandleObjsUpdateHook(ctx context.Context, parent string, objs []model.Obj) { for _, hook := range objsUpdateHooks { - hook(parent, objs) + hook(ctx, parent, objs) } } diff --git a/internal/op/recursive_list.go b/internal/op/recursive_list.go new file mode 100644 index 000000000..de8a27b2b --- /dev/null +++ b/internal/op/recursive_list.go @@ -0,0 +1,125 @@ +package op + +import ( + "context" + stdpath "path" + "sync" + "sync/atomic" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" +) + +var ( + ManualScanCancel = atomic.Pointer[context.CancelFunc]{} + ScannedCount = atomic.Uint64{} +) + +func ManualScanRunning() bool { + return ManualScanCancel.Load() != nil +} + +func BeginManualScan(rawPath string, limit float64) error { + rawPath = utils.FixAndCleanPath(rawPath) + ctx, cancel := context.WithCancel(context.Background()) + if !ManualScanCancel.CompareAndSwap(nil, &cancel) { + cancel() + return errors.New("manual scan is running, please try later") + } + ScannedCount.Store(0) + go func() { + defer func() { (*ManualScanCancel.Swap(nil))() }() + err := RecursivelyList(ctx, rawPath, rate.Limit(limit), &ScannedCount) + if err != nil { + log.Errorf("failed recursively list: %v", err) + } + }() + return nil +} + +func StopManualScan() { + c := ManualScanCancel.Load() + if c != nil { + (*c)() + } +} + +func RecursivelyList(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64) error { + storage, actualPath, err := GetStorageAndActualPath(rawPath) + if err != nil && !errors.Is(err, errs.StorageNotFound) { + return err + } else if err == nil { + var limiter *rate.Limiter + if limit > .0 { + limiter = rate.NewLimiter(limit, 1) + } + RecursivelyListStorage(ctx, storage, actualPath, limiter, counter) + } else { + var wg sync.WaitGroup + recursivelyListVirtual(ctx, rawPath, limit, counter, &wg) + wg.Wait() + } + return nil +} + +func recursivelyListVirtual(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64, wg *sync.WaitGroup) { + objs := GetStorageVirtualFilesByPath(rawPath) + if counter != nil { + counter.Add(uint64(len(objs))) + } + for _, obj := range objs { + if utils.IsCanceled(ctx) { + return + } + nextPath := stdpath.Join(rawPath, obj.GetName()) + storage, actualPath, err := GetStorageAndActualPath(nextPath) + if err != nil && !errors.Is(err, errs.StorageNotFound) { + log.Errorf("error recursively list: failed get storage [%s]: %v", nextPath, err) + } else if err == nil { + var limiter *rate.Limiter + if limit > .0 { + limiter = rate.NewLimiter(limit, 1) + } + wg.Add(1) + go func() { + defer wg.Done() + RecursivelyListStorage(ctx, storage, actualPath, limiter, counter) + }() + } else { + recursivelyListVirtual(ctx, nextPath, limit, counter, wg) + } + } +} + +func RecursivelyListStorage(ctx context.Context, storage driver.Driver, actualPath string, limiter *rate.Limiter, counter *atomic.Uint64) { + objs, err := List(ctx, storage, actualPath, model.ListArgs{Refresh: true}) + if err != nil { + if !errors.Is(err, context.Canceled) { + log.Errorf("error recursively list: failed list (%s)[%s]: %v", storage.GetStorage().MountPath, actualPath, err) + } + return + } + if counter != nil { + counter.Add(uint64(len(objs))) + } + for _, obj := range objs { + if utils.IsCanceled(ctx) { + return + } + if !obj.IsDir() { + continue + } + if limiter != nil { + if err = limiter.Wait(ctx); err != nil { + return + } + } + nextPath := stdpath.Join(actualPath, obj.GetName()) + RecursivelyListStorage(ctx, storage, nextPath, limiter, counter) + } +} diff --git a/internal/op/setting.go b/internal/op/setting.go index cbfa083ff..bda4daaf6 100644 --- a/internal/op/setting.go +++ b/internal/op/setting.go @@ -5,26 +5,21 @@ import ( "sort" "strconv" "strings" - "time" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/go-cache" "github.com/pkg/errors" ) -var settingCache = cache.NewMemCache(cache.WithShards[*model.SettingItem](4)) var settingG singleflight.Group[*model.SettingItem] var settingCacheF = func(item *model.SettingItem) { - settingCache.Set(item.Key, item, cache.WithEx[*model.SettingItem](time.Hour)) + Cache.SetSetting(item.Key, item) } -var settingGroupCache = cache.NewMemCache(cache.WithShards[[]model.SettingItem](4)) var settingGroupG singleflight.Group[[]model.SettingItem] -var settingGroupCacheF = func(key string, item []model.SettingItem) { - settingGroupCache.Set(key, item, cache.WithEx[[]model.SettingItem](time.Hour)) +var settingGroupCacheF = func(key string, items []model.SettingItem) { + Cache.SetSettingGroup(key, items) } var settingChangingCallbacks = make([]func(), 0) @@ -34,8 +29,7 @@ func RegisterSettingChangingCallback(f func()) { } func SettingCacheUpdate() { - settingCache.Clear() - settingGroupCache.Clear() + Cache.ClearAll() for _, cb := range settingChangingCallbacks { cb() } @@ -60,7 +54,7 @@ func GetSettingsMap() map[string]string { } func GetSettingItems() ([]model.SettingItem, error) { - if items, ok := settingGroupCache.Get("ALL_SETTING_ITEMS"); ok { + if items, exists := Cache.GetSettingGroup("ALL_SETTING_ITEMS"); exists { return items, nil } items, err, _ := settingGroupG.Do("ALL_SETTING_ITEMS", func() ([]model.SettingItem, error) { @@ -75,7 +69,7 @@ func GetSettingItems() ([]model.SettingItem, error) { } func GetPublicSettingItems() ([]model.SettingItem, error) { - if items, ok := settingGroupCache.Get("ALL_PUBLIC_SETTING_ITEMS"); ok { + if items, exists := Cache.GetSettingGroup("ALL_PUBLIC_SETTING_ITEMS"); exists { return items, nil } items, err, _ := settingGroupG.Do("ALL_PUBLIC_SETTING_ITEMS", func() ([]model.SettingItem, error) { @@ -90,7 +84,7 @@ func GetPublicSettingItems() ([]model.SettingItem, error) { } func GetSettingItemByKey(key string) (*model.SettingItem, error) { - if item, ok := settingCache.Get(key); ok { + if item, exists := Cache.GetSetting(key); exists { return item, nil } @@ -118,8 +112,8 @@ func GetSettingItemInKeys(keys []string) ([]model.SettingItem, error) { } func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) { - key := strconv.Itoa(group) - if items, ok := settingGroupCache.Get(key); ok { + key := fmt.Sprintf("GROUP_%d", group) + if items, exists := Cache.GetSettingGroup(key); exists { return items, nil } items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) { @@ -135,11 +129,14 @@ func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) { func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) { sort.Ints(groups) - key := strings.Join(utils.MustSliceConvert(groups, func(i int) string { - return strconv.Itoa(i) - }), ",") - if items, ok := settingGroupCache.Get(key); ok { + keyParts := make([]string, 0, len(groups)) + for _, g := range groups { + keyParts = append(keyParts, strconv.Itoa(g)) + } + key := "GROUPS_" + strings.Join(keyParts, "_") + + if items, exists := Cache.GetSettingGroup(key); exists { return items, nil } items, err, _ := settingGroupG.Do(key, func() ([]model.SettingItem, error) { @@ -165,10 +162,10 @@ func SaveSettingItems(items []model.SettingItem) error { } } err := db.SaveSettingItems(items) - if err != nil { + if err != nil { return fmt.Errorf("failed save setting: %+v", err) } - SettingCacheUpdate() + SettingCacheUpdate() return nil } diff --git a/internal/op/sharing.go b/internal/op/sharing.go index dbff1ba89..9db51c59d 100644 --- a/internal/op/sharing.go +++ b/internal/op/sharing.go @@ -137,3 +137,7 @@ func DeleteSharing(sid string) error { sharingCache.Del(sid) return db.DeleteSharingById(sid) } + +func DeleteSharingsByCreatorId(creatorId uint) error { + return db.DeleteSharingsByCreatorId(creatorId) +} diff --git a/internal/op/storage.go b/internal/op/storage.go index 53784802c..da4c84e31 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -15,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/generic_sync" + "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -239,6 +240,8 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) + Cache.DeleteDirectoryTree(storageDriver, "/") + Cache.InvalidateStorageDetails(storageDriver) } if err != nil { return errors.WithMessage(err, "failed get storage driver") @@ -259,6 +262,7 @@ func DeleteStorageById(ctx context.Context, id uint) error { if err != nil { return errors.WithMessage(err, "failed get storage") } + var dropErr error = nil if !storage.Disabled { storageDriver, err := GetStorageByMountPath(storage.MountPath) if err != nil { @@ -266,17 +270,19 @@ func DeleteStorageById(ctx context.Context, id uint) error { } // drop the storage in the driver if err := storageDriver.Drop(ctx); err != nil { - return errors.Wrapf(err, "failed drop storage") + dropErr = errors.Wrapf(err, "failed drop storage") } // delete the storage in the memory storagesMap.Delete(storage.MountPath) + Cache.DeleteDirectoryTree(storageDriver, "/") + Cache.InvalidateStorageDetails(storageDriver) go callStorageHooks("del", storageDriver) } // delete the storage in the database if err := db.DeleteStorageById(id); err != nil { return errors.WithMessage(err, "failed delete storage in database") } - return nil + return dropErr } // MustSaveDriverStorage call from specific driver @@ -335,38 +341,41 @@ func getStoragesByPath(path string) []driver.Driver { // for example, there are: /a/b,/a/c,/a/d/e,/a/b.balance1,/av // GetStorageVirtualFilesByPath(/a) => b,c,d func GetStorageVirtualFilesByPath(prefix string) []model.Obj { - return getStorageVirtualFilesByPath(prefix, func(_ driver.Driver, obj model.Obj) model.Obj { - return obj - }) + return getStorageVirtualFilesByPath(prefix, nil, "") } -func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, hideDetails ...bool) []model.Obj { - if utils.IsBool(hideDetails...) { - return GetStorageVirtualFilesByPath(prefix) +func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, hideDetails, refresh bool, filterByName string) []model.Obj { + if hideDetails { + return getStorageVirtualFilesByPath(prefix, nil, filterByName) } return getStorageVirtualFilesByPath(prefix, func(d driver.Driver, obj model.Obj) model.Obj { + if _, ok := obj.(*model.ObjStorageDetails); ok { + return obj + } ret := &model.ObjStorageDetails{ - Obj: obj, - StorageDetailsWithName: model.StorageDetailsWithName{ - StorageDetails: nil, - DriverName: d.Config().Name, - }, + Obj: obj, + StorageDetails: nil, } - timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - details, err := GetStorageDetails(timeoutCtx, d) - if err != nil { - if !errors.Is(err, errs.NotImplement) { - log.Errorf("failed get %s storage details: %+v", d.GetStorage().MountPath, err) + resultChan := make(chan *model.StorageDetails, 1) + go func(dri driver.Driver) { + details, err := GetStorageDetails(ctx, dri, refresh) + if err != nil { + if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) { + log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, err) + } } - return ret + resultChan <- details + }(d) + select { + case r := <-resultChan: + ret.StorageDetails = r + case <-time.After(time.Second): } - ret.StorageDetails = details return ret - }) + }, filterByName) } -func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver, model.Obj) model.Obj) []model.Obj { +func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver, model.Obj) model.Obj, filterByName string) []model.Obj { files := make([]model.Obj, 0) storages := storagesMap.Values() sort.Slice(storages, func(i, j int) bool { @@ -376,45 +385,60 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver return storages[i].GetStorage().Order < storages[j].GetStorage().Order }) - prefix = utils.FixAndCleanPath(prefix) + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } set := make(map[string]int) var wg sync.WaitGroup for _, v := range storages { - mountPath := utils.GetActualMountPath(v.GetStorage().MountPath) // Exclude prefix itself and non prefix - if len(prefix) >= len(mountPath) || !utils.IsSubPath(prefix, mountPath) { + p, found := strings.CutPrefix(utils.GetActualMountPath(v.GetStorage().MountPath), prefix) + if !found || p == "" { + continue + } + name, _, found := strings.Cut(p, "/") + if filterByName != "" && name != filterByName { continue } - names := strings.SplitN(strings.TrimPrefix(mountPath[len(prefix):], "/"), "/", 2) - idx, ok := set[names[0]] - if !ok { - set[names[0]] = len(files) - obj := &model.Object{ - Name: names[0], - Size: 0, - Modified: v.GetStorage().Modified, - IsFolder: true, + + if idx, ok := set[name]; ok { + if !found { + files[idx].(*model.Object).Mask = model.Locked | model.Virtual + if rootCallback != nil { + wg.Add(1) + go func() { + defer wg.Done() + files[idx] = rootCallback(v, files[idx]) + }() + } } - if len(names) == 1 { - idx = len(files) - files = append(files, obj) + continue + } + set[name] = len(files) + obj := &model.Object{ + Name: name, + Modified: v.GetStorage().Modified, + IsFolder: true, + } + if !found { + idx := len(files) + obj.Mask = model.Locked | model.Virtual + files = append(files, obj) + if rootCallback != nil { wg.Add(1) go func() { defer wg.Done() files[idx] = rootCallback(v, files[idx]) }() - } else { - files = append(files, obj) } - } else if len(names) == 1 { - wg.Add(1) - go func() { - defer wg.Done() - files[idx] = rootCallback(v, files[idx]) - }() + } else { + obj.Mask = model.ReadOnly | model.Virtual + files = append(files, obj) } } - wg.Wait() + if rootCallback != nil { + wg.Wait() + } return files } @@ -439,10 +463,28 @@ func GetBalancedStorage(path string) driver.Driver { } } -func GetStorageDetails(ctx context.Context, storage driver.Driver) (*model.StorageDetails, error) { +var detailsG singleflight.Group[*model.StorageDetails] + +func GetStorageDetails(ctx context.Context, storage driver.Driver, refresh ...bool) (*model.StorageDetails, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.WithMessagef(errs.StorageNotInit, "storage status: %s", storage.GetStorage().Status) + } wd, ok := storage.(driver.WithDetails) if !ok { return nil, errs.NotImplement } - return wd.GetDetails(ctx) + if !utils.IsBool(refresh...) { + if ret, ok := Cache.GetStorageDetails(storage); ok { + return ret, nil + } + } + details, err, _ := detailsG.Do(storage.GetStorage().MountPath, func() (*model.StorageDetails, error) { + ret, err := wd.GetDetails(ctx) + if err != nil { + return nil, err + } + Cache.SetStorageDetails(storage, ret) + return ret, nil + }) + return details, err } diff --git a/internal/op/user.go b/internal/op/user.go index 3f526625b..dc2a0ce3e 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -1,17 +1,14 @@ package op import ( - "time" - "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/singleflight" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/go-cache" + "github.com/pkg/errors" ) -var userCache = cache.NewMemCache(cache.WithShards[*model.User](2)) var userG singleflight.Group[*model.User] var guestUser *model.User var adminUser *model.User @@ -46,7 +43,7 @@ func GetUserByName(username string) (*model.User, error) { if username == "" { return nil, errs.EmptyUsername } - if user, ok := userCache.Get(username); ok { + if user, exists := Cache.GetUser(username); exists { return user, nil } user, err, _ := userG.Do(username, func() (*model.User, error) { @@ -54,7 +51,7 @@ func GetUserByName(username string) (*model.User, error) { if err != nil { return nil, err } - userCache.Set(username, _user, cache.WithEx[*model.User](time.Hour)) + Cache.SetUser(username, _user) return _user, nil }) return user, err @@ -81,7 +78,10 @@ func DeleteUserById(id uint) error { if old.IsAdmin() || old.IsGuest() { return errs.DeleteAdminOrGuest } - userCache.Del(old.Username) + Cache.DeleteUser(old.Username) + if err := DeleteSharingsByCreatorId(id); err != nil { + return errors.WithMessage(err, "failed to delete user's sharings") + } return db.DeleteUserById(id) } @@ -96,7 +96,7 @@ func UpdateUser(u *model.User) error { if u.IsGuest() { guestUser = nil } - userCache.Del(old.Username) + Cache.DeleteUser(old.Username) u.BasePath = utils.FixAndCleanPath(u.BasePath) return db.UpdateUser(u) } @@ -125,6 +125,6 @@ func DelUserCache(username string) error { if user.IsGuest() { guestUser = nil } - userCache.Del(username) + Cache.DeleteUser(username) return nil } diff --git a/internal/search/build.go b/internal/search/build.go index c5c74e094..bbed1f853 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -199,14 +199,13 @@ func Config(ctx context.Context) searcher.Config { return instance.Config() } -func Update(parent string, objs []model.Obj) { +func Update(ctx context.Context, parent string, objs []model.Obj) { if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running() { return } if isIgnorePath(parent) { return } - ctx := context.Background() // only update when index have built progress, err := Progress() if err != nil { @@ -216,6 +215,16 @@ func Update(parent string, objs []model.Obj) { if !progress.IsDone { return } + + // Use task queue for Meilisearch to avoid race conditions with async indexing + if msInstance, ok := instance.(interface { + EnqueueUpdate(parent string, objs []model.Obj) + }); ok { + // Enqueue task for async processing (diff calculation happens at consumption time) + msInstance.EnqueueUpdate(parent, objs) + return + } + nodes, err := instance.Get(ctx, parent) if err != nil { log.Errorf("update search index error while get nodes: %+v", err) @@ -242,27 +251,23 @@ func Update(parent string, objs []model.Obj) { } } } + // collect files and folders to add in batch + var toAddObjs []ObjWithParent for i := range objs { if toAdd.Contains(objs[i].GetName()) { - if !objs[i].IsDir() { - log.Debugf("add index: %s", path.Join(parent, objs[i].GetName())) - err = Index(ctx, parent, objs[i]) - if err != nil { - log.Errorf("update search index error while index new node: %+v", err) - return - } - } else { - // build index if it's a folder - dir := path.Join(parent, objs[i].GetName()) - err = BuildIndex(ctx, - []string{dir}, - conf.SlicesMap[conf.IgnorePaths], - setting.GetInt(conf.MaxIndexDepth, 20)-strings.Count(dir, "/"), false) - if err != nil { - log.Errorf("update search index error while build index: %+v", err) - return - } - } + log.Debugf("add index: %s", path.Join(parent, objs[i].GetName())) + toAddObjs = append(toAddObjs, ObjWithParent{ + Parent: parent, + Obj: objs[i], + }) + } + } + // batch index all files and folders at once + if len(toAddObjs) > 0 { + err = BatchIndex(ctx, toAddObjs) + if err != nil { + log.Errorf("update search index error while batch index new nodes: %+v", err) + return } } } diff --git a/internal/search/meilisearch/init.go b/internal/search/meilisearch/init.go index ae736fd97..3c379b790 100644 --- a/internal/search/meilisearch/init.go +++ b/internal/search/meilisearch/init.go @@ -91,6 +91,11 @@ func init() { return nil, err } } + + // Initialize and start task queue manager + m.taskQueue = NewTaskQueueManager(&m) + m.taskQueue.Start() + return &m, nil }) } diff --git a/internal/search/meilisearch/search.go b/internal/search/meilisearch/search.go index fbb8a3f85..75283228c 100644 --- a/internal/search/meilisearch/search.go +++ b/internal/search/meilisearch/search.go @@ -33,6 +33,7 @@ type Meilisearch struct { IndexUid string FilterableAttributes []string SearchableAttributes []string + taskQueue *TaskQueueManager } func (m *Meilisearch) Config() searcher.Config { @@ -82,14 +83,17 @@ func (m *Meilisearch) Index(ctx context.Context, node model.SearchNode) error { } func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { - documents, _ := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { + documents, err := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { parentHash := hashPath(src.Parent) nodePath := path.Join(src.Parent, src.Name) nodePathHash := hashPath(nodePath) parentPaths := utils.GetPathHierarchy(src.Parent) - parentPathHashes, _ := utils.SliceConvert(parentPaths, func(parentPath string) (string, error) { + parentPathHashes, err := utils.SliceConvert(parentPaths, func(parentPath string) (string, error) { return hashPath(parentPath), nil }) + if err != nil { + return nil, err + } return &searchDocument{ ID: nodePathHash, @@ -98,9 +102,12 @@ func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) SearchNode: src, }, nil }) + if err != nil { + return err + } // max up to 10,000 documents per batch to reduce error rate while uploading over the Internet - _, err := m.Client.Index(m.IndexUid).AddDocumentsInBatchesWithContext(ctx, documents, 10000) + _, err = m.Client.Index(m.IndexUid).AddDocumentsInBatchesWithContext(ctx, documents, 10000) if err != nil { return err } @@ -203,6 +210,9 @@ func (m *Meilisearch) Del(ctx context.Context, prefix string) error { } func (m *Meilisearch) Release(ctx context.Context) error { + if m.taskQueue != nil { + m.taskQueue.Stop() + } return nil } @@ -219,3 +229,115 @@ func (m *Meilisearch) getTaskStatus(ctx context.Context, taskUID int64) (meilise } return forTask.Status, nil } + +// EnqueueUpdate enqueues an update task to the task queue +func (m *Meilisearch) EnqueueUpdate(parent string, objs []model.Obj) { + if m.taskQueue == nil { + return + } + + m.taskQueue.Enqueue(parent, objs) +} + +// batchIndexWithTaskUID indexes documents and returns all taskUIDs +func (m *Meilisearch) batchIndexWithTaskUID(ctx context.Context, nodes []model.SearchNode) ([]int64, error) { + if len(nodes) == 0 { + return nil, nil + } + + documents, err := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { + parentHash := hashPath(src.Parent) + nodePath := path.Join(src.Parent, src.Name) + nodePathHash := hashPath(nodePath) + parentPaths := utils.GetPathHierarchy(src.Parent) + parentPathHashes, err := utils.SliceConvert(parentPaths, func(parentPath string) (string, error) { + return hashPath(parentPath), nil + }) + if err != nil { + return nil, err + } + + return &searchDocument{ + ID: nodePathHash, + ParentHash: parentHash, + ParentPathHashes: parentPathHashes, + SearchNode: src, + }, nil + }) + if err != nil { + return nil, err + } + + // max up to 10,000 documents per batch to reduce error rate while uploading over the Internet + tasks, err := m.Client.Index(m.IndexUid).AddDocumentsInBatchesWithContext(ctx, documents, 10000) + if err != nil { + return nil, err + } + + // Return all task UIDs + taskUIDs := make([]int64, 0, len(tasks)) + for _, task := range tasks { + taskUIDs = append(taskUIDs, task.TaskUID) + } + return taskUIDs, nil +} + +// batchDeleteWithTaskUID deletes documents and returns all taskUIDs +func (m *Meilisearch) batchDeleteWithTaskUID(ctx context.Context, paths []string) ([]int64, error) { + if len(paths) == 0 { + return nil, nil + } + + // Deduplicate paths first + pathSet := make(map[string]struct{}) + uniquePaths := make([]string, 0, len(paths)) + for _, p := range paths { + p = utils.FixAndCleanPath(p) + if _, exists := pathSet[p]; !exists { + pathSet[p] = struct{}{} + uniquePaths = append(uniquePaths, p) + } + } + + const batchSize = 100 // max paths per batch to avoid filter length limits + var taskUIDs []int64 + + // Process in batches to avoid filter length limits + for i := 0; i < len(uniquePaths); i += batchSize { + end := i + batchSize + if end > len(uniquePaths) { + end = len(uniquePaths) + } + batch := uniquePaths[i:end] + + // Build combined filter to delete all children in one request + // Format: parent_path_hashes = 'hash1' OR parent_path_hashes = 'hash2' OR ... + var filters []string + for _, p := range batch { + pathHash := hashPath(p) + filters = append(filters, fmt.Sprintf("parent_path_hashes = '%s'", pathHash)) + } + if len(filters) > 0 { + combinedFilter := strings.Join(filters, " OR ") + // Delete all children for all paths in one request + task, err := m.Client.Index(m.IndexUid).DeleteDocumentsByFilterWithContext(ctx, combinedFilter) + if err != nil { + return nil, err + } + taskUIDs = append(taskUIDs, task.TaskUID) + } + + // Convert paths to document IDs and batch delete + documentIDs := make([]string, 0, len(batch)) + for _, p := range batch { + documentIDs = append(documentIDs, hashPath(p)) + } + // Use batch delete API + task, err := m.Client.Index(m.IndexUid).DeleteDocumentsWithContext(ctx, documentIDs) + if err != nil { + return nil, err + } + taskUIDs = append(taskUIDs, task.TaskUID) + } + return taskUIDs, nil +} diff --git a/internal/search/meilisearch/task_queue.go b/internal/search/meilisearch/task_queue.go new file mode 100644 index 000000000..c5384e6e5 --- /dev/null +++ b/internal/search/meilisearch/task_queue.go @@ -0,0 +1,265 @@ +package meilisearch + +import ( + "context" + "path" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + mapset "github.com/deckarep/golang-set/v2" + log "github.com/sirupsen/logrus" +) + +// QueuedTask represents a task in the queue +type QueuedTask struct { + Parent string + Objs []model.Obj // current file system state + Depth int // path depth for sorting + EnqueueAt time.Time // enqueue time +} + +// TaskQueueManager manages the task queue for async index operations +type TaskQueueManager struct { + queue map[string]*QueuedTask // parent -> task + pendingTasks map[string][]int64 // parent -> all submitted taskUIDs + mu sync.RWMutex + ticker *time.Ticker + stopCh chan struct{} + m *Meilisearch + consuming atomic.Bool // flag to prevent concurrent consumption +} + +// NewTaskQueueManager creates a new task queue manager +func NewTaskQueueManager(m *Meilisearch) *TaskQueueManager { + return &TaskQueueManager{ + queue: make(map[string]*QueuedTask), + pendingTasks: make(map[string][]int64), + stopCh: make(chan struct{}), + m: m, + } +} + +// calculateDepth calculates the depth of a path +func calculateDepth(path string) int { + if path == "/" { + return 0 + } + return strings.Count(strings.Trim(path, "/"), "/") + 1 +} + +// Enqueue enqueues a task with current file system state +func (tqm *TaskQueueManager) Enqueue(parent string, objs []model.Obj) { + tqm.mu.Lock() + defer tqm.mu.Unlock() + + // deduplicate: overwrite existing task with the same parent + tqm.queue[parent] = &QueuedTask{ + Parent: parent, + Objs: objs, + Depth: calculateDepth(parent), + EnqueueAt: time.Now(), + } + log.Debugf("enqueued update task for parent: %s, depth: %d, objs: %d", parent, calculateDepth(parent), len(objs)) +} + +// Start starts the task queue consumer +func (tqm *TaskQueueManager) Start() { + tqm.ticker = time.NewTicker(30 * time.Second) + go func() { + for { + select { + case <-tqm.ticker.C: + tqm.consume() + case <-tqm.stopCh: + log.Info("task queue manager stopped") + return + } + } + }() + log.Info("task queue manager started, will consume every 30 seconds") +} + +// Stop stops the task queue consumer +func (tqm *TaskQueueManager) Stop() { + if tqm.ticker != nil { + tqm.ticker.Stop() + } + close(tqm.stopCh) +} + +// consume processes all tasks in the queue +func (tqm *TaskQueueManager) consume() { + // Prevent concurrent consumption + if !tqm.consuming.CompareAndSwap(false, true) { + log.Warn("previous consume still running, skip this round") + return + } + defer tqm.consuming.Store(false) + + tqm.mu.Lock() + + // extract all tasks + tasks := make([]*QueuedTask, 0, len(tqm.queue)) + for _, task := range tqm.queue { + tasks = append(tasks, task) + } + + // clear queue + tqm.queue = make(map[string]*QueuedTask) + + tqm.mu.Unlock() + + if len(tasks) == 0 { + return + } + + log.Infof("consuming task queue: %d tasks", len(tasks)) + + // sort tasks: shallow paths first, then by enqueue time + sort.Slice(tasks, func(i, j int) bool { + if tasks[i].Depth != tasks[j].Depth { + return tasks[i].Depth < tasks[j].Depth + } + return tasks[i].EnqueueAt.Before(tasks[j].EnqueueAt) + }) + + ctx := context.Background() + + // execute tasks in order + for _, task := range tasks { + // Check if there are pending tasks for this parent + tqm.mu.RLock() + pendingTaskUIDs, hasPending := tqm.pendingTasks[task.Parent] + tqm.mu.RUnlock() + + if hasPending && len(pendingTaskUIDs) > 0 { + // Check all pending task statuses + allCompleted := true + for _, taskUID := range pendingTaskUIDs { + taskStatus, err := tqm.m.getTaskStatus(ctx, taskUID) + if err != nil { + log.Errorf("failed to get task status for parent %s (taskUID: %d): %v", task.Parent, taskUID, err) + // If we can't get status, assume it's done and continue checking + continue + } + + // Check if task is still running + if taskStatus == "enqueued" || taskStatus == "processing" { + log.Warnf("skipping task for parent %s: previous task %d still %s", task.Parent, taskUID, taskStatus) + allCompleted = false + break // No need to check remaining tasks + } + } + + if !allCompleted { + // Re-enqueue the task if not already in queue (avoid overwriting newer snapshots) + tqm.mu.Lock() + if _, exists := tqm.queue[task.Parent]; !exists { + tqm.queue[task.Parent] = task + log.Debugf("re-enqueued skipped task for parent %s due to pending tasks", task.Parent) + } else { + log.Debugf("skipped task for parent %s not re-enqueued (newer task already in queue)", task.Parent) + } + tqm.mu.Unlock() + continue // Skip this task, some previous tasks are still running + } + + // All tasks are in terminal state, remove from pending + log.Debugf("all previous tasks for parent %s are completed, proceeding with new task", task.Parent) + tqm.mu.Lock() + delete(tqm.pendingTasks, task.Parent) + tqm.mu.Unlock() + } + + // Execute the task + tqm.executeTask(ctx, task) + } + + log.Infof("task queue consumption completed") +} + +// executeTask executes a single task +func (tqm *TaskQueueManager) executeTask(ctx context.Context, task *QueuedTask) { + parent := task.Parent + currentObjs := task.Objs + + // Query index to get old state + nodes, err := tqm.m.Get(ctx, parent) + if err != nil { + log.Errorf("failed to get indexed nodes for parent %s: %v", parent, err) + return + } + + // Calculate diff based on current index state + now := mapset.NewSet[string]() + for i := range currentObjs { + now.Add(currentObjs[i].GetName()) + } + old := mapset.NewSet[string]() + for i := range nodes { + old.Add(nodes[i].Name) + } + + toDelete := old.Difference(now) + toAdd := now.Difference(old) + + // Collect paths to delete + var pathsToDelete []string + for i := range nodes { + if toDelete.Contains(nodes[i].Name) && !op.HasStorage(path.Join(parent, nodes[i].Name)) { + pathsToDelete = append(pathsToDelete, path.Join(parent, nodes[i].Name)) + } + } + + var allTaskUIDs []int64 + + // Execute delete first + if len(pathsToDelete) > 0 { + log.Debugf("executing delete for parent %s: %d paths", parent, len(pathsToDelete)) + taskUIDs, err := tqm.m.batchDeleteWithTaskUID(ctx, pathsToDelete) + if err != nil { + log.Errorf("failed to batch delete for parent %s: %v", parent, err) + // Continue to add even if delete fails + } else { + allTaskUIDs = append(allTaskUIDs, taskUIDs...) + } + } + + // Collect objects to add + var nodesToAdd []model.SearchNode + for i := range currentObjs { + if toAdd.Contains(currentObjs[i].GetName()) { + log.Debugf("will add index: %s", path.Join(parent, currentObjs[i].GetName())) + nodesToAdd = append(nodesToAdd, model.SearchNode{ + Parent: parent, + Name: currentObjs[i].GetName(), + IsDir: currentObjs[i].IsDir(), + Size: currentObjs[i].GetSize(), + }) + } + } + + // Execute add + if len(nodesToAdd) > 0 { + log.Debugf("executing add for parent %s: %d nodes", parent, len(nodesToAdd)) + taskUIDs, err := tqm.m.batchIndexWithTaskUID(ctx, nodesToAdd) + if err != nil { + log.Errorf("failed to batch index for parent %s: %v", parent, err) + } else { + allTaskUIDs = append(allTaskUIDs, taskUIDs...) + } + } + + // Record all task UIDs for this parent + if len(allTaskUIDs) > 0 { + tqm.mu.Lock() + tqm.pendingTasks[parent] = allTaskUIDs + tqm.mu.Unlock() + log.Debugf("recorded %d taskUIDs for parent %s", len(allTaskUIDs), parent) + } +} diff --git a/internal/search/meilisearch/utils.go b/internal/search/meilisearch/utils.go index 9849c209e..3bed9b0f5 100644 --- a/internal/search/meilisearch/utils.go +++ b/internal/search/meilisearch/utils.go @@ -1,7 +1,6 @@ package meilisearch import ( - "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -13,16 +12,16 @@ func hashPath(path string) string { } func buildSearchDocumentFromResults(results map[string]any) *searchDocument { - searchNode := model.SearchNode{} - document := &searchDocument{ - SearchNode: searchNode, - } + document := &searchDocument{} // use assertion test to avoid panic - searchNode.Parent, _ = results["parent"].(string) - searchNode.Name, _ = results["name"].(string) - searchNode.IsDir, _ = results["is_dir"].(bool) - searchNode.Size, _ = results["size"].(int64) + document.SearchNode.Parent, _ = results["parent"].(string) + document.SearchNode.Name, _ = results["name"].(string) + document.SearchNode.IsDir, _ = results["is_dir"].(bool) + // JSON numbers are typically float64, not int64 + if size, ok := results["size"].(float64); ok { + document.SearchNode.Size = int64(size) + } document.ID, _ = results["id"].(string) document.ParentHash, _ = results["parent_hash"].(string) diff --git a/internal/setting/setting.go b/internal/setting/setting.go index 7ebd5e92a..32968ee74 100644 --- a/internal/setting/setting.go +++ b/internal/setting/setting.go @@ -28,3 +28,11 @@ func GetInt(key string, defaultVal int) int { func GetBool(key string) bool { return GetStr(key) == "true" || GetStr(key) == "1" } + +func GetFloat(key string, defaultVal float64) float64 { + f, err := strconv.ParseFloat(GetStr(key), 64) + if err != nil { + return defaultVal + } + return f +} diff --git a/internal/stream/limit.go b/internal/stream/limit.go index ee32840d5..8715c7365 100644 --- a/internal/stream/limit.go +++ b/internal/stream/limit.go @@ -7,7 +7,6 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" "golang.org/x/time/rate" ) @@ -42,17 +41,14 @@ type RateLimitReader struct { } func (r *RateLimitReader) Read(p []byte) (n int, err error) { - if r.Ctx != nil && utils.IsCanceled(r.Ctx) { - return 0, r.Ctx.Err() + if err = r.Ctx.Err(); err != nil { + return 0, err } n, err = r.Reader.Read(p) if err != nil { return } if r.Limiter != nil { - if r.Ctx == nil { - r.Ctx = context.Background() - } err = r.Limiter.WaitN(r.Ctx, n) } return @@ -72,17 +68,14 @@ type RateLimitWriter struct { } func (w *RateLimitWriter) Write(p []byte) (n int, err error) { - if w.Ctx != nil && utils.IsCanceled(w.Ctx) { - return 0, w.Ctx.Err() + if err = w.Ctx.Err(); err != nil { + return 0, err } n, err = w.Writer.Write(p) if err != nil { return } if w.Limiter != nil { - if w.Ctx == nil { - w.Ctx = context.Background() - } err = w.Limiter.WaitN(w.Ctx, n) } return @@ -102,34 +95,28 @@ type RateLimitFile struct { } func (r *RateLimitFile) Read(p []byte) (n int, err error) { - if r.Ctx != nil && utils.IsCanceled(r.Ctx) { - return 0, r.Ctx.Err() + if err = r.Ctx.Err(); err != nil { + return 0, err } n, err = r.File.Read(p) if err != nil { return } if r.Limiter != nil { - if r.Ctx == nil { - r.Ctx = context.Background() - } err = r.Limiter.WaitN(r.Ctx, n) } return } func (r *RateLimitFile) ReadAt(p []byte, off int64) (n int, err error) { - if r.Ctx != nil && utils.IsCanceled(r.Ctx) { - return 0, r.Ctx.Err() + if err = r.Ctx.Err(); err != nil { + return 0, err } n, err = r.File.ReadAt(p, off) if err != nil { return } if r.Limiter != nil { - if r.Ctx == nil { - r.Ctx = context.Background() - } err = r.Limiter.WaitN(r.Ctx, n) } return @@ -145,16 +132,16 @@ func (r *RateLimitFile) Close() error { type RateLimitRangeReaderFunc RangeReaderFunc func (f RateLimitRangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + if ServerDownloadLimit == nil { + return f(ctx, httpRange) + } rc, err := f(ctx, httpRange) if err != nil { return nil, err } - if ServerDownloadLimit != nil { - rc = &RateLimitReader{ - Ctx: ctx, - Reader: rc, - Limiter: ServerDownloadLimit, - } - } - return rc, nil + return &RateLimitReader{ + Ctx: ctx, + Reader: rc, + Limiter: ServerDownloadLimit, + }, nil } diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 8d2f504fd..4c8238100 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -10,7 +10,6 @@ import ( "sync" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/buffer" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" @@ -28,10 +27,8 @@ type FileStream struct { ForceStreamUpload bool Exist model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it utils.Closers - - tmpFile model.File //if present, tmpFile has full content, it will be deleted at last - peekBuff *buffer.Reader size int64 + peekBuff *buffer.Reader oriReader io.Reader // the original reader, used for caching } @@ -39,12 +36,6 @@ func (f *FileStream) GetSize() int64 { if f.size > 0 { return f.size } - if file, ok := f.tmpFile.(*os.File); ok { - info, err := file.Stat() - if err == nil { - return info.Size() - } - } return f.Obj.GetSize() } @@ -63,24 +54,10 @@ func (f *FileStream) IsForceStreamUpload() bool { func (f *FileStream) Close() error { if f.peekBuff != nil { f.peekBuff.Reset() + f.oriReader = nil f.peekBuff = nil } - - var err1, err2 error - err1 = f.Closers.Close() - if errors.Is(err1, os.ErrClosed) { - err1 = nil - } - if file, ok := f.tmpFile.(*os.File); ok { - err2 = os.RemoveAll(file.Name()) - if err2 != nil { - err2 = errs.NewErr(err2, "failed to remove tmpFile [%s]", file.Name()) - } else { - f.tmpFile = nil - } - } - - return errors.Join(err1, err2) + return f.Closers.Close() } func (f *FileStream) GetExist() model.Obj { @@ -94,28 +71,29 @@ func (f *FileStream) SetExist(obj model.Obj) { // It's not thread-safe! func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writer) (model.File, error) { if cache := f.GetFile(); cache != nil { + _, err := cache.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } if writer == nil { return cache, nil } - _, err := cache.Seek(0, io.SeekStart) - if err == nil { - reader := f.Reader - if up != nil { - cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) - *up = model.UpdateProgressWithRange(*up, 50, 100) - reader = &ReaderUpdatingProgress{ - Reader: &SimpleReaderWithSize{ - Reader: reader, - Size: f.GetSize(), - }, - UpdateProgress: cacheProgress, - } - } - _, err = utils.CopyWithBuffer(writer, reader) - if err == nil { - _, err = cache.Seek(0, io.SeekStart) + reader := f.Reader + if up != nil { + cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) + *up = model.UpdateProgressWithRange(*up, 50, 100) + reader = &ReaderUpdatingProgress{ + Reader: &SimpleReaderWithSize{ + Reader: reader, + Size: f.GetSize(), + }, + UpdateProgress: cacheProgress, } } + _, err = utils.CopyWithBuffer(writer, reader) + if err == nil { + _, err = cache.Seek(0, io.SeekStart) + } if err != nil { return nil, err } @@ -123,21 +101,20 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ } reader := f.Reader - if up != nil { - cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) - *up = model.UpdateProgressWithRange(*up, 50, 100) - reader = &ReaderUpdatingProgress{ - Reader: &SimpleReaderWithSize{ - Reader: reader, - Size: f.GetSize(), - }, - UpdateProgress: cacheProgress, + if f.peekBuff != nil { + f.peekBuff.Seek(0, io.SeekStart) + if writer != nil { + _, err := utils.CopyWithBuffer(writer, f.peekBuff) + if err != nil { + return nil, err + } + f.peekBuff.Seek(0, io.SeekStart) } + reader = f.oriReader } if writer != nil { reader = io.TeeReader(reader, writer) } - if f.GetSize() < 0 { if f.peekBuff == nil { f.peekBuff = &buffer.Reader{} @@ -174,7 +151,6 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ } } } - tmpF, err := utils.CreateTempFile(reader, 0) if err != nil { return nil, err @@ -191,22 +167,42 @@ func (f *FileStream) CacheFullAndWriter(up *model.UpdateProgress, writer io.Writ return peekF, nil } - f.Reader = reader + if up != nil { + cacheProgress := model.UpdateProgressWithRange(*up, 0, 50) + *up = model.UpdateProgressWithRange(*up, 50, 100) + size := f.GetSize() + if f.peekBuff != nil { + peekSize := f.peekBuff.Size() + cacheProgress(float64(peekSize) / float64(size) * 100) + size -= peekSize + } + reader = &ReaderUpdatingProgress{ + Reader: &SimpleReaderWithSize{ + Reader: reader, + Size: size, + }, + UpdateProgress: cacheProgress, + } + } + + if f.peekBuff != nil { + f.oriReader = reader + } else { + f.Reader = reader + } return f.cache(f.GetSize()) } func (f *FileStream) GetFile() model.File { - if f.tmpFile != nil { - return f.tmpFile - } if file, ok := f.Reader.(model.File); ok { return file } return nil } -// RangeRead have to cache all data first since only Reader is provided. -// It's not thread-safe! +// 从流读取指定范围的一块数据,并且不消耗流。 +// 当读取的边界超过内部设置大小后会缓存整个流。 +// 流未缓存时线程不完全 func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { if httpRange.Length < 0 || httpRange.Start+httpRange.Length > f.GetSize() { httpRange.Length = f.GetSize() - httpRange.Start @@ -215,12 +211,7 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { return io.NewSectionReader(f.GetFile(), httpRange.Start, httpRange.Length), nil } - size := httpRange.Start + httpRange.Length - if f.peekBuff != nil && size <= int64(f.peekBuff.Size()) { - return io.NewSectionReader(f.peekBuff, httpRange.Start, httpRange.Length), nil - } - - cache, err := f.cache(size) + cache, err := f.cache(httpRange.Start + httpRange.Length) if err != nil { return nil, err } @@ -232,14 +223,30 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { // 使用bytes.Buffer作为io.CopyBuffer的写入对象，CopyBuffer会调用Buffer.ReadFrom // 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大 +// 确保指定大小的数据被缓存 func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { if maxCacheSize > int64(conf.MaxBufferLimit) { - tmpF, err := utils.CreateTempFile(f.Reader, f.GetSize()) + size := f.GetSize() + reader := f.Reader + if f.peekBuff != nil { + size -= f.peekBuff.Size() + reader = f.oriReader + } + tmpF, err := utils.CreateTempFile(reader, size) if err != nil { return nil, err } - f.Add(tmpF) - f.tmpFile = tmpF + f.Add(utils.CloseFunc(func() error { + return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) + })) + if f.peekBuff != nil { + peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) + if err != nil { + return nil, err + } + f.Reader = peekF + return peekF, nil + } f.Reader = tmpF return tmpF, nil } @@ -247,8 +254,12 @@ func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { if f.peekBuff == nil { f.peekBuff = &buffer.Reader{} f.oriReader = f.Reader + f.Reader = io.MultiReader(f.peekBuff, f.oriReader) + } + bufSize := maxCacheSize - f.peekBuff.Size() + if bufSize <= 0 { + return f.peekBuff, nil } - bufSize := maxCacheSize - int64(f.peekBuff.Size()) var buf []byte if conf.MmapThreshold > 0 && bufSize >= int64(conf.MmapThreshold) { m, err := mmap.Alloc(int(bufSize)) @@ -267,37 +278,24 @@ func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", bufSize, n, err) } f.peekBuff.Append(buf) - if int64(f.peekBuff.Size()) >= f.GetSize() { + if f.peekBuff.Size() >= f.GetSize() { f.Reader = f.peekBuff - f.oriReader = nil - } else { - f.Reader = io.MultiReader(f.peekBuff, f.oriReader) } return f.peekBuff, nil } -func (f *FileStream) SetTmpFile(file model.File) { - f.AddIfCloser(file) - f.tmpFile = file - f.Reader = file -} - var _ model.FileStreamer = (*SeekableStream)(nil) var _ model.FileStreamer = (*FileStream)(nil) -//var _ seekableStream = (*FileStream)(nil) - -// for most internal stream, which is either RangeReadCloser or MFile -// Any functionality implemented based on SeekableStream should implement a Close method, -// whose only purpose is to close the SeekableStream object. If such functionality has -// additional resources that need to be closed, they should be added to the Closer property of -// the SeekableStream object and be closed together when the SeekableStream object is closed. type SeekableStream struct { *FileStream // should have one of belows to support rangeRead - rangeReadCloser model.RangeReadCloserIF + rangeReader model.RangeReaderIF } +// NewSeekableStream create a SeekableStream from FileStream and Link +// if FileStream.Reader is not nil, use it directly +// else create RangeReader from Link func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error) { if len(fs.Mimetype) == 0 { fs.Mimetype = utils.GetMimeType(fs.Obj.GetName()) @@ -317,30 +315,31 @@ func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error if err != nil { return nil, err } - rrc := &model.RangeReadCloser{ - RangeReader: rr, - } if _, ok := rr.(*model.FileRangeReader); ok { - fs.Reader, err = rrc.RangeRead(fs.Ctx, http_range.Range{Length: -1}) + var rc io.ReadCloser + rc, err = rr.RangeRead(fs.Ctx, http_range.Range{Length: -1}) if err != nil { return nil, err } + fs.Reader = rc + fs.Add(rc) } fs.size = size fs.Add(link) - fs.Add(rrc) - return &SeekableStream{FileStream: fs, rangeReadCloser: rrc}, nil + return &SeekableStream{FileStream: fs, rangeReader: rr}, nil } return nil, fmt.Errorf("illegal seekableStream") } -// RangeRead is not thread-safe, pls use it in single thread only. +// 如果使用缓存或者rangeReader读取指定范围的数据，是线程安全的 +// 其他特性继承自FileStream.RangeRead func (ss *SeekableStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { - if ss.GetFile() == nil && ss.rangeReadCloser != nil { - rc, err := ss.rangeReadCloser.RangeRead(ss.Ctx, httpRange) + if ss.GetFile() == nil && ss.rangeReader != nil { + rc, err := ss.rangeReader.RangeRead(ss.Ctx, httpRange) if err != nil { return nil, err } + ss.Add(rc) return rc, nil } return ss.FileStream.RangeRead(httpRange) @@ -356,13 +355,14 @@ func (ss *SeekableStream) Read(p []byte) (n int, err error) { func (ss *SeekableStream) generateReader() error { if ss.Reader == nil { - if ss.rangeReadCloser == nil { + if ss.rangeReader == nil { return fmt.Errorf("illegal seekableStream") } - rc, err := ss.rangeReadCloser.RangeRead(ss.Ctx, http_range.Range{Length: -1}) + rc, err := ss.rangeReader.RangeRead(ss.Ctx, http_range.Range{Length: -1}) if err != nil { return err } + ss.Add(rc) ss.Reader = rc } return nil @@ -456,7 +456,7 @@ func (r *headCache) Close() error { } func (r *RangeReadReadAtSeeker) InitHeadCache() { - if r.ss.GetFile() == nil && r.masterOff == 0 { + if r.masterOff == 0 { value, _ := r.readerMap.LoadAndDelete(int64(0)) r.headCache = &headCache{reader: value.(io.Reader)} r.ss.Closers.Add(r.headCache) @@ -464,12 +464,12 @@ func (r *RangeReadReadAtSeeker) InitHeadCache() { } func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (model.File, error) { - if ss.GetFile() != nil { - _, err := ss.GetFile().Seek(offset, io.SeekStart) + if cache := ss.GetFile(); cache != nil { + _, err := cache.Seek(offset, io.SeekStart) if err != nil { return nil, err } - return ss.GetFile(), nil + return cache, nil } r := &RangeReadReadAtSeeker{ ss: ss, @@ -479,10 +479,11 @@ func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (mode if offset < 0 || offset > ss.GetSize() { return nil, errors.New("offset out of range") } - _, err := r.getReaderAtOffset(offset) + reader, err := r.getReaderAtOffset(offset) if err != nil { return nil, err } + r.readerMap.Store(int64(offset), reader) } else { r.readerMap.Store(int64(offset), ss) } @@ -502,39 +503,41 @@ func NewMultiReaderAt(ss []*SeekableStream) (readerutil.SizeReaderAt, error) { } func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (io.Reader, error) { - var rr io.Reader - var cur int64 = -1 - r.readerMap.Range(func(key, value any) bool { - k := key.(int64) - if off == k { - cur = k - rr = value.(io.Reader) - return false + for { + var cur int64 = -1 + r.readerMap.Range(func(key, value any) bool { + k := key.(int64) + if off == k { + cur = k + return false + } + if off > k && off-k <= 4*utils.MB && k > cur { + cur = k + } + return true + }) + if cur < 0 { + break } - if off > k && off-k <= 4*utils.MB && (rr == nil || k < cur) { - rr = value.(io.Reader) - cur = k + v, ok := r.readerMap.LoadAndDelete(int64(cur)) + if !ok { + continue + } + rr := v.(io.Reader) + if off == int64(cur) { + // logrus.Debugf("getReaderAtOffset match_%d", off) + return rr, nil } - return true - }) - if cur >= 0 { - r.readerMap.Delete(int64(cur)) - } - if off == int64(cur) { - // logrus.Debugf("getReaderAtOffset match_%d", off) - return rr, nil - } - - if rr != nil { n, _ := utils.CopyWithBufferN(io.Discard, rr, off-cur) cur += n if cur == off { // logrus.Debugf("getReaderAtOffset old_%d", off) return rr, nil } + break } - // logrus.Debugf("getReaderAtOffset new_%d", off) + // logrus.Debugf("getReaderAtOffset new_%d", off) reader, err := r.ss.RangeRead(http_range.Range{Start: off, Length: -1}) if err != nil { return nil, err diff --git a/internal/stream/stream_test.go b/internal/stream/stream_test.go index 52c2abee0..9a81e7d41 100644 --- a/internal/stream/stream_test.go +++ b/internal/stream/stream_test.go @@ -7,13 +7,12 @@ import ( "io" "testing" - "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) func TestFileStream_RangeRead(t *testing.T) { - conf.MaxBufferLimit = 16 * 1024 * 1024 type args struct { httpRange http_range.Range } @@ -73,16 +72,38 @@ func TestFileStream_RangeRead(t *testing.T) { } }) } - t.Run("after", func(t *testing.T) { - if f.GetFile() == nil { - t.Error("not cached") - } - buf2 := make([]byte, len(buf)) - if _, err := io.ReadFull(f, buf2); err != nil { - t.Errorf("FileStream.Read() error = %v", err) - } - if !bytes.Equal(buf, buf2) { - t.Errorf("FileStream.Read() = %s, want %s", buf2, buf) - } - }) + if f.GetFile() == nil { + t.Error("not cached") + } + buf2 := make([]byte, len(buf)) + if _, err := io.ReadFull(f, buf2); err != nil { + t.Errorf("FileStream.Read() error = %v", err) + } + if !bytes.Equal(buf, buf2) { + t.Errorf("FileStream.Read() = %s, want %s", buf2, buf) + } +} + +func TestFileStream_With_PreHash(t *testing.T) { + buf := []byte("github.com/OpenListTeam/OpenList") + f := &FileStream{ + Obj: &model.Object{ + Size: int64(len(buf)), + }, + Reader: io.NopCloser(bytes.NewReader(buf)), + } + + const hashSize int64 = 20 + reader, _ := f.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + preHash, _ := utils.HashReader(utils.SHA1, reader) + if preHash == "" { + t.Error("preHash is empty") + } + tmpF, fullHash, _ := CacheFullAndHash(f, nil, utils.SHA1) + fmt.Println(fullHash) + fileFullHash, _ := utils.HashFile(utils.SHA1, tmpF) + fmt.Println(fileFullHash) + if fullHash != fileFullHash { + t.Errorf("fullHash and fileFullHash should match: fullHash=%s fileFullHash=%s", fullHash, fileFullHash) + } } diff --git a/internal/stream/util.go b/internal/stream/util.go index cba29e789..6aa3dda5d 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -11,6 +11,7 @@ import ( "os" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" @@ -27,47 +28,61 @@ func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Ran } func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { - if link.MFile != nil { - return GetRangeReaderFromMFile(size, link.MFile), nil + if link.RangeReader != nil { + if link.Concurrency < 1 && link.PartSize < 1 { + return link.RangeReader, nil + } + down := net.NewDownloader(func(d *net.Downloader) { + d.Concurrency = link.Concurrency + d.PartSize = link.PartSize + d.HttpClient = net.GetRangeReaderHttpRequestFunc(link.RangeReader) + }) + rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return down.Download(ctx, &net.HttpRequestParams{ + Range: httpRange, + Size: size, + }) + } + // RangeReader只能在驱动限速 + return RangeReaderFunc(rangeReader), nil + } + + if len(link.URL) == 0 { + return nil, errors.New("invalid link: must have at least one of URL or RangeReader") } + if link.Concurrency > 0 || link.PartSize > 0 { down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize - }) - var rangeReader RangeReaderFunc = func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { - var req *net.HttpRequestParams - if link.RangeReader != nil { - req = &net.HttpRequestParams{ - Range: httpRange, - Size: size, + d.HttpClient = func(ctx context.Context, params *net.HttpRequestParams) (*http.Response, error) { + if ServerDownloadLimit == nil { + return net.DefaultHttpRequestFunc(ctx, params) } - } else { - requestHeader, _ := ctx.Value(conf.RequestHeaderKey).(http.Header) - header := net.ProcessHeader(requestHeader, link.Header) - req = &net.HttpRequestParams{ - Range: httpRange, - Size: size, - URL: link.URL, - HeaderRef: header, + resp, err := net.DefaultHttpRequestFunc(ctx, params) + if err == nil && resp.Body != nil { + resp.Body = &RateLimitReader{ + Ctx: ctx, + Reader: resp.Body, + Limiter: ServerDownloadLimit, + } } + return resp, err } - return down.Download(ctx, req) - } - if link.RangeReader != nil { - down.HttpClient = net.GetRangeReaderHttpRequestFunc(link.RangeReader) - return rangeReader, nil + }) + rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + requestHeader, _ := ctx.Value(conf.RequestHeaderKey).(http.Header) + header := net.ProcessHeader(requestHeader, link.Header) + return down.Download(ctx, &net.HttpRequestParams{ + Range: httpRange, + Size: size, + URL: link.URL, + HeaderRef: header, + }) } - return RateLimitRangeReaderFunc(rangeReader), nil - } - - if link.RangeReader != nil { - return link.RangeReader, nil + return RangeReaderFunc(rangeReader), nil } - if len(link.URL) == 0 { - return nil, errors.New("invalid link: must have at least one of MFile, URL, or RangeReader") - } rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { if httpRange.Length < 0 || httpRange.Start+httpRange.Length > size { httpRange.Length = size - httpRange.Start @@ -78,12 +93,20 @@ func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, response, err := net.RequestHttp(ctx, "GET", header, link.URL) if err != nil { - if _, ok := errors.Unwrap(err).(net.HttpStatusCodeError); ok { + if _, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { return nil, err } return nil, fmt.Errorf("http request failure, err:%w", err) } - if httpRange.Start == 0 && (httpRange.Length == -1 || httpRange.Length == size) || response.StatusCode == http.StatusPartialContent || + if ServerDownloadLimit != nil { + response.Body = &RateLimitReader{ + Ctx: ctx, + Reader: response.Body, + Limiter: ServerDownloadLimit, + } + } + if httpRange.Start == 0 && httpRange.Length == size || + response.StatusCode == http.StatusPartialContent || checkContentRange(&response.Header, httpRange.Start) { return response.Body, nil } else if response.StatusCode == http.StatusOK { @@ -96,11 +119,10 @@ func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, } return response.Body, nil } - return RateLimitRangeReaderFunc(rangeReader), nil + return RangeReaderFunc(rangeReader), nil } -// RangeReaderIF.RangeRead返回的io.ReadCloser保留file的签名。 -func GetRangeReaderFromMFile(size int64, file model.File) model.RangeReaderIF { +func GetRangeReaderFromMFile(size int64, file model.File) *model.FileRangeReader { return &model.FileRangeReader{ RangeReaderIF: RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length @@ -172,7 +194,7 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode return nil, err } - if f.Truncate((file.GetSize()+int64(maxBufferSize-1))/int64(maxBufferSize)*int64(maxBufferSize)) != nil { + if f.Truncate(file.GetSize()) != nil { // fallback to full cache _, _ = f.Close(), os.Remove(f.Name()) cache, err := file.CacheFullAndWriter(up, nil) @@ -182,11 +204,11 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode return &cachedSectionReader{cache}, nil } - ss := &fileSectionReader{Reader: file, temp: f} + ss := &fileSectionReader{file: file, temp: f} ss.bufPool = &pool.Pool[*offsetWriterWithBase]{ New: func() *offsetWriterWithBase { - base := ss.fileOff - ss.fileOff += int64(maxBufferSize) + base := ss.tempOffset + ss.tempOffset += int64(maxBufferSize) return &offsetWriterWithBase{io.NewOffsetWriter(ss.temp, base), base} }, } @@ -203,7 +225,7 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode New: func() []byte { buf, err := mmap.Alloc(maxBufferSize) if err == nil { - ss.file.Add(utils.CloseFunc(func() error { + file.Add(utils.CloseFunc(func() error { return mmap.Free(buf) })) } else { @@ -240,11 +262,11 @@ func (s *cachedSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker func (*cachedSectionReader) FreeSectionReader(sr io.ReadSeeker) {} type fileSectionReader struct { - io.Reader - off int64 - temp *os.File - fileOff int64 - bufPool *pool.Pool[*offsetWriterWithBase] + file model.FileStreamer + fileOffset int64 + temp *os.File + tempOffset int64 + bufPool *pool.Pool[*offsetWriterWithBase] } type offsetWriterWithBase struct { @@ -254,14 +276,14 @@ type offsetWriterWithBase struct { // 线程不安全 func (ss *fileSectionReader) DiscardSection(off int64, length int64) error { - if off != ss.off { - return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.off) + if off != ss.fileOffset { + return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } - _, err := utils.CopyWithBufferN(io.Discard, ss.Reader, length) + n, err := utils.CopyWithBufferN(io.Discard, ss.file, length) + ss.fileOffset += n if err != nil { - return fmt.Errorf("failed to skip data: (expect =%d) %w", length, err) + return fmt.Errorf("failed to skip data: (expect =%d, actual =%d) %w", length, n, err) } - ss.off += length return nil } @@ -270,17 +292,18 @@ type fileBufferSectionReader struct { fileBuf *offsetWriterWithBase } +// 线程不安全 func (ss *fileSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { - if off != ss.off { - return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.off) + if off != ss.fileOffset { + return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } fileBuf := ss.bufPool.Get() _, _ = fileBuf.Seek(0, io.SeekStart) - n, err := utils.CopyWithBufferN(fileBuf, ss.Reader, length) + n, err := utils.CopyWithBufferN(fileBuf, ss.file, length) + ss.fileOffset += n if err != nil { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) } - ss.off += length return &fileBufferSectionReader{io.NewSectionReader(ss.temp, fileBuf.base, length), fileBuf}, nil } @@ -293,21 +316,21 @@ func (ss *fileSectionReader) FreeSectionReader(rs io.ReadSeeker) { } type directSectionReader struct { - file model.FileStreamer - off int64 - bufPool *pool.Pool[[]byte] + file model.FileStreamer + fileOffset int64 + bufPool *pool.Pool[[]byte] } // 线程不安全 func (ss *directSectionReader) DiscardSection(off int64, length int64) error { - if off != ss.off { - return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.off) + if off != ss.fileOffset { + return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } - _, err := utils.CopyWithBufferN(io.Discard, ss.file, length) + n, err := utils.CopyWithBufferN(io.Discard, ss.file, length) + ss.fileOffset += n if err != nil { - return fmt.Errorf("failed to skip data: (expect =%d) %w", length, err) + return fmt.Errorf("failed to skip data: (expect =%d, actual =%d) %w", length, n, err) } - ss.off += length return nil } @@ -318,16 +341,16 @@ type bufferSectionReader struct { // 线程不安全 func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { - if off != ss.off { - return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.off) + if off != ss.fileOffset { + return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } tempBuf := ss.bufPool.Get() buf := tempBuf[:length] n, err := io.ReadFull(ss.file, buf) + ss.fileOffset += int64(n) if int64(n) != length { return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) } - ss.off += int64(n) return &bufferSectionReader{bytes.NewReader(buf), buf}, nil } func (ss *directSectionReader) FreeSectionReader(rs io.ReadSeeker) { diff --git a/internal/task/base.go b/internal/task/base.go index 8976ed90b..f969e6c27 100644 --- a/internal/task/base.go +++ b/internal/task/base.go @@ -14,7 +14,7 @@ type TaskExtension struct { Creator *model.User startTime *time.Time endTime *time.Time - totalBytes int64 + TotalBytes int64 ApiUrl string } @@ -58,25 +58,25 @@ func (t *TaskExtension) ClearEndTime() { } func (t *TaskExtension) SetTotalBytes(totalBytes int64) { - t.totalBytes = totalBytes + t.TotalBytes = totalBytes } func (t *TaskExtension) GetTotalBytes() int64 { - return t.totalBytes + return t.TotalBytes } -func (t *TaskExtension) ReinitCtx() error { +func (t *TaskExtension) SetRetry(retry int, maxRetry int) { + t.Base.SetRetry(retry, maxRetry) + if retry > 0 || !conf.Conf.Tasks.AllowRetryCanceled || t.Ctx() == nil { + return + } select { case <-t.Ctx().Done(): - if !conf.Conf.Tasks.AllowRetryCanceled { - return t.Ctx().Err() - } ctx, cancel := context.WithCancel(context.Background()) t.SetCtx(ctx) t.SetCancelFunc(cancel) default: } - return nil } type TaskExtensionInfo interface { diff --git a/internal/task_group/group.go b/internal/task_group/group.go index 3cf78c8a2..edd51fe74 100644 --- a/internal/task_group/group.go +++ b/internal/task_group/group.go @@ -1,12 +1,13 @@ package task_group import ( + "context" "sync" "github.com/sirupsen/logrus" ) -type OnCompletionFunc func(groupID string, payloads ...any) +type OnCompletionFunc func(ctx context.Context, groupID string, payloads ...any) type TaskGroupCoordinator struct { name string mu sync.Mutex @@ -53,7 +54,7 @@ func (tgc *TaskGroupCoordinator) AppendPayload(groupID string, payload any) { tgc.groupPayloads[groupID] = append(tgc.groupPayloads[groupID], payload) } -func (tgc *TaskGroupCoordinator) Done(groupID string, success bool) { +func (tgc *TaskGroupCoordinator) Done(ctx context.Context, groupID string, success bool) { tgc.mu.Lock() defer tgc.mu.Unlock() state, ok := tgc.groupStates[groupID] @@ -71,7 +72,7 @@ func (tgc *TaskGroupCoordinator) Done(groupID string, success bool) { if tgc.onCompletion != nil && state.hasSuccess { logrus.Debugf("OnCompletion:%s", groupID) tgc.mu.Unlock() - tgc.onCompletion(groupID, payloads...) + tgc.onCompletion(ctx, groupID, payloads...) tgc.mu.Lock() } return diff --git a/internal/task_group/transfer.go b/internal/task_group/transfer.go index 3cef57865..0e9661b45 100644 --- a/internal/task_group/transfer.go +++ b/internal/task_group/transfer.go @@ -5,46 +5,66 @@ import ( "fmt" "path" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) type SrcPathToRemove string // ActualPath -type DstPathToRefresh string +type DstPathToHook string -func RefreshAndRemove(dstPath string, payloads ...any) { +func HookAndRemove(ctx context.Context, dstPath string, payloads ...any) { dstStorage, dstActualPath, err := op.GetStorageAndActualPath(dstPath) if err != nil { log.Error(errors.WithMessage(err, "failed get dst storage")) return } - _, dstNeedRefresh := dstStorage.(driver.Put) - dstNeedRefresh = dstNeedRefresh && !dstStorage.Config().NoCache - if dstNeedRefresh { - op.DeleteCache(dstStorage, dstActualPath) + dstNeedHandleHook := setting.GetBool(conf.HandleHookAfterWriting) + dstHandleHookLimit := setting.GetFloat(conf.HandleHookRateLimit, .0) + var listLimiter *rate.Limiter + if dstNeedHandleHook && dstHandleHookLimit > .0 { + listLimiter = rate.NewLimiter(rate.Limit(dstHandleHookLimit), 1) + } + hookedPaths := make(map[string]struct{}) + handleHook := func(actualPath string) { + if _, ok := hookedPaths[actualPath]; ok { + return + } + if listLimiter != nil { + _ = listLimiter.Wait(ctx) + } + files, e := op.List(ctx, dstStorage, actualPath, model.ListArgs{SkipHook: true}) + if e != nil { + log.Errorf("failed handle objs update hook: %v", e) + } else { + op.HandleObjsUpdateHook(ctx, utils.GetFullPath(dstStorage.GetStorage().MountPath, actualPath), files) + hookedPaths[actualPath] = struct{}{} + } + } + if dstNeedHandleHook { + handleHook(dstActualPath) } - var ctx context.Context for _, payload := range payloads { switch p := payload.(type) { - case DstPathToRefresh: - if dstNeedRefresh { - op.DeleteCache(dstStorage, string(p)) + case DstPathToHook: + if dstNeedHandleHook { + handleHook(string(p)) } case SrcPathToRemove: - if ctx == nil { - ctx = context.Background() - } srcStorage, srcActualPath, err := op.GetStorageAndActualPath(string(p)) if err != nil { log.Error(errors.WithMessage(err, "failed get src storage")) continue } - err = verifyAndRemove(ctx, srcStorage, dstStorage, srcActualPath, dstActualPath, dstNeedRefresh) + err = verifyAndRemove(ctx, srcStorage, dstStorage, srcActualPath, dstActualPath) if err != nil { log.Error(err) } @@ -52,14 +72,14 @@ func RefreshAndRemove(dstPath string, payloads ...any) { } } -func verifyAndRemove(ctx context.Context, srcStorage, dstStorage driver.Driver, srcPath, dstPath string, refresh bool) error { - srcObj, err := op.Get(ctx, srcStorage, srcPath) +func verifyAndRemove(ctx context.Context, srcStorage, dstStorage driver.Driver, srcPath, dstPath string) error { + srcObj, err := op.GetUnwrap(ctx, srcStorage, srcPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", path.Join(srcStorage.GetStorage().MountPath, srcPath)) } dstObjPath := path.Join(dstPath, srcObj.GetName()) - dstObj, err := op.Get(ctx, dstStorage, dstObjPath) + dstObj, err := op.GetUnwrap(ctx, dstStorage, dstObjPath) if err != nil { return errors.WithMessagef(err, "failed get dst [%s] file", path.Join(dstStorage.GetStorage().MountPath, dstObjPath)) } @@ -78,13 +98,10 @@ func verifyAndRemove(ctx context.Context, srcStorage, dstStorage driver.Driver, return errors.WithMessagef(err, "failed list src [%s] objs", path.Join(srcStorage.GetStorage().MountPath, srcPath)) } - if refresh { - op.DeleteCache(dstStorage, dstObjPath) - } hasErr := false for _, obj := range srcObjs { srcSubPath := path.Join(srcPath, obj.GetName()) - err := verifyAndRemove(ctx, srcStorage, dstStorage, srcSubPath, dstObjPath, refresh) + err := verifyAndRemove(ctx, srcStorage, dstStorage, srcSubPath, dstObjPath) if err != nil { log.Error(err) hasErr = true @@ -100,4 +117,4 @@ func verifyAndRemove(ctx context.Context, srcStorage, dstStorage driver.Driver, return nil } -var TransferCoordinator *TaskGroupCoordinator = NewTaskGroupCoordinator("RefreshAndRemove", RefreshAndRemove) +var TransferCoordinator *TaskGroupCoordinator = NewTaskGroupCoordinator("HookAndRemove", HookAndRemove) diff --git a/pkg/aria2/rpc/call.go b/pkg/aria2/rpc/call.go index a2af84617..e8f7d4d6e 100644 --- a/pkg/aria2/rpc/call.go +++ b/pkg/aria2/rpc/call.go @@ -69,14 +69,11 @@ func (h *httpCaller) setNotifier(ctx context.Context, u url.URL, notifier Notifi go func() { defer h.wg.Done() defer conn.Close() - select { - case <-ctx.Done(): - conn.SetWriteDeadline(time.Now().Add(time.Second)) - if err := conn.WriteMessage(websocket.CloseMessage, - websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")); err != nil { - log.Printf("sending websocket close message: %v", err) - } - return + <-ctx.Done() + conn.SetWriteDeadline(time.Now().Add(time.Second)) + if err := conn.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")); err != nil { + log.Printf("sending websocket close message: %v", err) } }() h.wg.Add(1) @@ -120,7 +117,7 @@ func (h *httpCaller) setNotifier(ctx context.Context, u url.URL, notifier Notifi return } -func (h httpCaller) Call(method string, params, reply interface{}) (err error) { +func (h *httpCaller) Call(method string, params, reply interface{}) (err error) { payload, err := EncodeClientRequest(method, params) if err != nil { return @@ -236,7 +233,7 @@ func (w *websocketCaller) Close() (err error) { return } -func (w websocketCaller) Call(method string, params, reply interface{}) (err error) { +func (w *websocketCaller) Call(method string, params, reply interface{}) (err error) { ctx, cancel := context.WithTimeout(context.Background(), w.timeout) defer cancel() select { @@ -251,11 +248,9 @@ func (w websocketCaller) Call(method string, params, reply interface{}) (err err return errors.New("sending channel blocking") } - select { - case <-ctx.Done(): - if err := ctx.Err(); err == context.DeadlineExceeded { - return err - } + <-ctx.Done() + if err := ctx.Err(); err == context.DeadlineExceeded { + return err } return } diff --git a/pkg/errgroup/errgroup.go b/pkg/errgroup/errgroup.go index d3c4feaff..7d72a1d8e 100644 --- a/pkg/errgroup/errgroup.go +++ b/pkg/errgroup/errgroup.go @@ -29,6 +29,7 @@ func NewGroupWithContext(ctx context.Context, limit int, retryOpts ...retry.Opti } // OrderedGroup +// 使得Lifecycle.Before是有序且线程安全 func NewOrderedGroupWithContext(ctx context.Context, limit int, retryOpts ...retry.Option) (*Group, context.Context) { group, ctx := NewGroupWithContext(ctx, limit, retryOpts...) group.startChan = make(chan token, 1) @@ -53,11 +54,11 @@ func (g *Group) Go(do func(ctx context.Context) error) { } type Lifecycle struct { - // Before在OrderedGroup是线程安全的。 + // Before在OrderedGroup是有序且线程安全的 // 只会被调用一次 - Before func(ctx context.Context) error + Before func(ctx context.Context) (err error) // 如果Before返回err就不调用Do - Do func(ctx context.Context) error + Do func(ctx context.Context) (err error) // 最后调用一次After After func(err error) } diff --git a/pkg/gowebdav/errors.go b/pkg/gowebdav/errors.go index bbf1e929e..de03e7b96 100644 --- a/pkg/gowebdav/errors.go +++ b/pkg/gowebdav/errors.go @@ -1,8 +1,8 @@ package gowebdav import ( - "fmt" "os" + "strconv" ) // StatusError implements error and wraps @@ -12,7 +12,7 @@ type StatusError struct { } func (se StatusError) Error() string { - return fmt.Sprintf("%d", se.Status) + return strconv.Itoa(se.Status) } // IsErrCode returns true if the given error diff --git a/pkg/singleflight/singleflight.go b/pkg/singleflight/singleflight.go index 3555d5bd0..483834785 100644 --- a/pkg/singleflight/singleflight.go +++ b/pkg/singleflight/singleflight.go @@ -73,9 +73,6 @@ type call[T any] struct { type Group[T any] struct { mu sync.Mutex // protects m m map[string]*call[T] // lazily initialized - - // Won't remember error - Remember bool } // Result holds the results of Do, so they can be passed @@ -159,7 +156,7 @@ func (g *Group[T]) doCall(c *call[T], key string, fn func() (T, error)) { g.mu.Lock() defer g.mu.Unlock() c.wg.Done() - if (!g.Remember || c.err != nil) && g.m[key] == c { + if g.m[key] == c { delete(g.m, key) } diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 1f819edf8..4fa58d397 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -185,3 +185,20 @@ const ( GB TB ) + +// IsSystemFile checks if a filename is a common system file that should be ignored +// Returns true for files like .DS_Store, desktop.ini, Thumbs.db, and Apple Double files (._*) +func IsSystemFile(filename string) bool { + // Common system files + switch filename { + case ".DS_Store", "desktop.ini", "Thumbs.db", "@eaDir": + return true + } + + // Apple Double files (._*) + if strings.HasPrefix(filename, "._") { + return true + } + + return false +} diff --git a/pkg/utils/file_test.go b/pkg/utils/file_test.go new file mode 100644 index 000000000..bcfff7934 --- /dev/null +++ b/pkg/utils/file_test.go @@ -0,0 +1,43 @@ +package utils + +import ( + "testing" +) + +func TestIsSystemFile(t *testing.T) { + testCases := []struct { + filename string + expected bool + }{ + // System files that should be filtered + {".DS_Store", true}, + {"desktop.ini", true}, + {"Thumbs.db", true}, + {"._test.txt", true}, + {"._", true}, + {"._somefile", true}, + {"._folder_name", true}, + {"@eaDir", true}, + + // Regular files that should not be filtered + {"test.txt", false}, + {"file.pdf", false}, + {"document.docx", false}, + {".gitignore", false}, + {".env", false}, + {"_underscore.txt", false}, + {"normal_file.txt", false}, + {"", false}, + {".hidden", false}, + {"..special", false}, + } + + for _, tc := range testCases { + t.Run(tc.filename, func(t *testing.T) { + result := IsSystemFile(tc.filename) + if result != tc.expected { + t.Errorf("IsSystemFile(%q) = %v, want %v", tc.filename, result, tc.expected) + } + }) + } +} diff --git a/pkg/utils/io.go b/pkg/utils/io.go index ff52f843e..f398161f6 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -187,51 +187,38 @@ func NewClosers(c ...io.Closer) Closers { return Closers(c) } -type SyncClosersIF interface { - ClosersIF - AcquireReference() bool -} - type SyncClosers struct { closers []io.Closer ref int32 } -var _ SyncClosersIF = (*SyncClosers)(nil) - +// if closed, return false func (c *SyncClosers) AcquireReference() bool { - for { - ref := atomic.LoadInt32(&c.ref) - if ref < 0 { - return false - } - newRef := ref + 1 - if atomic.CompareAndSwapInt32(&c.ref, ref, newRef) { - // log.Debugf("AcquireReference %p: %d", c, newRef) - return true - } + ref := atomic.AddInt32(&c.ref, 1) + if ref > 0 { + // log.Debugf("AcquireReference %p: %d", c, ref) + return true } + atomic.StoreInt32(&c.ref, closersClosed) + return false } -const closersClosed = math.MinInt16 +const closersClosed = math.MinInt32 func (c *SyncClosers) Close() error { - ref := atomic.AddInt32(&c.ref, -1) - if ref > 0 { - // log.Debugf("ReleaseReference %p: %d", c, ref) - return nil - } - - if ref < -1 { - atomic.StoreInt32(&c.ref, closersClosed) - return nil - } - - // Attempt to acquire FinalClose permission. - // At this point, ref must be 0 or -1. We try to atomically change it to the closersClosed state. - // Only the first successful goroutine gets the cleanup permission. - if !atomic.CompareAndSwapInt32(&c.ref, ref, closersClosed) { - return nil + for { + ref := atomic.LoadInt32(&c.ref) + if ref < 0 { + return nil + } + if ref > 1 { + if atomic.CompareAndSwapInt32(&c.ref, ref, ref-1) { + // log.Debugf("ReleaseReference %p: %d", c, ref) + return nil + } + } else if atomic.CompareAndSwapInt32(&c.ref, ref, closersClosed) { + break + } } // log.Debugf("FinalClose %p", c) @@ -264,6 +251,16 @@ func (c *SyncClosers) AddIfCloser(a any) { } } +var _ ClosersIF = (*SyncClosers)(nil) + +// 实现cache.Expirable接口 +func (c *SyncClosers) Expired() bool { + return atomic.LoadInt32(&c.ref) < 0 +} +func (c *SyncClosers) Length() int { + return len(c.closers) +} + func NewSyncClosers(c ...io.Closer) SyncClosers { return SyncClosers{closers: c} } diff --git a/pkg/utils/path.go b/pkg/utils/path.go index 2a104ff51..ec75501e2 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -44,11 +44,15 @@ func IsSubPath(path string, subPath string) bool { } func Ext(path string) string { + return strings.ToLower(SourceExt(path)) +} + +func SourceExt(path string) string { ext := stdpath.Ext(path) if len(ext) > 0 && ext[0] == '.' { ext = ext[1:] } - return strings.ToLower(ext) + return ext } func EncodePath(path string, all ...bool) string { diff --git a/server/common/ldap.go b/server/common/ldap.go new file mode 100644 index 000000000..9dc3b93fd --- /dev/null +++ b/server/common/ldap.go @@ -0,0 +1,107 @@ +package common + +import ( + "crypto/tls" + "fmt" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/ldap.v3" +) + +var ErrFailedLdapAuth = errors.New("failed to auth") + +func HandleLdapLogin(username, password string) error { + // Auth start + ldapServer := setting.GetStr(conf.LdapServer) + skipTlsVerify := setting.GetBool(conf.LdapSkipTlsVerify) + ldapManagerDN := setting.GetStr(conf.LdapManagerDN) + ldapManagerPassword := setting.GetStr(conf.LdapManagerPassword) + ldapUserSearchBase := setting.GetStr(conf.LdapUserSearchBase) + ldapUserSearchFilter := setting.GetStr(conf.LdapUserSearchFilter) // (uid=%s) + + // Connect to LdapServer + l, err := dial(ldapServer, skipTlsVerify) + if err != nil { + return errors.WithMessagef(err, "failed to connect to LDAP") + } + defer l.Close() + + // First bind with a read only user + if ldapManagerDN != "" && ldapManagerPassword != "" { + err = l.Bind(ldapManagerDN, ldapManagerPassword) + if err != nil { + return errors.WithMessagef(err, "failed to bind to LDAP") + } + } + + // Search for the given username + searchRequest := ldap.NewSearchRequest( + ldapUserSearchBase, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + fmt.Sprintf(ldapUserSearchFilter, ldap.EscapeFilter(username)), + []string{"dn"}, + nil, + ) + sr, err := l.Search(searchRequest) + if err != nil { + return errors.WithMessagef(err, "failed login ldap: LDAP search failed") + } + if len(sr.Entries) != 1 { + return errors.New("failed login ldap: user does not exist or too many entries returned") + } + userDN := sr.Entries[0].DN + + // Bind as the user to verify their password + err = l.Bind(userDN, password) + if err != nil { + return errors.WithMessagef(ErrFailedLdapAuth, "%v", err) + } + log.Infof("LDAP auth successful for %s", username) + // Auth finished + return nil +} + +func LdapRegister(username string) (*model.User, error) { + if username == "" { + return nil, errors.New("cannot get username from ldap provider") + } + user := &model.User{ + Username: username, + Password: "", + Authn: "[]", + Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), + BasePath: setting.GetStr(conf.LdapDefaultDir), + Role: 0, + Disabled: false, + AllowLdap: true, + } + user.SetPassword(random.String(16)) + if err := op.CreateUser(user); err != nil { + return nil, err + } + return user, nil +} + +func dial(ldapServer string, skipTlsVerify ...bool) (*ldap.Conn, error) { + tlsEnabled := false + if strings.HasPrefix(ldapServer, "ldaps://") { + tlsEnabled = true + ldapServer = strings.TrimPrefix(ldapServer, "ldaps://") + } else if strings.HasPrefix(ldapServer, "ldap://") { + ldapServer = strings.TrimPrefix(ldapServer, "ldap://") + } + + if tlsEnabled { + return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: utils.IsBool(skipTlsVerify...)}) + } else { + return ldap.Dial("tcp", ldapServer) + } +} diff --git a/server/common/proxy.go b/server/common/proxy.go index d352582eb..c7c975d25 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -18,11 +18,11 @@ import ( ) func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error { - if link.MFile != nil { - attachHeader(w, file, link) - http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile) - return nil - } + // if link.MFile != nil { + // attachHeader(w, file, link) + // http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile) + // return nil + // } if link.Concurrency > 0 || link.PartSize > 0 { attachHeader(w, file, link) @@ -101,7 +101,7 @@ func GetEtag(file model.Obj, size int64) string { } func ProxyRange(ctx context.Context, link *model.Link, size int64) *model.Link { - if link.MFile == nil && link.RangeReader == nil && !strings.HasPrefix(link.URL, GetApiUrl(ctx)+"/") { + if link.RangeReader == nil && !strings.HasPrefix(link.URL, GetApiUrl(ctx)+"/") { if link.ContentLength > 0 { size = link.ContentLength } diff --git a/server/ftp.go b/server/ftp.go index fb7d86c8e..d07a62dd9 100644 --- a/server/ftp.go +++ b/server/ftp.go @@ -19,6 +19,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/ftp" ftpserver "github.com/fclairamb/ftpserverlib" ) @@ -112,6 +113,12 @@ func (d *FtpMainDriver) ClientDisconnected(cc ftpserver.ClientContext) { } func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) { + ip := cc.RemoteAddr().String() + count, ok := model.LoginCache.Get(ip) + if ok && count >= model.DefaultMaxAuthRetries { + model.LoginCache.Expire(ip, model.DefaultLockDuration) + return nil, errors.New("Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.") + } var userObj *model.User var err error if user == "anonymous" || user == "guest" { @@ -121,17 +128,24 @@ func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) } } else { userObj, err = op.GetUserByName(user) - if err != nil { - return nil, err + if err == nil { + err = userObj.ValidateRawPassword(pass) + if err != nil && setting.GetBool(conf.LdapLoginEnabled) && userObj.AllowLdap { + err = common.HandleLdapLogin(user, pass) + } + } else if setting.GetBool(conf.LdapLoginEnabled) && model.CanFTPAccess(int32(setting.GetInt(conf.LdapDefaultPermission, 0))) { + userObj, err = tryLdapLoginAndRegister(user, pass) } - passHash := model.StaticHash(pass) - if err = userObj.ValidatePwdStaticHash(passHash); err != nil { + if err != nil { + model.LoginCache.Set(ip, count+1) return nil, err } } if userObj.Disabled || !userObj.CanFTPAccess() { + model.LoginCache.Set(ip, count+1) return nil, errors.New("user is not allowed to access via FTP") } + model.LoginCache.Del(ip) ctx := context.Background() ctx = context.WithValue(ctx, conf.UserKey, userObj) @@ -140,7 +154,7 @@ func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) } else { ctx = context.WithValue(ctx, conf.MetaPassKey, "") } - ctx = context.WithValue(ctx, conf.ClientIPKey, cc.RemoteAddr().String()) + ctx = context.WithValue(ctx, conf.ClientIPKey, ip) ctx = context.WithValue(ctx, conf.ProxyHeaderKey, d.proxyHeader) return ftp.NewAferoAdapter(ctx), nil } @@ -167,7 +181,7 @@ func lookupIP(host string) string { } ips, err := net.LookupIP(host) if err != nil || len(ips) == 0 { - utils.Log.Fatalf("given FTP public host is invalid, and the default value will be used: %v", err) + utils.Log.Errorf("given FTP public host is invalid, and the default value will be used: %v", err) return "" } for _, ip := range ips { @@ -272,7 +286,7 @@ func newPortMapper(str string) ftpserver.PasvPortGetter { break } if err != nil { - utils.Log.Fatalf("failed to convert FTP PASV port mapper %s: %v, the port mapper will be ignored.", mapper, err) + utils.Log.Errorf("failed to convert FTP PASV port mapper %s: %v, the port mapper will be ignored.", mapper, err) return nil } } diff --git a/server/ftp/afero.go b/server/ftp/afero.go index 8b7b467f7..02c956a50 100644 --- a/server/ftp/afero.go +++ b/server/ftp/afero.go @@ -104,7 +104,7 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve return nil, err } if (flags & os.O_EXCL) != 0 { - return nil, errors.New("file already exists") + return nil, errs.ObjectAlreadyExists } if (flags & os.O_WRONLY) != 0 { return nil, errors.New("cannot write to uploading file") @@ -122,7 +122,7 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve return nil, errs.ObjectNotFound } if (flags&os.O_EXCL) != 0 && exists { - return nil, errors.New("file already exists") + return nil, errs.ObjectAlreadyExists } if (flags & os.O_WRONLY) != 0 { if offset != 0 { diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index 34fb36c63..c549a1943 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -15,7 +15,9 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" ftpserver "github.com/fclairamb/ftpserverlib" "github.com/pkg/errors" @@ -49,6 +51,11 @@ func OpenUpload(ctx context.Context, path string, trunc bool) (*FileUploadProxy, if err != nil { return nil, err } + // Check if system file should be ignored + _, name := stdpath.Split(path) + if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) { + return nil, errs.IgnoredSystemFile + } tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err @@ -150,6 +157,11 @@ func OpenUploadWithLength(ctx context.Context, path string, trunc bool, length i if err != nil { return nil, err } + // Check if system file should be ignored + _, name := stdpath.Split(path) + if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(name) { + return nil, errs.IgnoredSystemFile + } if trunc { _ = fs.Remove(ctx, path) } diff --git a/server/handles/archive.go b/server/handles/archive.go index 418de6f23..4fd405688 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -1,7 +1,6 @@ package handles import ( - "encoding/json" "fmt" "io" stdpath "path" @@ -229,30 +228,15 @@ func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { }) } -type StringOrArray []string - -func (s *StringOrArray) UnmarshalJSON(data []byte) error { - var value string - if err := json.Unmarshal(data, &value); err == nil { - *s = []string{value} - return nil - } - var sliceValue []string - if err := json.Unmarshal(data, &sliceValue); err != nil { - return err - } - *s = sliceValue - return nil -} - type ArchiveDecompressReq struct { - SrcDir string `json:"src_dir" form:"src_dir"` - DstDir string `json:"dst_dir" form:"dst_dir"` - Name StringOrArray `json:"name" form:"name"` - ArchivePass string `json:"archive_pass" form:"archive_pass"` - InnerPath string `json:"inner_path" form:"inner_path"` - CacheFull bool `json:"cache_full" form:"cache_full"` - PutIntoNewDir bool `json:"put_into_new_dir" form:"put_into_new_dir"` + SrcDir string `json:"src_dir" form:"src_dir"` + DstDir string `json:"dst_dir" form:"dst_dir"` + Names []string `json:"name" form:"name"` + ArchivePass string `json:"archive_pass" form:"archive_pass"` + InnerPath string `json:"inner_path" form:"inner_path"` + CacheFull bool `json:"cache_full" form:"cache_full"` + PutIntoNewDir bool `json:"put_into_new_dir" form:"put_into_new_dir"` + Overwrite bool `json:"overwrite" form:"overwrite"` } func FsArchiveDecompress(c *gin.Context) { @@ -266,8 +250,8 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcPaths := make([]string, 0, len(req.Name)) - for _, name := range req.Name { + srcPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) if err != nil { common.ErrorResp(c, err, 403) @@ -295,6 +279,7 @@ func FsArchiveDecompress(c *gin.Context) { }, CacheFull: req.CacheFull, PutIntoNewDir: req.PutIntoNewDir, + Overwrite: req.Overwrite, }) if e != nil { if errors.Is(e, errs.WrongArchivePassword) { diff --git a/server/handles/auth.go b/server/handles/auth.go index 35776ba6a..780069091 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -45,27 +45,28 @@ func loginHash(c *gin.Context, req *LoginReq) { ip := c.ClientIP() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { - common.ErrorStrResp(c, "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.", 429) + common.ErrorStrResp(c, model.TooManyAttempts, 429) model.LoginCache.Expire(ip, model.DefaultLockDuration) return } // check username user, err := op.GetUserByName(req.Username) if err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // check 2FA if user.OtpSecret != "" { if !totp.Validate(req.OtpCode, user.OtpSecret) { - common.ErrorStrResp(c, "Invalid 2FA code", 402) + // 402 - need opt + common.ErrorStrResp(c, model.Invalid2FACode, 402) model.LoginCache.Set(ip, count+1) return } @@ -73,7 +74,7 @@ func loginHash(c *gin.Context, req *LoginReq) { // generate token token, err := common.GenerateToken(user) if err != nil { - common.ErrorResp(c, err, 400, true) + common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, gin.H{"token": token}) @@ -107,7 +108,7 @@ func UpdateCurrent(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not update profile", 403) + common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } user.Username = req.Username @@ -125,7 +126,7 @@ func UpdateCurrent(c *gin.Context) { func Generate2FA(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } key, err := totp.Generate(totp.GenerateOpts{ @@ -164,11 +165,11 @@ func Verify2FA(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } if !totp.Validate(req.Code, req.Secret) { - common.ErrorStrResp(c, "Invalid 2FA code", 400) + common.ErrorStrResp(c, model.Invalid2FACode, 400) return } user.OtpSecret = req.Secret diff --git a/server/handles/direct_upload.go b/server/handles/direct_upload.go new file mode 100644 index 000000000..69cfd2fa8 --- /dev/null +++ b/server/handles/direct_upload.go @@ -0,0 +1,54 @@ +package handles + +import ( + "net/url" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +type FsGetDirectUploadInfoReq struct { + Path string `json:"path" form:"path"` + FileName string `json:"file_name" form:"file_name"` + FileSize int64 `json:"file_size" form:"file_size"` + Tool string `json:"tool" form:"tool"` +} + +// FsGetDirectUploadInfo returns the direct upload info if supported by the driver +// If the driver does not support direct upload, returns null for upload_info +func FsGetDirectUploadInfo(c *gin.Context) { + var req FsGetDirectUploadInfoReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + // Decode path + path, err := url.PathUnescape(req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + // Get user and join path + user := c.Request.Context().Value(conf.UserKey).(*model.User) + path, err = user.JoinPath(path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + overwrite := c.GetHeader("Overwrite") != "false" + if !overwrite { + if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, "file exists", 403) + return + } + } + directUploadInfo, err := fs.GetDirectUploadInfo(c, req.Tool, path, req.FileName, req.FileSize) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, directUploadInfo) +} diff --git a/server/handles/down.go b/server/handles/down.go index 84ebdc44f..d4d634cbe 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -9,6 +9,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/net" @@ -147,7 +148,7 @@ func proxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) { if Writer.IsWritten() { log.Errorf("%s %s local proxy error: %+v", c.Request.Method, c.Request.URL.Path, err) } else { - if statusCode, ok := errors.Unwrap(err).(net.HttpStatusCodeError); ok { + if statusCode, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { common.ErrorPage(c, err, int(statusCode), true) } else { common.ErrorPage(c, err, 500, true) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index f45da69bb..62382a27c 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -6,18 +6,18 @@ import ( "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/task" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/generic" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type MkdirOrLinkReq struct { @@ -57,10 +57,12 @@ func FsMkdir(c *gin.Context) { } type MoveCopyReq struct { - SrcDir string `json:"src_dir"` - DstDir string `json:"dst_dir"` - Names []string `json:"names"` - Overwrite bool `json:"overwrite"` + SrcDir string `json:"src_dir"` + DstDir string `json:"dst_dir"` + Names []string `json:"names"` + Overwrite bool `json:"overwrite"` + SkipExisting bool `json:"skip_existing"` + Merge bool `json:"merge"` } func FsMove(c *gin.Context) { @@ -78,31 +80,44 @@ func FsMove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcDir, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) - return - } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - if !req.Overwrite { - for _, name := range req.Names { - if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { - common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + validPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + // ensure req.Names is not a relative path + srcPath := stdpath.Join(req.SrcDir, name) + srcPath, err = user.JoinPath(srcPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !req.Overwrite { + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) return } + if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { + if !req.SkipExisting { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + return + } else { + continue + } + } } + validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo - for i, name := range req.Names { - t, err := fs.Move(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + for i, p := range validPaths { + t, err := fs.Move(c.Request.Context(), p, dstDir, len(validPaths) > i+1) if t != nil { addedTasks = append(addedTasks, t) } @@ -140,31 +155,49 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcDir, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) - return - } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - if !req.Overwrite { - for _, name := range req.Names { - if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { - common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + validPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + // ensure req.Names is not a relative path + srcPath := stdpath.Join(req.SrcDir, name) + srcPath, err = user.JoinPath(srcPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !req.Overwrite { + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) return } + if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { + if !req.SkipExisting && !req.Merge { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + return + } else if !req.Merge || !res.IsDir() { + continue + } + } } + validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo - for i, name := range req.Names { - t, err := fs.Copy(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + for i, p := range validPaths { + var t task.TaskExtensionInfo + if req.Merge { + t, err = fs.Merge(c.Request.Context(), p, dstDir, len(validPaths) > i+1) + } else { + t, err = fs.Copy(c.Request.Context(), p, dstDir, len(validPaths) > i+1) + } if t != nil { addedTasks = append(addedTasks, t) } @@ -255,13 +288,25 @@ func FsRemove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - reqDir, err := user.JoinPath(req.Dir) - if err != nil { - common.ErrorResp(c, err, 403) - return + for i, name := range req.Names { + if strings.TrimSpace(utils.FixAndCleanPath(name)) == "/" { + log.Warnf("FsRemove: invalid item skipped: %s (parent directory: %s)\n", name, req.Dir) + req.Names[i] = "" + continue + } + // ensure req.Names is not a relative path + var err error + req.Names[i], err = user.JoinPath(stdpath.Join(req.Dir, name)) + if err != nil { + common.ErrorResp(c, err, 403) + return + } } - for _, name := range req.Names { - err := fs.Remove(c.Request.Context(), stdpath.Join(reqDir, name)) + for _, path := range req.Names { + if path == "" { + continue + } + err := fs.Remove(c.Request.Context(), path) if err != nil { common.ErrorResp(c, err, 500) return @@ -386,7 +431,7 @@ func Link(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if storage.Config().NoLinkURL || storage.Config().OnlyLinkMFile { + if storage.Config().NoLinkURL { common.SuccessResp(c, model.Link{ URL: fmt.Sprintf("%s/p%s?d&sign=%s", common.GetApiUrl(c), diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 6665094cd..886da9dc9 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -33,28 +33,27 @@ type DirReq struct { } type ObjResp struct { - Id string `json:"id"` - Path string `json:"path"` - Name string `json:"name"` - Size int64 `json:"size"` - IsDir bool `json:"is_dir"` - Modified time.Time `json:"modified"` - Created time.Time `json:"created"` - Sign string `json:"sign"` - Thumb string `json:"thumb"` - Type int `json:"type"` - HashInfoStr string `json:"hashinfo"` - HashInfo map[*utils.HashType]string `json:"hash_info"` - MountDetails *model.StorageDetailsWithName `json:"mount_details,omitempty"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + MountDetails *model.StorageDetails `json:"mount_details,omitempty"` } type FsListResp struct { - Content []ObjResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` + Content []ObjResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + Provider string `json:"provider"` + DirectUploadTools []string `json:"direct_upload_tools,omitempty"` } func FsListSplit(c *gin.Context) { @@ -109,17 +108,20 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { } total, objs := pagination(objs, &req.PageReq) provider := "unknown" - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) - if err == nil { - provider = storage.GetStorage().Driver + var directUploadTools []string + if user.CanWrite() { + if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { + directUploadTools = op.GetDirectUploadTools(storage) + } } common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), - Total: int64(total), - Readme: getReadme(meta, reqPath), - Header: getHeader(meta, reqPath), - Write: user.CanWrite() || common.CanWrite(meta, reqPath), - Provider: provider, + Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), + Total: int64(total), + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Write: user.CanWrite() || common.CanWrite(meta, reqPath), + Provider: provider, + DirectUploadTools: directUploadTools, }) } @@ -230,8 +232,6 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) resp = append(resp, ObjResp{ - Id: obj.GetID(), - Path: obj.GetPath(), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), @@ -361,8 +361,6 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { mountDetails, _ := model.GetStorageDetails(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ - Id: obj.GetID(), - Path: obj.GetPath(), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 71d9dbae7..0f46398cd 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -8,8 +8,10 @@ import ( "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -28,6 +30,14 @@ func getLastModified(c *gin.Context) time.Time { return lastModified } +// shouldIgnoreSystemFile checks if the filename should be ignored based on settings +func shouldIgnoreSystemFile(filename string) bool { + if setting.GetBool(conf.IgnoreSystemFiles) { + return utils.IsSystemFile(filename) + } + return false +} + func FsStream(c *gin.Context) { defer func() { if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 { @@ -56,6 +66,11 @@ func FsStream(c *gin.Context) { } } dir, name := stdpath.Split(path) + // Check if system file should be ignored + if shouldIgnoreSystemFile(name) { + common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403) + return + } // 如果请求头 Content-Length 和 X-File-Size 都没有，则 size=-1，表示未知大小的流式上传 size := c.Request.ContentLength if size < 0 { @@ -97,7 +112,7 @@ func FsStream(c *gin.Context) { if asTask { t, err = fs.PutAsTask(c.Request.Context(), dir, s) } else { - err = fs.PutDirectly(c.Request.Context(), dir, s, true) + err = fs.PutDirectly(c.Request.Context(), dir, s) } if err != nil { common.ErrorResp(c, err, 500) @@ -160,6 +175,11 @@ func FsForm(c *gin.Context) { } defer f.Close() dir, name := stdpath.Split(path) + // Check if system file should be ignored + if shouldIgnoreSystemFile(name) { + common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403) + return + } h := make(map[*utils.HashType]string) if md5 := c.GetHeader("X-File-Md5"); md5 != "" { h[utils.MD5] = md5 @@ -192,7 +212,7 @@ func FsForm(c *gin.Context) { }{f} t, err = fs.PutAsTask(c.Request.Context(), dir, s) } else { - err = fs.PutDirectly(c.Request.Context(), dir, s, true) + err = fs.PutDirectly(c.Request.Context(), dir, s) } if err != nil { common.ErrorResp(c, err, 500) diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go index dff3aebe3..ba44615f7 100644 --- a/server/handles/ldap_login.go +++ b/server/handles/ldap_login.go @@ -1,21 +1,13 @@ package handles import ( - "crypto/tls" - "errors" - "fmt" - "strings" - "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/OpenListTeam/OpenList/v4/pkg/utils/random" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" - "gopkg.in/ldap.v3" + "github.com/pkg/errors" ) func LoginLdap(c *gin.Context) { @@ -24,15 +16,16 @@ func LoginLdap(c *gin.Context) { common.ErrorResp(c, err, 400) return } - loginLdap(c, &req) -} - -func loginLdap(c *gin.Context, req *LoginReq) { enabled := setting.GetBool(conf.LdapLoginEnabled) if !enabled { common.ErrorStrResp(c, "ldap is not enabled", 403) return } + user, err := op.GetUserByName(req.Username) + if err == nil && !user.AllowLdap { + common.ErrorStrResp(c, "login via ldap is not allowed", 403) + return + } // check count of login ip := c.ClientIP() @@ -43,67 +36,19 @@ func loginLdap(c *gin.Context, req *LoginReq) { return } - // Auth start - ldapServer := setting.GetStr(conf.LdapServer) - ldapManagerDN := setting.GetStr(conf.LdapManagerDN) - ldapManagerPassword := setting.GetStr(conf.LdapManagerPassword) - ldapUserSearchBase := setting.GetStr(conf.LdapUserSearchBase) - ldapUserSearchFilter := setting.GetStr(conf.LdapUserSearchFilter) // (uid=%s) - - // Connect to LdapServer - l, err := dial(ldapServer) + err = common.HandleLdapLogin(req.Username, req.Password) if err != nil { - utils.Log.Errorf("failed to connect to LDAP: %v", err) - common.ErrorResp(c, err, 500) - return - } - - // First bind with a read only user - if ldapManagerDN != "" && ldapManagerPassword != "" { - err = l.Bind(ldapManagerDN, ldapManagerPassword) - if err != nil { - utils.Log.Errorf("Failed to bind to LDAP: %v", err) + if errors.Is(err, common.ErrFailedLdapAuth) { + model.LoginCache.Set(ip, count+1) + common.ErrorResp(c, err, 400) + } else { common.ErrorResp(c, err, 500) - return } - } - - // Search for the given username - searchRequest := ldap.NewSearchRequest( - ldapUserSearchBase, - ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, - fmt.Sprintf(ldapUserSearchFilter, req.Username), - []string{"dn"}, - nil, - ) - sr, err := l.Search(searchRequest) - if err != nil { - utils.Log.Errorf("LDAP search failed: %v", err) - common.ErrorResp(c, err, 500) - return - } - if len(sr.Entries) != 1 { - utils.Log.Errorf("User does not exist or too many entries returned") - common.ErrorResp(c, err, 500) return } - userDN := sr.Entries[0].DN - // Bind as the user to verify their password - err = l.Bind(userDN, req.Password) - if err != nil { - utils.Log.Errorf("Failed to auth. %v", err) - common.ErrorResp(c, err, 400) - model.LoginCache.Set(ip, count+1) - return - } else { - utils.Log.Infof("Auth successful username:%s", req.Username) - } - // Auth finished - - user, err := op.GetUserByName(req.Username) - if err != nil { - user, err = ladpRegister(req.Username) + if user == nil { + user, err = common.LdapRegister(req.Username) if err != nil { common.ErrorResp(c, err, 400) model.LoginCache.Set(ip, count+1) @@ -120,38 +65,3 @@ func loginLdap(c *gin.Context, req *LoginReq) { common.SuccessResp(c, gin.H{"token": token}) model.LoginCache.Del(ip) } - -func ladpRegister(username string) (*model.User, error) { - if username == "" { - return nil, errors.New("cannot get username from ldap provider") - } - user := &model.User{ - ID: 0, - Username: username, - Password: random.String(16), - Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), - BasePath: setting.GetStr(conf.LdapDefaultDir), - Role: 0, - Disabled: false, - } - if err := db.CreateUser(user); err != nil { - return nil, err - } - return user, nil -} - -func dial(ldapServer string) (*ldap.Conn, error) { - var tlsEnabled bool = false - if strings.HasPrefix(ldapServer, "ldaps://") { - tlsEnabled = true - ldapServer = strings.TrimPrefix(ldapServer, "ldaps://") - } else if strings.HasPrefix(ldapServer, "ldap://") { - ldapServer = strings.TrimPrefix(ldapServer, "ldap://") - } - - if tlsEnabled { - return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: true}) - } else { - return ldap.Dial("tcp", ldapServer) - } -} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index fb6527349..153b27293 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -2,9 +2,11 @@ package handles import ( "strings" - + _115 "github.com/OpenListTeam/OpenList/v4/drivers/115" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" + _123 "github.com/OpenListTeam/OpenList/v4/drivers/123" + _123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open" "github.com/OpenListTeam/OpenList/v4/drivers/pikpak" "github.com/OpenListTeam/OpenList/v4/drivers/thunder" "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" @@ -200,6 +202,96 @@ func Set115Open(c *gin.Context) { common.SuccessResp(c, "ok") } +type Set123PanReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func Set123Pan(c *gin.Context) { + var req Set123PanReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*_123.Pan123); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123Pan is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.Pan123TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("123Pan") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type Set123OpenReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` + CallbackUrl string `json:"callback_url" form:"callback_url"` +} + +func Set123Open(c *gin.Context) { + var req Set123OpenReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*_123_open.Open123); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only 123 Open is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.Pan123OpenTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.Pan123OpenOfflineDownloadCallbackUrl, Value: req.CallbackUrl, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("123 Open") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + type SetPikPakReq struct { TempDir string `json:"temp_dir" form:"temp_dir"` } @@ -413,7 +505,7 @@ func AddOfflineDownload(c *gin.Context) { if trimmedUrl == "" { continue } - + t, err := tool.AddURL(c, &tool.AddURLArgs{ URL: trimmedUrl, DstDirPath: reqPath, diff --git a/server/handles/scan.go b/server/handles/scan.go new file mode 100644 index 000000000..fc5e80f66 --- /dev/null +++ b/server/handles/scan.go @@ -0,0 +1,47 @@ +package handles + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/gin-gonic/gin" +) + +type ManualScanReq struct { + Path string `json:"path"` + Limit float64 `json:"limit"` +} + +func StartManualScan(c *gin.Context) { + var req ManualScanReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.BeginManualScan(req.Path, req.Limit); err != nil { + common.ErrorResp(c, err, 400) + return + } + common.SuccessResp(c) +} + +func StopManualScan(c *gin.Context) { + if !op.ManualScanRunning() { + common.ErrorStrResp(c, "manual scan is not running", 400) + return + } + op.StopManualScan() + common.SuccessResp(c) +} + +type ManualScanResp struct { + ObjCount uint64 `json:"obj_count"` + IsDone bool `json:"is_done"` +} + +func GetManualScanProgress(c *gin.Context) { + ret := ManualScanResp{ + ObjCount: op.ScannedCount.Load(), + IsDone: !op.ManualScanRunning(), + } + common.SuccessResp(c, ret) +} diff --git a/server/handles/sharing.go b/server/handles/sharing.go index 5a6ea265f..43f855afb 100644 --- a/server/handles/sharing.go +++ b/server/handles/sharing.go @@ -34,9 +34,9 @@ func SharingGet(c *gin.Context, req *FsGetReq) { return } _ = countAccess(c.ClientIP(), s) - fakePath := fmt.Sprintf("/%s/%s", sid, path) url := "" if !obj.IsDir() { + fakePath := fmt.Sprintf("/%s/%s", sid, path) url = fmt.Sprintf("%s/sd%s", common.GetApiUrl(c), utils.EncodePath(fakePath, true)) if s.Pwd != "" { url += "?pwd=" + s.Pwd @@ -45,8 +45,6 @@ func SharingGet(c *gin.Context, req *FsGetReq) { thumb, _ := model.GetThumb(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ - Id: "", - Path: fakePath, Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), @@ -80,14 +78,11 @@ func SharingList(c *gin.Context, req *ListReq) { return } _ = countAccess(c.ClientIP(), s) - fakePath := fmt.Sprintf("/%s/%s", sid, path) total, objs := pagination(objs, &req.PageReq) common.SuccessResp(c, FsListResp{ Content: utils.MustSliceConvert(objs, func(obj model.Obj) ObjResp { thumb, _ := model.GetThumb(obj) return ObjResp{ - Id: "", - Path: stdpath.Join(fakePath, obj.GetName()), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), @@ -408,7 +403,7 @@ func ListSharings(c *gin.Context) { }) } -type CreateSharingReq struct { +type UpdateSharingReq struct { Files []string `json:"files"` Expires *time.Time `json:"expires"` Pwd string `json:"pwd"` @@ -418,12 +413,9 @@ type CreateSharingReq struct { Readme string `json:"readme"` Header string `json:"header"` model.Sort -} - -type UpdateSharingReq struct { - ID string `json:"id"` - Accessed int `json:"accessed"` - CreateSharingReq + CreatorName string `json:"creator"` + Accessed int `json:"accessed"` + ID string `json:"id"` } func UpdateSharing(c *gin.Context) { @@ -436,24 +428,38 @@ func UpdateSharing(c *gin.Context) { common.ErrorStrResp(c, "must add at least 1 object", 400) return } - user := c.Request.Context().Value(conf.UserKey).(*model.User) - if !user.CanShare() { - common.ErrorStrResp(c, "permission denied", 403) - return + var user *model.User + var err error + reqUser := c.Request.Context().Value(conf.UserKey).(*model.User) + if reqUser.IsAdmin() && req.CreatorName != "" { + user, err = op.GetUserByName(req.CreatorName) + if err != nil { + common.ErrorStrResp(c, "no such a user", 400) + return + } + } else { + user = reqUser + if !user.CanShare() { + common.ErrorStrResp(c, "permission denied", 403) + return + } } for i, s := range req.Files { s = utils.FixAndCleanPath(s) req.Files[i] = s - if !user.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { + if !reqUser.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { common.ErrorStrResp(c, fmt.Sprintf("permission denied to share path [%s]", s), 500) return } } s, err := op.GetSharingById(req.ID) - if err != nil || (!user.IsAdmin() && s.CreatorId != user.ID) { + if err != nil || (!reqUser.IsAdmin() && s.CreatorId != user.ID) { common.ErrorStrResp(c, "sharing not found", 404) return } + if reqUser.IsAdmin() && req.CreatorName == "" { + user = s.Creator + } s.Files = req.Files s.Expires = req.Expires s.Pwd = req.Pwd @@ -464,6 +470,7 @@ func UpdateSharing(c *gin.Context) { s.Header = req.Header s.Readme = req.Readme s.Remark = req.Remark + s.Creator = user if err = op.UpdateSharing(s); err != nil { common.ErrorResp(c, err, 500) } else { @@ -476,7 +483,7 @@ func UpdateSharing(c *gin.Context) { } func CreateSharing(c *gin.Context) { - var req CreateSharingReq + var req UpdateSharingReq var err error if err = c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) @@ -486,24 +493,35 @@ func CreateSharing(c *gin.Context) { common.ErrorStrResp(c, "must add at least 1 object", 400) return } - user := c.Request.Context().Value(conf.UserKey).(*model.User) - if !user.CanShare() { - common.ErrorStrResp(c, "permission denied", 403) - return + var user *model.User + reqUser := c.Request.Context().Value(conf.UserKey).(*model.User) + if reqUser.IsAdmin() && req.CreatorName != "" { + user, err = op.GetUserByName(req.CreatorName) + if err != nil { + common.ErrorStrResp(c, "no such a user", 400) + return + } + } else { + user = reqUser + if !user.CanShare() || (!user.IsAdmin() && req.ID != "") { + common.ErrorStrResp(c, "permission denied", 403) + return + } } for i, s := range req.Files { s = utils.FixAndCleanPath(s) req.Files[i] = s - if !user.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { + if !reqUser.IsAdmin() && !strings.HasPrefix(s, user.BasePath) { common.ErrorStrResp(c, fmt.Sprintf("permission denied to share path [%s]", s), 500) return } } s := &model.Sharing{ SharingDB: &model.SharingDB{ + ID: req.ID, Expires: req.Expires, Pwd: req.Pwd, - Accessed: 0, + Accessed: req.Accessed, MaxAccessed: req.MaxAccessed, Disabled: req.Disabled, Sort: req.Sort, diff --git a/server/handles/storage.go b/server/handles/storage.go index e802caad4..687c93551 100644 --- a/server/handles/storage.go +++ b/server/handles/storage.go @@ -4,7 +4,6 @@ import ( "context" "errors" "strconv" - "sync" "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -24,9 +23,15 @@ type StorageResp struct { MountDetails *model.StorageDetails `json:"mount_details,omitempty"` } -func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp { +type detailWithIndex struct { + idx int + val *model.StorageDetails +} + +func makeStorageResp(ctx *gin.Context, storages []model.Storage) []*StorageResp { ret := make([]*StorageResp, len(storages)) - var wg sync.WaitGroup + detailsChan := make(chan detailWithIndex, len(storages)) + workerCount := 0 for i, s := range storages { ret[i] = &StorageResp{ Storage: s, @@ -43,22 +48,26 @@ func makeStorageResp(c *gin.Context, storages []model.Storage) []*StorageResp { if !ok { continue } - wg.Add(1) - go func() { - defer wg.Done() - ctx, cancel := context.WithTimeout(c, time.Second*3) - defer cancel() - details, err := op.GetStorageDetails(ctx, d) - if err != nil { - if !errors.Is(err, errs.NotImplement) { - log.Errorf("failed get %s details: %+v", s.MountPath, err) + workerCount++ + go func(dri driver.Driver, idx int) { + details, e := op.GetStorageDetails(ctx, dri) + if e != nil { + if !errors.Is(e, errs.NotImplement) && !errors.Is(e, errs.StorageNotInit) { + log.Errorf("failed get %s details: %+v", dri.GetStorage().MountPath, e) } - return } - ret[i].MountDetails = details - }() + detailsChan <- detailWithIndex{idx: idx, val: details} + }(d, i) + } + for workerCount > 0 { + select { + case r := <-detailsChan: + ret[r.idx].MountDetails = r.val + workerCount-- + case <-time.After(time.Second * 3): + workerCount = 0 + } } - wg.Wait() return ret } @@ -175,7 +184,7 @@ func LoadAllStorages(c *gin.Context) { common.ErrorResp(c, err, 500, true) return } - conf.StoragesLoaded = false + conf.ResetStoragesLoadSignal() go func(storages []model.Storage) { for _, storage := range storages { storageDriver, err := op.GetStorageByMountPath(storage.MountPath) @@ -195,7 +204,7 @@ func LoadAllStorages(c *gin.Context) { log.Infof("success load storage: [%s], driver: [%s]", storage.MountPath, storage.Driver) } - conf.StoragesLoaded = true + conf.SendStoragesLoadedSignal() }(storages) common.SuccessResp(c) } diff --git a/server/middlewares/check.go b/server/middlewares/check.go index a1011de3b..c7203a490 100644 --- a/server/middlewares/check.go +++ b/server/middlewares/check.go @@ -22,9 +22,12 @@ func StoragesLoaded(c *gin.Context) { return } } - common.ErrorStrResp(c, "Loading storage, please wait", 500) - c.Abort() - return + select { + case <-conf.StoragesLoadSignal(): + case <-c.Request.Context().Done(): + c.Abort() + return + } } common.GinWithValue(c, conf.ApiUrlKey, common.GetApiUrlFromRequest(c.Request), diff --git a/server/middlewares/search.go b/server/middlewares/search.go index 7807a210b..dc89a08ad 100644 --- a/server/middlewares/search.go +++ b/server/middlewares/search.go @@ -11,7 +11,7 @@ import ( func SearchIndex(c *gin.Context) { mode := setting.GetStr(conf.SearchIndex) if mode == "none" { - common.ErrorResp(c, errs.SearchNotAvailable, 500) + common.ErrorResp(c, errs.SearchNotAvailable, 404) c.Abort() } else { c.Next() diff --git a/server/router.go b/server/router.go index 66f0539ba..57d1166ae 100644 --- a/server/router.go +++ b/server/router.go @@ -160,6 +160,8 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_transmission", handles.SetTransmission) setting.POST("/set_115", handles.Set115) setting.POST("/set_115_open", handles.Set115Open) + setting.POST("/set_123_pan", handles.Set123Pan) + setting.POST("/set_123_open", handles.Set123Open) setting.POST("/set_pikpak", handles.SetPikPak) setting.POST("/set_thunder", handles.SetThunder) setting.POST("/set_thunderx", handles.SetThunderX) @@ -178,6 +180,11 @@ func admin(g *gin.RouterGroup) { index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) + + scan := g.Group("/scan") + scan.POST("/start", handles.StartManualScan) + scan.POST("/stop", handles.StopManualScan) + scan.GET("/progress", handles.GetManualScanProgress) } func fsAndShare(g *gin.RouterGroup) { @@ -210,6 +217,8 @@ func _fs(g *gin.RouterGroup) { // g.POST("/add_transmission", handles.SetTransmission) g.POST("/add_offline_download", handles.AddOfflineDownload) g.POST("/archive/decompress", handles.FsArchiveDecompress) + // Direct upload (client-side upload to storage) + g.POST("/get_direct_upload_info", middlewares.FsUp, handles.FsGetDirectUploadInfo) } func _task(g *gin.RouterGroup) { diff --git a/server/s3/backend.go b/server/s3/backend.go index ac398699c..f00d2149c 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -19,6 +19,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" @@ -257,7 +258,7 @@ func (b *s3Backend) PutObject( if err != nil { if errs.IsObjectNotFound(err) && strings.Contains(objectName, "/") { log.Debugf("reqPath: %s not found and objectName contains /, need to makeDir", reqPath) - err = fs.MakeDir(ctx, reqPath, true) + err = fs.MakeDir(ctx, reqPath) if err != nil { return result, errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath) } @@ -280,12 +281,21 @@ func (b *s3Backend) PutObject( ti, _ = swift.FloatStringToTime(val) } + // If Modified is not set, use current time + if ti.IsZero() { + ti = time.Now() + } + obj := model.Object{ Name: path.Base(fp), Size: size, Modified: ti, Ctime: time.Now(), } + // Check if system file should be ignored + if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { + return result, errs.IgnoredSystemFile + } stream := &stream.FileStream{ Obj: &obj, Reader: input, diff --git a/server/sftp.go b/server/sftp.go index 055f79730..37dc9870d 100644 --- a/server/sftp.go +++ b/server/sftp.go @@ -11,6 +11,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/ftp" "github.com/OpenListTeam/OpenList/v4/server/sftp" "github.com/OpenListTeam/sftpd-openlist" @@ -92,17 +93,31 @@ func (d *SftpDriver) NoClientAuth(conn ssh.ConnMetadata) (*ssh.Permissions, erro } func (d *SftpDriver) PasswordAuth(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) { + ip := conn.RemoteAddr().String() + count, ok := model.LoginCache.Get(ip) + if ok && count >= model.DefaultMaxAuthRetries { + model.LoginCache.Expire(ip, model.DefaultLockDuration) + return nil, errors.New("Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.") + } + pass := string(password) userObj, err := op.GetUserByName(conn.User()) + if err == nil { + err = userObj.ValidateRawPassword(pass) + if err != nil && setting.GetBool(conf.LdapLoginEnabled) && userObj.AllowLdap { + err = common.HandleLdapLogin(conn.User(), pass) + } + } else if setting.GetBool(conf.LdapLoginEnabled) && model.CanFTPAccess(int32(setting.GetInt(conf.LdapDefaultPermission, 0))) { + userObj, err = tryLdapLoginAndRegister(conn.User(), pass) + } if err != nil { + model.LoginCache.Set(ip, count+1) return nil, err } if userObj.Disabled || !userObj.CanFTPAccess() { + model.LoginCache.Set(ip, count+1) return nil, errors.New("user is not allowed to access via SFTP") } - passHash := model.StaticHash(string(password)) - if err = userObj.ValidatePwdStaticHash(passHash); err != nil { - return nil, err - } + model.LoginCache.Del(ip) return nil, nil } diff --git a/server/sftp/hostkey.go b/server/sftp/hostkey.go index ffa25c665..6d857666a 100644 --- a/server/sftp/hostkey.go +++ b/server/sftp/hostkey.go @@ -24,7 +24,7 @@ func InitHostKey() { if !utils.Exists(sshPath) { err := utils.CreateNestedDirectory(sshPath) if err != nil { - utils.Log.Fatalf("failed to create ssh directory: %+v", err) + utils.Log.Errorf("failed to create ssh directory: %+v", err) return } } @@ -54,30 +54,30 @@ func LoadOrGenerateRSAHostKey(parentDir string) (ssh.Signer, bool) { _ = os.Remove(publicKeyPath) privateKey, err := rsa.GenerateKey(rand.Reader, 4096) if err != nil { - utils.Log.Fatalf("failed to generate RSA private key: %+v", err) + utils.Log.Errorf("failed to generate RSA private key: %+v", err) return nil, false } publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey) if err != nil { - utils.Log.Fatalf("failed to generate RSA public key: %+v", err) + utils.Log.Errorf("failed to generate RSA public key: %+v", err) return nil, false } ret, err := ssh.NewSignerFromKey(privateKey) if err != nil { - utils.Log.Fatalf("failed to generate RSA signer: %+v", err) + utils.Log.Errorf("failed to generate RSA signer: %+v", err) return nil, false } privateBytes := rsaEncodePrivateKey(privateKey) publicBytes := ssh.MarshalAuthorizedKey(publicKey) err = os.WriteFile(privateKeyPath, privateBytes, 0600) if err != nil { - utils.Log.Fatalf("failed to write RSA private key to file: %+v", err) + utils.Log.Errorf("failed to write RSA private key to file: %+v", err) return nil, false } err = os.WriteFile(publicKeyPath, publicBytes, 0644) if err != nil { _ = os.Remove(privateKeyPath) - utils.Log.Fatalf("failed to write RSA public key to file: %+v", err) + utils.Log.Errorf("failed to write RSA public key to file: %+v", err) return nil, false } return ret, true diff --git a/server/utils.go b/server/utils.go new file mode 100644 index 000000000..0e9d68809 --- /dev/null +++ b/server/utils.go @@ -0,0 +1,14 @@ +package server + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/server/common" +) + +func tryLdapLoginAndRegister(user, pass string) (*model.User, error) { + err := common.HandleLdapLogin(user, pass) + if err != nil { + return nil, err + } + return common.LdapRegister(user) +} diff --git a/server/webdav.go b/server/webdav.go index b2afe581d..789236b8b 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -6,14 +6,13 @@ import ( "path" "strings" + "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/OpenList/v4/server/middlewares" - - "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/server/webdav" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -94,8 +93,8 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - user, err := op.GetUserByName(username) - if err != nil || user.ValidateRawPassword(password) != nil { + user, ok := tryLogin(username, password) + if !ok { if c.Request.Method == "OPTIONS" { common.GinWithValue(c, conf.UserKey, guest) c.Next() @@ -146,3 +145,16 @@ func WebDAVAuth(c *gin.Context) { common.GinWithValue(c, conf.UserKey, user) c.Next() } + +func tryLogin(username, password string) (*model.User, bool) { + user, err := op.GetUserByName(username) + if err == nil { + err = user.ValidateRawPassword(password) + if err != nil && setting.GetBool(conf.LdapLoginEnabled) && user.AllowLdap { + err = common.HandleLdapLogin(username, password) + } + } else if setting.GetBool(conf.LdapLoginEnabled) && model.CanWebdavRead(int32(setting.GetInt(conf.LdapDefaultPermission, 0))) { + user, err = tryLdapLoginAndRegister(username, password) + } + return user, err == nil +} diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 0c4f0922c..504c5fc1d 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/net" + "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -272,7 +273,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta } err = common.Proxy(w, r, link, fi) if err != nil { - if statusCode, ok := errors.Unwrap(err).(net.HttpStatusCodeError); ok { + if statusCode, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { return int(statusCode), err } return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err) @@ -358,6 +359,10 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, Modified: h.getModTime(r), Ctime: h.getCreateTime(r), } + // Check if system file should be ignored + if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { + return http.StatusForbidden, errs.IgnoredSystemFile + } fsStream := &stream.FileStream{ Obj: &obj, Reader: r.Body,