diff --git a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml index 7dadfc3f4..c6aa3fc62 100644 --- a/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml +++ b/.github/ISSUE_TEMPLATE/00-bug_report_zh.yml @@ -13,7 +13,7 @@ body: attributes: label: 请确认以下事项 description: | - 您必须阅读并检查以下内容，否则您的问题一定会被直接关闭。 + 您必须阅读、检查、确认、同意以下内容，否则您的问题一定会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | @@ -35,8 +35,7 @@ body: - label: | 我已确认这个问题在最新版本中没有被修复。 - label: | - 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue - + 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue 。 - type: input id: version attributes: diff --git a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml index 5d5d91d00..d99968d90 100644 --- a/.github/ISSUE_TEMPLATE/01-bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/01-bug_report_en.yml @@ -13,7 +13,7 @@ body: attributes: label: Please confirm the following description: | - You must read and check all the following, otherwise your issue will definitely be closed directly. + You must read, check, confirm, and agree to all the following, otherwise your issue will definitely be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | @@ -35,8 +35,7 @@ body: - label: | I confirm this issue is not fixed in the latest version. - label: | - I have not read these checkboxes and therefore I just ticked them all, Please close this issue - + I have not read these checkboxes and therefore I just ticked them all, Please close this issue. - type: input id: version attributes: diff --git a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml index 821b2c446..8339d947f 100644 --- a/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml +++ b/.github/ISSUE_TEMPLATE/02-feature_request_zh.yml @@ -7,7 +7,7 @@ body: attributes: label: 请确认以下事项 description: | - 您必须勾选以下内容，否则您的问题可能会被直接关闭。 + 您必须阅读、检查、确认、同意以下内容，否则您的问题可能会被直接关闭。 或者您可以去[讨论区](https://github.com/OpenListTeam/OpenList/discussions)。 options: - label: | @@ -28,6 +28,8 @@ body: 我已确认此功能尚未被实现。 - label: | 我已确认此功能是合理的，且有普遍需求，并非我个人需要。 + - label: | + 我没有阅读这个清单，只是闭眼选中了所有的复选框，请关闭这个 Issue 。 - type: textarea id: feature-description attributes: diff --git a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml index 85b024883..41c9990cf 100644 --- a/.github/ISSUE_TEMPLATE/03-feature_request_en.yml +++ b/.github/ISSUE_TEMPLATE/03-feature_request_en.yml @@ -1,13 +1,13 @@ name: "Feature Request" description: Feature Request / Enhancement -title: "[Feature] Please change the title to your feature name" +title: "[Feature] Please modify the title to your feature name" labels: [enhancement] body: - type: checkboxes attributes: label: Please confirm the following description: | - You must check all the following, otherwise your request may be closed directly. + You must read, check, confirm, and agree to all the following, otherwise your request may be closed directly. Or you can go to the [discussions](https://github.com/OpenListTeam/OpenList/discussions). options: - label: | @@ -28,6 +28,8 @@ body: I confirm this feature has not been implemented yet. - label: | I confirm this feature is reasonable and has general demand, not just my personal need. + - label: | + I have not read these checkboxes and therefore I just ticked them all, Please close this issue. - type: textarea id: feature-description attributes: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index e74e59631..f1687eabf 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,11 +2,13 @@ Provide a general summary of your changes in the Title above. The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`. If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body. + For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`. --> ## Description / 描述 diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 268a5833f..1fd87625a 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 @@ -48,7 +48,7 @@ jobs: tag_name: beta - name: Upload assets to github artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: beta changelog path: ${{ github.workspace }}/CHANGELOG.md @@ -61,31 +61,56 @@ jobs: strategy: matrix: include: - - target: "!(*musl*|*windows-arm64*|*windows7-*|*android*|*freebsd*)" # xgo and loongarch + - target: "!(*musl*|*windows-arm64*|*windows7-*|*android*|*freebsd*)" # xgo and loongarch (exclude mips64le) hash: "md5" - - target: "linux-!(arm*)-musl*" #musl-not-arm + flags: "" + goflags: "" + - target: "linux-(mips|mips64|mipsle|mips64le|loong64)-musl*" # musl-compat-family + hash: "md5-linux-musl-mips" + flags: "-ldflags=-linkmode external -extldflags '-static -fpic'" + goflags: "" + musl_static: "true" + - target: "linux-!(arm*|mips|mips64|mipsle|mips64le|loong64)-musl*" # musl-not-arm (exclude compat-family) hash: "md5-linux-musl" + flags: "-ldflags=-linkmode external -extldflags '-static -fpic'" + goflags: "" + musl_static: "true" - target: "linux-arm*-musl*" #musl-arm hash: "md5-linux-musl-arm" + flags: "-ldflags=-linkmode external -extldflags '-static -fpic'" + goflags: "" + musl_static: "true" - target: "windows-arm64" #win-arm64 hash: "md5-windows-arm64" + flags: "" + goflags: "" + musl_static: "false" - target: "windows7-*" #win7 hash: "md5-windows7" + flags: "" + goflags: "-tags=sqlite_cgo_compat" + musl_static: "false" - target: "android-*" #android hash: "md5-android" + flags: "" + goflags: "" + musl_static: "false" - target: "freebsd-*" #freebsd hash: "md5-freebsd" + flags: "" + goflags: "" + musl_static: "false" name: Beta Release runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "1.25.0" @@ -99,6 +124,8 @@ jobs: uses: OpenListTeam/cgo-actions@v1.2.2 with: targets: ${{ matrix.target }} + flags: ${{ matrix.flags || '-ldflags=' }} + static-link-for-musl: true musl-target-format: $os-$musl-$arch github-token: ${{ secrets.GITHUB_TOKEN }} out-dir: build @@ -110,6 +137,27 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} + env: + GOFLAGS: ${{ matrix.goflags }} + + - name: Verify musl binaries are static + if: matrix.musl_static == 'true' + run: | + set -e + shopt -s nullglob + files=(build/openlist-*-musl-*) + if [ ${#files[@]} -eq 0 ]; then + echo "No musl binaries found" + exit 1 + fi + for f in "${files[@]}"; do + if readelf -l "$f" | grep -q "Requesting program interpreter"; then + echo "Dynamic binary detected: $f" + readelf -l "$f" | grep "Requesting program interpreter" || true + exit 1 + fi + done - name: Compress run: | @@ -135,7 +183,7 @@ jobs: echo "cleaned_target=$CLEANED_TARGET" >> $GITHUB_ENV - name: Upload assets to github artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: beta builds for ${{ env.cleaned_target }} path: ${{ github.workspace }}/build/compress/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2393b84b..a38d8b715 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: build: strategy: @@ -25,13 +28,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - uses: benjlevesque/short-sha@v3.0 id: short-sha - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: "1.25.0" @@ -45,6 +48,8 @@ jobs: uses: OpenListTeam/cgo-actions@v1.2.2 with: targets: ${{ matrix.target }} + flags: ${{ contains(matrix.target, '-musl') && '-ldflags=-linkmode external -extldflags ''-static -fpic''' || '-ldflags=' }} + static-link-for-musl: true musl-target-format: $os-$musl-$arch github-token: ${{ secrets.GITHUB_TOKEN }} out-dir: build @@ -53,11 +58,22 @@ jobs: github.com/OpenListTeam/OpenList/v4/internal/conf.GitAuthor=The OpenList Projects Contributors github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$git_commit github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$tag - github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=rolling + github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=latest + github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=${{ vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} output: openlist$ext + - name: Verify musl binary is static + if: contains(matrix.target, '-musl') + run: | + set -e + if readelf -l build/openlist | grep -q "Requesting program interpreter"; then + echo "Dynamic binary detected: build/openlist" + readelf -l build/openlist | grep "Requesting program interpreter" || true + exit 1 + fi + - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: openlist_${{ steps.short-sha.outputs.sha }}_${{ matrix.target }} path: build/* diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 8d458314a..e14eaaa76 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/issue_pr_comment.yml b/.github/workflows/issue_pr_comment.yml index c618485f0..bc29e6bf6 100644 --- a/.github/workflows/issue_pr_comment.yml +++ b/.github/workflows/issue_pr_comment.yml @@ -16,14 +16,52 @@ jobs: if: github.event_name == 'issues' steps: - name: Check issue for unchecked tasks and reply - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | + let comment = ""; + const issueTitle = context.payload.issue.title || ""; + const titleNotEdited = /(请修改标题|Please modify the title)/i.test(issueTitle); + if (titleNotEdited) { + comment = "⚠️ 请修改标题以更好地描述您的问题或需求，并删除示例提示。当前 Issue 将被自动关闭。如需继续提交，请创建新的 Issue。\n"; + comment += "⚠️ Please modify the title to better describe your issue or request, and remove the example prompt. This issue will be automatically closed. If you wish to proceed, please create a new issue.\n"; + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); + await github.rest.issues.update({ + ...context.repo, + issue_number: context.issue.number, + state: 'closed', + state_reason: 'not_planned', + labels: ['invalid'] + }); + return; + } const issueBody = context.payload.issue.body || ""; - const unchecked = /- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); - let comment = "感谢您联系OpenList。我们会尽快回复您。\n"; - comment += "Thanks for contacting OpenList. We will reply to you as soon as possible.\n\n"; - if (unchecked) { + const confirmHasRead = /- \[ \] (?!我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); + const confirmNotRead = /- \[[xX]\] (?:我没有阅读这个清单|I have not read these checkboxes)/.test(issueBody); + if (confirmNotRead) { + comment = "⚠️ 你的 Issue 不符合提交规则。请先阅读相关规范后再重新提交。当前 Issue 将被自动关闭。如需继续提交，请确认已了解规则后重新打开或创建新的 Issue。\n"; + comment += "⚠️ Your issue does not comply with the submission rules. Please read the guidelines before submitting again. This issue will be automatically closed. If you wish to proceed, please confirm that you have reviewed the rules before reopening or creating a new issue.\n"; + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); + await github.rest.issues.update({ + ...context.repo, + issue_number: context.issue.number, + state: 'closed', + state_reason: 'not_planned', + labels: ['invalid'] + }); + return; + } + if (confirmHasRead) { + comment = "感谢您联系OpenList。我们会尽快回复您。\n"; + comment += "Thanks for contacting OpenList. We will reply to you as soon as possible.\n\n"; comment += "由于您提出的 Issue 中包含部分未确认的项目，为了更好地管理项目，在人工审核后可能会直接关闭此问题。\n"; comment += "如果您能确认并补充相关未确认项目的信息，欢迎随时重新提交。我们会及时关注并处理。感谢您的理解与支持！\n"; comment += "Since your issue contains some unchecked tasks, it may be closed after manual review.\n"; @@ -31,28 +69,30 @@ jobs: comment += "We will pay attention and handle it in a timely manner.\n\n"; comment += "感谢您的理解与支持！\n"; comment += "Thank you for your understanding and support!\n"; + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: comment + }); } - await github.rest.issues.createComment({ - ...context.repo, - issue_number: context.issue.number, - body: comment - }); pr-title-check: runs-on: ubuntu-latest if: github.event_name == 'pull_request' steps: - name: Check PR title for required prefix and comment - uses: actions/github-script@v7 + uses: actions/github-script@v8 with: script: | const title = context.payload.pull_request.title || ""; - const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\): /i.test(title); + const ok = /^(feat|docs|fix|style|refactor|chore)\(.+?\)!?: /i.test(title); if (!ok) { let comment = "⚠️ PR 标题需以 `feat(): `, `docs(): `, `fix(): `, `style(): `, `refactor(): `, `chore(): ` 其中之一开头，例如：`feat(component): 新增功能`。\n"; comment += "⚠️ The PR title must start with `feat(): `, `docs(): `, `fix(): `, `style(): `, or `refactor(): `, `chore(): `. For example: `feat(component): add new feature`.\n\n"; comment += "如果跨多个组件，请使用主要组件作为前缀，并在标题中枚举、描述中说明。\n"; comment += "If it spans multiple components, use the main component as the prefix and enumerate in the title, describe in the body.\n\n"; + comment += "如果是破坏性变更，请在类型后添加 `!`，例如 `feat(component)!: 破坏性变更`。\n"; + comment += "For breaking changes, add `!` after the type, e.g., `feat(component)!: breaking change`.\n\n"; await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9bbf0f1ca..76152ef24 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,12 +44,12 @@ jobs: swap-storage: true - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: '1.25.0' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 80c065647..7503a3eda 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -43,15 +43,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version: '1.25.0' - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -69,7 +69,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ env.ARTIFACT_NAME }} overwrite: true @@ -83,15 +83,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version: '1.25.0' - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 @@ -109,7 +109,7 @@ jobs: FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: ${{ env.ARTIFACT_NAME_LITE }} overwrite: true @@ -146,8 +146,8 @@ jobs: tag_favor: "suffix=-aio,onlatest=true" steps: - name: Checkout - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' @@ -230,8 +230,8 @@ jobs: tag_favor: "suffix=-lite-aio,onlatest=true" steps: - name: Checkout - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 with: name: ${{ env.ARTIFACT_NAME_LITE }} path: 'build/' diff --git a/.github/workflows/sync_repo.yml b/.github/workflows/sync_repo.yml index 621d5fb5d..fe57905dc 100644 --- a/.github/workflows/sync_repo.yml +++ b/.github/workflows/sync_repo.yml @@ -12,7 +12,7 @@ jobs: name: Sync GitHub to Gitee steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index aa6fe8966..33999732e 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -1,150 +1,196 @@ name: Beta Release (Docker) - on: workflow_dispatch: + inputs: + frontend_repo: + description: 'Frontend repo, e.g. Ironboxplus/OpenList-Frontend' + required: false + default: 'OpenListTeam/OpenList-Frontend' + type: string + frontend_channel: + description: 'Frontend release channel to build' + required: false + default: rolling + type: choice + options: + - rolling + - latest + - both push: branches: - main pull_request: branches: - - main + - copy + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: - DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} - GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} - IMAGE_NAME: openlist-git - IMAGE_NAME_DOCKERHUB: openlist + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 + FRONTEND_REPO: ${{ github.event.inputs.frontend_repo || vars.FRONTEND_REPO || 'OpenListTeam/OpenList-Frontend' }} + IMAGE_NAME: openlist REGISTRY: ghcr.io - ARTIFACT_NAME: 'binaries_docker_release' - RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. - IMAGE_PUSH: ${{ github.event_name == 'push' }} - IMAGE_TAGS_BETA: | - type=ref,event=pr - type=raw,value=beta,enable={{is_default_branch}} + ARTIFACT_NAME_PREFIX: 'binaries_docker_release' + # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 + RELEASE_PLATFORMS: 'linux/amd64' + # 👇 关键修改：强制允许推送，不用管是不是 push 事件 + IMAGE_PUSH: 'true' + # 👇 使用默认的前端仓库 (OpenListTeam/OpenList-Frontend) + # FRONTEND_REPO: 'Ironboxplus/OpenList-Frontend' jobs: build_binary: - name: Build Binaries for Docker Release (Beta) + name: Build Binaries (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest + strategy: + matrix: + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["latest","rolling"]') }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - - uses: actions/setup-go@v5 + - name: Setup Go + uses: actions/setup-go@v6 with: go-version: '1.25.0' + cache: true + cache-dependency-path: go.sum + + - name: Get Frontend Cache Version + id: frontend-cache + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WEB_VERSION: ${{ matrix.frontend_channel }} + run: | + frontend_repo="${{ env.FRONTEND_REPO }}" + web_version="$WEB_VERSION" + github_auth_args=() + + if [ -n "$GH_TOKEN" ]; then + github_auth_args=(-H "Authorization: Bearer $GH_TOKEN") + fi + + if [ "$web_version" = "latest" ]; then + frontend_version=$(curl -fsSL "${github_auth_args[@]}" "https://api.github.com/repos/$frontend_repo/releases/latest" | jq -r '.tag_name') + else + frontend_version="$web_version" + fi + + echo "repo=$frontend_repo" >> "$GITHUB_OUTPUT" + echo "version=$frontend_version" >> "$GITHUB_OUTPUT" + echo "Frontend repo: $frontend_repo" + echo "Frontend cache version: $frontend_version" + + - name: Cache Frontend + id: cache-frontend + uses: actions/cache@v5 + with: + path: public/dist + key: frontend-${{ steps.frontend-cache.outputs.repo }}-${{ steps.frontend-cache.outputs.version }} + restore-keys: | + frontend-${{ steps.frontend-cache.outputs.repo }}- - name: Cache Musl id: cache-musl - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: build/musl-libs key: docker-musl-libs-v2 - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' - run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash build.sh prepare docker-multiplatform - - name: Build go binary (beta) + - name: Build go binary run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - FRONTEND_REPO: ${{ vars.FRONTEND_REPO }} + WEB_VERSION: ${{ matrix.frontend_channel }} + FRONTEND_REPO: ${{ env.FRONTEND_REPO }} + SKIP_FRONTEND_FETCH: ${{ steps.cache-frontend.outputs.cache-hit == 'true' && 'true' || 'false' }} - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: - name: ${{ env.ARTIFACT_NAME }} + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} overwrite: true - path: | - build/ - !build/*.tgz - !build/musl-libs/** + path: build/linux/amd64/openlist release_docker: needs: build_binary - name: Release Docker image (Beta) + name: Release Docker (x64, front-${{ matrix.frontend_channel }}) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: + frontend_channel: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.frontend_channel == 'both' && fromJSON('["rolling","latest"]') || fromJSON(format('["{0}"]', github.event.inputs.frontend_channel))) || fromJSON('["latest","rolling"]') }} + # 构建所有变体 image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" base_image_tag: "base" build_arg: "" - tag_favor: "" + image_tag_suffix: "" - image: "ffmpeg" base_image_tag: "ffmpeg" build_arg: INSTALL_FFMPEG=true - tag_favor: "suffix=-ffmpeg,onlatest=true" + image_tag_suffix: "-ffmpeg" - image: "aria2" base_image_tag: "aria2" build_arg: INSTALL_ARIA2=true - tag_favor: "suffix=-aria2,onlatest=true" + image_tag_suffix: "-aria2" - image: "aio" base_image_tag: "aio" build_arg: | INSTALL_FFMPEG=true INSTALL_ARIA2=true - tag_favor: "suffix=-aio,onlatest=true" + image_tag_suffix: "-aio" steps: - name: Checkout - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 + uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 with: - name: ${{ env.ARTIFACT_NAME }} - path: 'build/' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - + name: ${{ env.ARTIFACT_NAME_PREFIX }}-${{ matrix.frontend_channel }} + path: 'build/linux/amd64' - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # 👇 只保留 GitHub 登录，删除了 DockerHub 登录 - name: Login to GitHub Container Registry - if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to DockerHub Container Registry - if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} - ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} - tags: ${{ env.IMAGE_TAGS_BETA }} - flavor: | - ${{ matrix.tag_favor }} + tags: | + type=raw,value=front-${{ matrix.frontend_channel }}${{ matrix.image_tag_suffix }} + type=raw,value=latest,enable=${{ matrix.frontend_channel == 'latest' && matrix.image == 'latest' }} - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci - push: ${{ env.IMAGE_PUSH == 'true' }} + push: true build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ env.RELEASE_PLATFORMS }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}:buildcache-front-${{ matrix.frontend_channel }}-${{ matrix.image }},mode=max diff --git a/.gitignore b/.gitignore index 1d71f0d60..d42155110 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .DS_Store output/ /dist/ - +.omx # Binaries for programs and plugins *.exe *.exe~ @@ -31,4 +31,5 @@ output/ /public/dist/* /!public/dist/README.md -.VSCodeCounter \ No newline at end of file +.VSCodeCounter +nul diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..c3ffdded0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,346 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Core Development Principles + +1. **最小代码改动原则** (Minimum code changes): Make the smallest change necessary to achieve the goal +2. **不缓存整个文件原则** (No full file caching for seekable streams): For SeekableStream, use RangeRead instead of caching entire file +3. **必要情况下可以多遍上传原则** (Multi-pass upload when necessary): If rapid upload fails, fall back to normal upload + +## Build and Development Commands + +```bash +# Development +go run main.go # Run backend server (default port 5244) +air # Hot reload during development (uses .air.toml) +./build.sh dev # Build development version with frontend +./build.sh release # Build release version + +# Testing +go test ./... # Run all tests + +# Docker +docker-compose up # Run with docker-compose +docker build -f Dockerfile . # Build docker image +``` + +**Build Script Details** (`build.sh`): +- Fetches frontend from OpenListTeam/OpenList-Frontend releases +- Injects version info via ldflags: `-X "github.com/OpenListTeam/OpenList/v4/internal/conf.BuiltAt=$(date +'%F %T %z')"` +- Supports `dev`, `beta`, and release builds +- Downloads prebuilt frontend distribution automatically + +**Go Version**: Requires Go 1.23.4+ + +## Architecture Overview + +### Driver System (Storage Abstraction) + +OpenList uses a **driver pattern** to support 70+ cloud storage providers. Each driver implements the core `Driver` interface. + +**Location**: `drivers/*/` + +**Core Interfaces** (`internal/driver/driver.go`): +- `Reader`: List directories, generate download links (REQUIRED) +- `Writer`: Upload, delete, move files (optional) +- `ArchiveDriver`: Extract archives (optional) +- `LinkCacheModeResolver`: Custom cache TTL strategies (optional) + +**Driver Registration Pattern**: +```go +// In drivers/your_driver/meta.go +var config = driver.Config{ + Name: "YourDriver", + LocalSort: false, + NoCache: false, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &YourDriver{} + }) +} +``` + +**Adding a New Driver**: +1. Copy `drivers/template/` to `drivers/your_driver/` +2. Implement `List()` and `Link()` methods (required) +3. Define `Addition` struct with configuration fields using struct tags: + - `json:"field_name"` - JSON field name + - `type:"select"` - Input type (select, string, text, bool, number) + - `required:"true"` - Required field + - `options:"a,b,c"` - Dropdown options + - `default:"value"` - Default value +4. Register driver in `init()` function + +**Example Driver Structure**: +```go +type YourDriver struct { + model.Storage + Addition + client *YourClient +} + +func (d *YourDriver) Init(ctx context.Context) error { + // Initialize client, login, etc. +} + +func (d *YourDriver) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + // Return list of files/folders +} + +func (d *YourDriver) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + // Return download URL or RangeReader +} +``` + +### Request Flow + +``` +HTTP Request (Gin Router) + ↓ +Middleware (Auth, CORS, Logging) + ↓ +Handler (server/handles/) + ↓ +fs.List/Get/Link (mount path → storage path conversion) + ↓ +op.List/Get/Link (caching, driver lookup) + ↓ +Driver.List/Link (storage-specific API calls) + ↓ +Response (JSON / Proxy / Redirect) +``` + +### Internal Package Structure + +| Package | Purpose | +|---------|---------| +| `bootstrap/` | Initialization sequence: config, DB, storages, servers | +| `conf/` | Configuration management | +| `db/` | Database models (SQLite/MySQL/Postgres) | +| `driver/` | Driver interface definitions | +| `fs/` | Mount path abstraction (converts `/mount/path` to storage + path) | +| `op/` | Core operations with caching and driver management | +| `stream/` | Streaming, range readers, link refresh, rate limiting | +| `model/` | Data models (Obj, Link, Storage, User) | +| `cache/` | Multi-level caching (directories, links, users, settings) | +| `net/` | HTTP utilities, proxy config, download manager | + +### Link Generation and Caching + +**Link Types**: +1. **Direct URL** (`link.URL`): Simple redirect to storage provider +2. **RangeReader** (`link.RangeReader`): Custom streaming implementation +3. **Refreshable Link** (`link.Refresher`): Auto-refresh on expiration + +**Cache System** (`internal/op/cache.go`): +- **Directory Cache**: Stores file listings with configurable TTL +- **Link Cache**: Stores download URLs (30min default) +- **User Cache**: Authentication data (1hr default) +- **Custom Policies**: Pattern-based TTL via `pattern:ttl` format + +**Cache Key Pattern**: `{storageMountPath}/{relativePath}` + +**Invalidation**: Recursive tree deletion for directory operations + +### Range Reader and Streaming + +**Location**: `internal/stream/` + +**Purpose**: Handle partial content requests (HTTP 206), multi-threaded downloads, and link refresh during streaming. + +**Key Components**: + +1. **RangeReaderIF**: Core interface for range-based reading + ```go + type RangeReaderIF interface { + RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) + } + ``` + +2. **RefreshableRangeReader**: Wraps RangeReader with automatic link refresh + - Detects expired links via error strings or HTTP status codes (401, 403, 410, 500) + - Calls `link.Refresher(ctx)` to get new link + - Resumes download from current byte position + - Max 3 refresh attempts to prevent infinite loops + +3. **Multi-threaded Downloader** (`internal/net/downloader.go`): + - Splits file into parts based on `Concurrency` and `PartSize` + - Downloads parts in parallel + - Assembles final stream + +**Stream Types and Reader Management**: + +⚠️ **CRITICAL**: SeekableStream.Reader must NEVER be created early! + +- **FileStream**: One-time sequential stream (e.g., HTTP body) + - `Reader` is set at creation and consumed sequentially + - Cannot be rewound or re-read + +- **SeekableStream**: Reusable stream with RangeRead capability + - Has `rangeReader` for creating new readers on-demand + - `Reader` should ONLY be created when actually needed for sequential reading + - **DO NOT create Reader early** - use lazy initialization via `generateReader()` + +**Common Pitfall - Early Reader Creation**: +```go +// ❌ WRONG: Creating Reader early +if _, ok := rr.(*model.FileRangeReader); ok { + rc, _ := rr.RangeRead(ctx, http_range.Range{Length: -1}) + fs.Reader = rc // This will be consumed by intermediate operations! +} + +// ✅ CORRECT: Let generateReader() create it on-demand +// Reader will be created only when Read() is called +return &SeekableStream{FileStream: fs, rangeReader: rr}, nil +``` + +**Why This Matters**: +- Hash calculation uses `StreamHashFile()` which reads the file via RangeRead +- If Reader is created early, it may be at EOF when HTTP upload actually needs it +- Result: `http: ContentLength=X with Body length 0` error + +**Hash Calculation for Uploads**: +```go +// For SeekableStream: Use RangeRead to avoid consuming Reader +if _, ok := file.(*SeekableStream); ok { + hash, err = stream.StreamHashFile(file, utils.MD5, 40, &up) + // StreamHashFile uses RangeRead internally, Reader remains unused +} + +// For FileStream: Must cache first, then calculate hash +_, hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) +``` + +**Link Refresh Pattern**: +```go +// In op.Link(), a refresher is automatically attached +link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + // Get fresh link from storage driver + file, err := GetUnwrap(refreshCtx, storage, path) + newLink, err := storage.Link(refreshCtx, file, args) + return newLink, file, nil +} + +// RefreshableRangeReader uses this during streaming +if IsLinkExpiredError(err) && r.link.Refresher != nil { + newLink, _, err := r.link.Refresher(ctx) + // Resume from current position +} +``` + +**Proxy Function** (`server/common/proxy.go`): + +Handles multiple scenarios: +1. Multi-threaded download (`link.Concurrency > 0`) +2. Direct RangeReader (`link.RangeReader != nil`) +3. Refreshable link (`link.Refresher != nil`) ← Wraps with RefreshableRangeReader +4. Transparent proxy (forwards to `link.URL`) + +### Startup Sequence + +**Location**: `internal/bootstrap/run.go` + +Order of initialization: +1. `InitConfig()` - Load config, environment variables +2. `Log()` - Initialize logging +3. `InitDB()` - Connect to database +4. `data.InitData()` - Initialize default data +5. `LoadStorages()` - Load and initialize all storage drivers +6. `InitTaskManager()` - Start background tasks +7. `Start()` - Start HTTP/HTTPS/WebDAV/FTP/SFTP servers + +## Common Patterns + +### Error Handling + +Use custom errors from `internal/errs/`: +- `errs.NotImplement` - Feature not implemented +- `errs.ObjectNotFound` - File/folder not found +- `errs.NotFolder` - Path is not a directory +- `errs.StorageNotInit` - Storage driver not initialized + +**Link Expiry Detection**: +```go +// Checks error string for keywords: "expired", "invalid signature", "token expired" +// Also checks HTTP status: 401, 403, 410, 500 +if stream.IsLinkExpiredError(err) { + // Refresh link +} +``` + +### Saving Driver State + +When updating tokens or credentials: +```go +d.AccessToken = newToken +op.MustSaveDriverStorage(d) // Persists to database +``` + +### Rate Limiting + +Use `rate.Limiter` for API rate limits: +```go +type YourDriver struct { + limiter *rate.Limiter +} + +func (d *YourDriver) Init(ctx context.Context) error { + d.limiter = rate.NewLimiter(rate.Every(time.Second), 1) // 1 req/sec +} + +func (d *YourDriver) List(...) { + d.limiter.Wait(ctx) + // Make API call +} +``` + +### Context Cancellation + +Always respect context cancellation in long operations: +```go +select { +case <-ctx.Done(): + return nil, ctx.Err() +default: + // Continue operation +} +``` + +## Important Conventions + +**Naming**: +- Drivers: lowercase with underscores (e.g., `baidu_netdisk`, `aliyundrive_open`) +- Packages: lowercase (e.g., `internal/op`) +- Interfaces: PascalCase with suffix (e.g., `Reader`, `Writer`) + +**Driver Configuration Fields**: +- Use `driver.RootPath` or `driver.RootID` for root folder +- Add `omitempty` to optional JSON fields +- Use descriptive help text in struct tags + +**Retries and Timeouts**: +- Use `github.com/avast/retry-go` for retry logic +- Set reasonable timeouts on HTTP clients (default 30s in `base.RestyClient`) +- For unstable APIs, implement exponential backoff + +**Logging**: +- Use `logrus` via `log` package +- Levels: `log.Debugf`, `log.Infof`, `log.Warnf`, `log.Errorf` +- Include driver name in logs: `log.Infof("[driver_name] message")` + +## Project Context + +OpenList is a community-driven fork of AList, focused on: +- Long-term governance and trust +- Support for 70+ cloud storage providers +- Web UI for file management +- Multi-protocol support (HTTP, WebDAV, FTP, SFTP, S3) +- Offline downloads (Aria2, Transmission) +- Full-text search +- Archive extraction + +**License**: AGPL-3.0 diff --git a/COMPATIBILITY_REPORT.md b/COMPATIBILITY_REPORT.md new file mode 100644 index 000000000..0b8be3b52 --- /dev/null +++ b/COMPATIBILITY_REPORT.md @@ -0,0 +1,204 @@ +# Rebase兼容性分析报告 + +## 提交概览 +共引入 **21个commits**，主要涉及以下模块： + +### 核心功能改动 + +#### 1. **链接刷新机制** (`internal/stream/util.go`) +**Commits**: +- `4c33ffa4` feat(link): add link refresh capability for expired download links +- `f38fe180` fix(stream): 修复链接过期检测逻辑，避免将上下文取消视为链接过期 +- `7cf362c6` fix(stream): 更新过期链接检查逻辑，支持所有4xx客户端错误 +- `03fbaf1c` refactor(stream): 移除过时的链接刷新逻辑，添加自愈读取器以处理0字节读取 + +**核心代码**: +```go +// 新增常量 +MAX_LINK_REFRESH_COUNT = 50 // 链接最大刷新次数 +MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead重试次数（从3提升到5） + +// 新增函数 +IsLinkExpiredError(err error) bool // 判断是否为链接过期错误 + +// 新增结构 +RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // 防止无限循环 +} + +selfHealingReadCloser struct { + // 检测0字节读取，自动刷新链接 +} +``` + +**功能说明**: +1. **链接过期检测**: 识别多种云盘的过期错误（expired, token expired, access denied, 4xx状态码等） +2. **自动刷新**: 检测到过期时自动调用Refresher获取新链接，最多刷新50次 +3. **自愈机制**: 处理某些云盘返回200但内容为空的情况（0字节读取检测） +4. **并发安全**: 使用sync.Mutex保护共享状态 +5. **Context隔离**: 刷新时使用WithoutCancel避免用户取消操作影响刷新 + +**潜在风险**: +- ✅ Context.WithoutCancel需要Go 1.21+ +- ✅ 并发场景下的锁竞争 +- ✅ refreshCount可能在某些场景下不递增导致无限循环 + +--- + +#### 2. **目录预创建优化** (`internal/fs/copy_move.go`) +**Commit**: `ce0da112` fix(copy_move): 将预创建子目录的深度从2级调整为1级 + +**核心代码**: +```go +func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, dstBasePath string, maxDepth int) error { + // 第一轮：创建直接子目录 + for _, obj := range objs { + if obj.IsDir() { + subdirPath := stdpath.Join(dstBasePath, obj.GetName()) + op.MakeDir(t.Ctx(), t.DstStorage, subdirPath) + subdirs = append(subdirs, obj) + } + } + + // 停止递归条件 + if maxDepth <= 0 { + return nil + } + + // 第二轮：递归创建嵌套目录 + for _, subdir := range subdirs { + subObjs := op.List(...) + preCreateDirectoryTree(subObjs, subdirDstPath, maxDepth-1) + } +} +``` + +**功能说明**: +1. **深度控制**: 默认maxDepth=1，只预创建2级目录（当前+子级） +2. **防止深度递归**: 避免在大型项目中递归过深导致栈溢出或性能问题 +3. **错误容忍**: MakeDir失败时继续处理其他目录 +4. **Context感知**: 每次循环检查ctx.Err()支持取消操作 + +**潜在风险**: +- ✅ op.MakeDir和op.List调用需要存储初始化 +- ✅ 大量目录时的性能问题 +- ✅ Context取消时的资源清理 + +--- + +#### 3. **网络优化** (`drivers/`, `internal/net/`) +**Commits**: +- `b9dafa65` feat(network): 增加对慢速网络的支持，调整超时和重试机制 +- `bce47884` fix(driver): 增加夸克分片大小调整逻辑，支持重试机制 +- `0b8471f6` feat(quark_open): 添加速率限制和重试逻辑 + +**功能说明**: +1. 提升RangeRead重试次数: 3 → 5 +2. 调整网络超时参数 +3. 添加分片上传重试逻辑 + +--- + +#### 4. **驱动修复** +**Commits**: +- `da2812c0` fix(google_drive): 更新Put方法以支持可重复读取流和不可重复读取流的MD5校验 +- `5a6bad90` feat(google_drive): 添加文件夹创建的锁机制和重试逻辑 +- `a54b2388` feat(google_drive): 添加处理重复文件名的功能 +- `9ef22ec9` fix(driver): fix file copy failure to 123pan due to incorrect etag +- `0ead87ef` fix(alias): update storage retrieval method in listRoot function +- `311f6246` fix: 修复500 panic和NaN问题 + +--- + +## 兼容性评估 + +### ✅ 编译兼容性 +- 构建成功，无语法错误 +- 依赖版本无冲突 + +### ✅ API兼容性 +- 新增函数不破坏现有接口 +- RefreshableRangeReader实现model.RangeReaderIF接口 +- 向后兼容旧代码 + +### ⚠️ 运行时兼容性 +**需要验证的场景**: +1. **并发安全**: RefreshableRangeReader的并发读取 +2. **资源泄漏**: Context取消时goroutine是否正确退出 +3. **边界条件**: + - refreshCount达到50次的行为 + - 0字节读取检测的准确性 + - maxDepth=0时的目录创建 +4. **错误处理**: + - nil Refresher时的处理 + - 链接刷新失败时的回退机制 +5. **性能**: + - 大文件下载时的刷新开销 + - 深层目录结构的预创建性能 + +--- + +## 测试需求 + +### 必须测试的场景 + +#### Stream包测试 +1. **IsLinkExpiredError准确性** + - 各种云盘的过期错误格式 + - Context取消不应判断为过期 + - HTTP 4xx/5xx的区分 + +2. **RefreshableRangeReader可靠性** + - 正常读取流程 + - 自动刷新触发和成功 + - 达到最大刷新次数 + - 并发读取安全性 + - Context取消的正确处理 + +3. **selfHealingReadCloser** + - 0字节读取检测 + - 刷新重试机制 + - 资源正确关闭 + +#### FS包测试 +1. **preCreateDirectoryTree** + - 深度控制正确性(0, 1, 2级) + - 大量目录的性能 + - Context取消的响应 + - 错误容忍性 + +--- + +## 风险等级: **中等** + +**原因**: +- ✅ 新功能设计合理，有明确的边界和错误处理 +- ⚠️ 并发场景需要充分测试 +- ⚠️ 链接刷新逻辑复杂，需要验证各种边界情况 +- ⚠️ 依赖op包的函数需要正确的初始化 + +--- + +## 推送建议: **通过测试后可推送** + +**前置条件**: +1. 完成全面的单元测试（见下方测试代码） +2. 验证并发安全性 +3. 确认Context取消不会导致资源泄漏 +4. 性能测试通过（大文件、深层目录） + +**建议测试命令**: +```bash +# 单元测试 +go test ./internal/stream ./internal/fs -v -count=1 -race + +# 压力测试 +go test ./internal/stream -run Stress -v -count=10 + +# 完整测试套件 +go test ./... -short -count=1 +``` diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..ff558d64e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,89 @@ +# Security Policy + +## Supported Versions + +Only the latest stable release receives security patches. We strongly recommend always keeping OpenList up to date. + +| Version | Supported | +| -------------------- | ------------------ | +| Latest stable (v4.x) | :white_check_mark: | +| Older versions | :x: | + +## Reporting a Vulnerability + +**Please do NOT report security vulnerabilities through public GitHub Issues.** + +If you discover a security vulnerability in OpenList, please report it responsibly by using one of the following channels: + +- **GitHub Private Security Advisory** (preferred): [Submit here](https://github.com/OpenListTeam/OpenList/security/advisories/new) +- **Telegram**: Contact a maintainer privately via [@OpenListTeam](https://t.me/OpenListTeam) + +When reporting, please include as much of the following as possible: + +- A description of the vulnerability and its potential impact +- The affected version(s) +- Step-by-step instructions to reproduce the issue +- Any proof-of-concept code or screenshots (if applicable) +- Suggested mitigation or fix (optional but appreciated) + +## Security Best Practices for Users + +To keep your OpenList instance secure: + +- Always update to the latest release. +- Use a strong, unique admin password and change it after first login. +- Enable HTTPS (TLS) for your deployment — do **not** expose OpenList over plain HTTP on the public internet. +- Limit exposed ports using a reverse proxy (e.g., Nginx, Caddy). +- Set up access controls and avoid enabling guest access unless necessary. +- Regularly review mounted storage permissions and revoke unused API tokens. +- When using Docker, avoid running the container as root if possible. + +## Acknowledgments + +We sincerely thank all security researchers and community members who responsibly disclose vulnerabilities and help make OpenList safer for everyone. + +--- + +# 安全政策 + +## 支持的版本 + +我们仅对最新稳定版本提供安全补丁。强烈建议始终保持 OpenList 为最新版本。 + +| 版本 | 是否支持 | +| ------------------ | ------------------ | +| 最新稳定版（v4.x） | :white_check_mark: | +| 旧版本 | :x: | + +## 报告漏洞 + +**请勿通过公开的 GitHub Issues 报告安全漏洞。** + +如果您在 OpenList 中发现安全漏洞，请通过以下渠道之一负责任地进行报告： + +- **GitHub 私密安全公告**（推荐）：[点击提交](https://github.com/OpenListTeam/OpenList/security/advisories/new) +- **Telegram**：通过 [@OpenListTeam](https://t.me/OpenListTeam) 私信联系维护者 + +报告时，请尽量提供以下信息： + +- 漏洞描述及其潜在影响 +- 受影响的版本 +- 复现问题的详细步骤 +- 概念验证代码或截图（如有） +- 建议的缓解措施或修复方案（可选，但非常欢迎） + +## 用户安全最佳实践 + +为保障您的 OpenList 实例安全： + +- 始终更新至最新版本。 +- 使用强且唯一的管理员密码，并在首次登录后立即修改。 +- 为您的部署启用 HTTPS（TLS）—— **请勿**在公网上以明文 HTTP 方式暴露 OpenList。 +- 使用反向代理（如 Nginx、Caddy）限制对外暴露的端口。 +- 配置访问控制，非必要情况下不要开启访客访问。 +- 定期检查已挂载存储的权限，并撤销未使用的 API 令牌。 +- 使用 Docker 部署时，尽可能避免以 root 用户运行容器。 + +## 致谢 + +我们衷心感谢所有负责任地披露漏洞、帮助 OpenList 变得更加安全的安全研究人员和社区成员。 diff --git a/build.sh b/build.sh index 26e5a301b..d37ddad73 100644 --- a/build.sh +++ b/build.sh @@ -18,6 +18,8 @@ if [[ "$*" == *"lite"* ]]; then useLite=true fi +skipFrontendFetch="${SKIP_FRONTEND_FETCH:-false}" + if [ "$1" = "dev" ]; then version="dev" webVersion="rolling" @@ -31,6 +33,10 @@ else webVersion=$(eval "curl -fsSL --max-time 2 $githubAuthArgs \"https://api.github.com/repos/$frontendRepo/releases/latest\"" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi +if [ -n "$WEB_VERSION" ]; then + webVersion="$WEB_VERSION" +fi + echo "backend version: $version" echo "frontend version: $webVersion" if [ "$useLite" = true ]; then @@ -46,9 +52,63 @@ ldflags="\ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.GitCommit=$gitCommit' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.Version=$version' \ -X 'github.com/OpenListTeam/OpenList/v4/internal/conf.WebVersion=$webVersion' \ +-X 'github.com/OpenListTeam/OpenList/v4/internal/conf.FrontendRepoDefault=$frontendRepo' \ " +# Keep sqlite driver tag selection centralized to avoid target drift. +GetBuildTagsForTarget() { + local target="$1" + case "$target" in + linux-loong64|linux-mips|linux-mips64|linux-mips64le|linux-mipsle|linux-musl-loong64|linux-musl-mips|linux-musl-mips64|linux-musl-mips64le|linux-musl-mipsle|windows-386|windows7-386|windows7-amd64) + echo "jsoniter,sqlite_cgo_compat" + ;; + *) + echo "jsoniter" + ;; + esac +} + +# Keep musl static link flags centralized for all musl build paths. +GetMuslStaticLdflags() { + echo "-linkmode external -extldflags '-static -fpic' $ldflags" +} + +# Fail fast if a musl build artifact is not fully static. +AssertStaticBinary() { + local binary="$1" + if [ ! -f "$binary" ]; then + echo "Error: binary not found: $binary" + return 1 + fi + + if command -v readelf >/dev/null 2>&1; then + if readelf -l "$binary" 2>/dev/null | grep -q "Requesting program interpreter"; then + echo "Error: binary is not fully static: $binary" + readelf -l "$binary" | grep "Requesting program interpreter" || true + return 1 + fi + return 0 + fi + + if command -v file >/dev/null 2>&1; then + if file "$binary" | grep -qi "dynamically linked"; then + echo "Error: binary is dynamically linked: $binary" + file "$binary" + return 1 + fi + return 0 + fi + + echo "Warning: readelf/file not found, skip static verification for $binary" + return 0 +} + FetchWebRolling() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + pre_release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/tags/rolling\"") pre_release_assets=$(echo "$pre_release_json" | jq -r '.assets[].browser_download_url') @@ -62,6 +122,11 @@ FetchWebRolling() { } FetchWebRelease() { + if [ "$skipFrontendFetch" = "true" ] && [ -n "$(find public/dist -mindepth 1 -print -quit 2>/dev/null)" ]; then + echo "using cached frontend dist from public/dist" + return 0 + fi + release_json=$(eval "curl -fsSL --max-time 2 $githubAuthArgs -H \"Accept: application/vnd.github.v3+json\" \"https://api.github.com/repos/$frontendRepo/releases/latest\"") release_assets=$(echo "$release_json" | jq -r '.assets[].browser_download_url') @@ -110,6 +175,7 @@ BuildWin7() { # Build for both 386 and amd64 architectures for arch in "386" "amd64"; do echo "building for windows7-${arch}" + build_tags=$(GetBuildTagsForTarget "windows7-${arch}") export GOOS=windows export GOARCH=${arch} export CGO_ENABLED=1 @@ -124,14 +190,14 @@ BuildWin7() { fi # Use the patched Go compiler for Win7 compatibility - $(pwd)/go-win7/bin/go build -o "${1}-${arch}.exe" -ldflags="$ldflags" -tags=jsoniter . + $(pwd)/go-win7/bin/go build -o "${1}-${arch}.exe" -ldflags="$ldflags" -tags="$build_tags" . done } BuildDev() { rm -rf .git/ mkdir -p "dist" - muslflags="--extldflags '-static -fpic' $ldflags" + muslflags="$(GetMuslStaticLdflags)" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross) for i in "${FILES[@]}"; do @@ -149,7 +215,8 @@ BuildDev() { export GOARCH=${os_arch##*-} export CC=${cgo_cc} export CGO_ENABLED=1 - go build -o ./dist/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + CGO_LDFLAGS="-static" go build -o ./dist/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + AssertStaticBinary "./dist/$appName-$os_arch" done xgo -targets=windows/amd64,darwin/amd64,darwin/arm64 -out "$appName" -ldflags="$ldflags" -tags=jsoniter . mv "$appName"-* dist @@ -183,21 +250,23 @@ BuildDockerMultiplatform() { # run PrepareBuildDockerMusl before build export PATH=$PATH:$PWD/build/musl-libs/bin - docker_lflags="--extldflags '-static -fpic' $ldflags" + docker_lflags="$(GetMuslStaticLdflags)" export CGO_ENABLED=1 - OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds - CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds + OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds + CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} os=${os_arch%%-*} arch=${os_arch##*-} + build_tags=$(GetBuildTagsForTarget "$os_arch") export GOOS=$os export GOARCH=$arch export CC=${cgo_cc} echo "building for $os_arch" - go build -o build/$os/$arch/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + CGO_LDFLAGS="-static" go build -o build/$os/$arch/"$appName" -ldflags="$docker_lflags" -tags="$build_tags" . + AssertStaticBinary "build/$os/$arch/$appName" done DOCKER_ARM_ARCHES=(linux-arm/v6 linux-arm/v7) @@ -205,14 +274,17 @@ BuildDockerMultiplatform() { GO_ARM=(6 7) export GOOS=linux export GOARCH=arm - for i in "${!DOCKER_ARM_ARCHES[@]}"; do - docker_arch=${DOCKER_ARM_ARCHES[$i]} - cgo_cc=${CGO_ARGS[$i]} - export GOARM=${GO_ARM[$i]} - export CC=${cgo_cc} - echo "building for $docker_arch" - go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . - done + # ARM docker variants stay disabled on this branch to keep the workflow x64-only. + # If they are re-enabled later, they should follow the same static-link pattern. + # for i in "${!DOCKER_ARM_ARCHES[@]}"; do + # docker_arch=${DOCKER_ARM_ARCHES[$i]} + # cgo_cc=${CGO_ARGS[$i]} + # export GOARM=${GO_ARM[$i]} + # export CC=${cgo_cc} + # echo "building for $docker_arch" + # CGO_LDFLAGS="-static" go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + # AssertStaticBinary "build/${docker_arch%%-*}/${docker_arch##*-}/$appName" + # done } BuildRelease() { @@ -237,6 +309,8 @@ BuildLoongGLIBC() { local target_abi="$2" local output_file="$1" local oldWorldGoVersion="1.25.0" + local loong_tags + loong_tags=$(GetBuildTagsForTarget "linux-loong64") if [ "$target_abi" = "abi1.0" ]; then echo building for linux-loong64-abi1.0 @@ -311,7 +385,7 @@ BuildLoongGLIBC() { CXX="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++" \ CGO_ENABLED=1 \ GOCACHE="$abi1_cache_dir" \ - $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then + $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags="$loong_tags" .; then echo "Error: Build failed with patched Go compiler" echo "Attempting retry with cache cleanup..." env GOCACHE="$abi1_cache_dir" $(pwd)/go-loong64-abi1.0/bin/go clean -cache @@ -320,7 +394,7 @@ BuildLoongGLIBC() { CXX="$(pwd)/gcc8-loong64-abi1.0/bin/loongarch64-linux-gnu-g++" \ CGO_ENABLED=1 \ GOCACHE="$abi1_cache_dir" \ - $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then + $(pwd)/go-loong64-abi1.0/bin/go build -a -o "$output_file" -ldflags="$ldflags" -tags="$loong_tags" .; then echo "Error: Build failed again after cache cleanup" echo "Build environment details:" echo "GOOS=linux" @@ -366,11 +440,11 @@ BuildLoongGLIBC() { # Use standard Go compiler for new-world build echo "Building with standard Go compiler for new-world ABI2.0..." - if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then + if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags="$loong_tags" .; then echo "Error: Build failed with standard Go compiler" echo "Attempting retry with cache cleanup..." go clean -cache - if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags=jsoniter .; then + if ! go build -a -o "$output_file" -ldflags="$ldflags" -tags="$loong_tags" .; then echo "Error: Build failed again after cache cleanup" echo "Build environment details:" echo "GOOS=$GOOS" @@ -389,8 +463,9 @@ BuildLoongGLIBC() { BuildReleaseLinuxMusl() { rm -rf .git/ mkdir -p "build" - muslflags="--extldflags '-static -fpic' $ldflags" + muslflags="$(GetMuslStaticLdflags)" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" + # Keep mips-family targets enabled; sqlite driver selection is handled by Go build tags. FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross mips-linux-musl-cross mips64-linux-musl-cross mips64el-linux-musl-cross mipsel-linux-musl-cross powerpc64le-linux-musl-cross s390x-linux-musl-cross loongarch64-linux-musl-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" @@ -403,19 +478,21 @@ BuildReleaseLinuxMusl() { for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} + build_tags=$(GetBuildTagsForTarget "$os_arch") echo building for ${os_arch} export GOOS=${os_arch%%-*} export GOARCH=${os_arch##*-} export CC=${cgo_cc} export CGO_ENABLED=1 - go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + CGO_LDFLAGS="-static" go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags="$build_tags" . + AssertStaticBinary "./build/$appName-$os_arch" done } BuildReleaseLinuxMuslArm() { rm -rf .git/ mkdir -p "build" - muslflags="--extldflags '-static -fpic' $ldflags" + muslflags="$(GetMuslStaticLdflags)" BASE="https://github.com/OpenListTeam/musl-compilers/releases/latest/download/" FILES=(arm-linux-musleabi-cross arm-linux-musleabihf-cross armel-linux-musleabi-cross armel-linux-musleabihf-cross armv5l-linux-musleabi-cross armv5l-linux-musleabihf-cross armv6-linux-musleabi-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross armv7m-linux-musleabi-cross armv7r-linux-musleabihf-cross) for i in "${FILES[@]}"; do @@ -437,7 +514,8 @@ BuildReleaseLinuxMuslArm() { export CC=${cgo_cc} export CGO_ENABLED=1 export GOARM=${arm} - go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + CGO_LDFLAGS="-static" go build -o ./build/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . + AssertStaticBinary "./build/$appName-$os_arch" done } @@ -596,7 +674,11 @@ if [ "$buildType" = "dev" ]; then fi elif [ "$buildType" = "release" -o "$buildType" = "beta" ]; then if [ "$buildType" = "beta" ]; then - FetchWebRolling + if [ "$WEB_VERSION" = "latest" ]; then + FetchWebRelease + else + FetchWebRolling + fi else FetchWebRelease fi diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 162d835d0..d4f5741d0 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -68,8 +68,7 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, err } userAgent := args.Header.Get("User-Agent") - downloadInfo, err := d. - DownloadWithUA(file.(*FileObj).PickCode, userAgent) + downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } diff --git a/drivers/115/types.go b/drivers/115/types.go index 28a8ced30..3477ffed0 100644 --- a/drivers/115/types.go +++ b/drivers/115/types.go @@ -22,6 +22,10 @@ func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + type UploadResult struct { driver.BasicResp Data struct { diff --git a/drivers/115/util.go b/drivers/115/util.go index b000436b2..57c987349 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "net/url" "strconv" "strings" @@ -20,13 +19,12 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + netutil "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/aliyun/aliyun-oss-go-sdk/oss" - cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" - crypto "github.com/SheltonZhu/115driver/pkg/crypto/m115" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/pkg/errors" ) @@ -108,60 +106,6 @@ func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } -func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { - key := crypto.GenerateKey() - result := driver115.DownloadResp{} - params, err := utils.Json.Marshal(map[string]string{"pick_code": pickCode}) - if err != nil { - return nil, err - } - - data := crypto.Encode(params, key) - - bodyReader := strings.NewReader(url.Values{"data": []string{data}}.Encode()) - reqUrl := fmt.Sprintf("%s?t=%s", driver115.AndroidApiDownloadGetUrl, driver115.Now().String()) - req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Cookie", d.Cookie) - req.Header.Set("User-Agent", ua) - - resp, err := d.client.Client.GetClient().Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - if err := utils.Json.Unmarshal(body, &result); err != nil { - return nil, err - } - - if err = result.Err(string(body)); err != nil { - return nil, err - } - - b, err := crypto.Decode(string(result.EncodedData), key) - if err != nil { - return nil, err - } - - downloadInfo := struct { - Url string `json:"url"` - }{} - if err := utils.Json.Unmarshal(b, &downloadInfo); err != nil { - return nil, err - } - - info := &driver115.DownloadInfo{} - info.PickCode = pickCode - info.Header = resp.Request.Header - info.Url.Url = downloadInfo.Url - return info, nil -} - func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { userID := strconv.FormatInt(c.client.UserID, 10) userIDMd5 := md5.Sum([]byte(userID)) @@ -279,7 +223,7 @@ func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSPar if err != nil { return nil, err } - ossClient, err := oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret) + ossClient, err := netutil.NewOSSClient(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret) if err != nil { return nil, err } @@ -309,7 +253,8 @@ func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSPar // UploadByMultipart upload by mutipart blocks func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.UploadOSSParams, fileSize int64, s model.FileStreamer, - dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption) (*UploadResult, error) { + dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption, +) (*UploadResult, error) { var ( chunks []oss.FileChunk parts []oss.UploadPart @@ -339,7 +284,7 @@ func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.Upload return nil, err } - if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil { + if ossClient, err = netutil.NewOSSClient(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil { return nil, err } diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index c1a855749..f22958429 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -2,6 +2,7 @@ package _115_open import ( "context" + "encoding/json" "fmt" "net/http" "strconv" @@ -17,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -27,6 +29,12 @@ type Open115 struct { limiter *rate.Limiter } +var ( + // 回收站列表存在短暂最终一致性延迟，永久删除 fallback 查找增加短重试。 + recycleBinLookupMaxAttempts = 4 + recycleBinLookupRetryDelay = 300 * time.Millisecond +) + func (d *Open115) Config() driver.Config { return config } @@ -74,13 +82,20 @@ func (d *Open115) Drop(ctx context.Context) error { } func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + start := time.Now() + log.Infof("[115] List request started for dir: %s (ID: %s)", dir.GetName(), dir.GetID()) + var res []model.Obj pageSize := int64(d.PageSize) offset := int64(0) + pageCount := 0 + for { if err := d.WaitLimit(ctx); err != nil { return nil, err } + + pageStart := time.Now() resp, err := d.client.GetFiles(ctx, &sdk.GetFilesReq{ CID: dir.GetID(), Limit: pageSize, @@ -90,7 +105,12 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) // Cur: 1, ShowDir: true, }) + pageDuration := time.Since(pageStart) + pageCount++ + log.Infof("[115] GetFiles page %d took: %v (offset=%d, limit=%d)", pageCount, pageDuration, offset, pageSize) + if err != nil { + log.Errorf("[115] GetFiles page %d failed after %v: %v", pageCount, pageDuration, err) return nil, err } res = append(res, utils.MustSliceConvert(resp.Data, func(src sdk.GetFilesResp_File) model.Obj { @@ -102,10 +122,17 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) } offset += pageSize } + + totalDuration := time.Since(start) + log.Infof("[115] List request completed in %v (%d pages, %d files)", totalDuration, pageCount, len(res)) + return res, nil } func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + start := time.Now() + log.Infof("[115] Link request started for file: %s", file.GetName()) + if err := d.WaitLimit(ctx); err != nil { return nil, err } @@ -121,14 +148,25 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, fmt.Errorf("can't convert obj") } pc := obj.Pc + + apiStart := time.Now() + log.Infof("[115] Calling DownURL API...") resp, err := d.client.DownURL(ctx, pc, ua) + apiDuration := time.Since(apiStart) + log.Infof("[115] DownURL API took: %v", apiDuration) + if err != nil { + log.Errorf("[115] DownURL API failed after %v: %v", apiDuration, err) return nil, err } u, ok := resp[obj.GetID()] if !ok { return nil, fmt.Errorf("can't get link") } + + totalDuration := time.Since(start) + log.Infof("[115] Link request completed in %v (API: %v)", totalDuration, apiDuration) + return &model.Link{ URL: u.URL.URL, Header: http.Header{ @@ -175,8 +213,8 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) return nil, err } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ - FileID: srcObj.GetID(), - FileNma: newName, + FileID: srcObj.GetID(), + FileName: newName, }) if err != nil { return nil, err @@ -211,13 +249,164 @@ func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { if !ok { return fmt.Errorf("can't convert obj") } - _, err := d.client.DelFile(ctx, &sdk.DelFileReq{ + resp, err := d.client.DelFile(ctx, &sdk.DelFileReq{ FileIDs: _obj.GetID(), ParentID: _obj.Pid, }) if err != nil { return err } + if d.RemoveWay != "delete" { + return nil + } + return d.removePermanently(ctx, _obj, resp) +} + +func (d *Open115) removePermanently(ctx context.Context, obj *Obj, deleteResp []string) error { + var directDeleteErr error + for _, tid := range deleteResp { + tid = strings.TrimSpace(tid) + if tid == "" { + continue + } + if err := d.deleteRecycleBinEntry(ctx, tid); err == nil { + return nil + } else if directDeleteErr == nil { + directDeleteErr = err + } + } + + recycleEntry, err := d.findRecycleBinEntryWithRetry(ctx, obj) + if err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin candidate: %w; fallback lookup failed: %v", directDeleteErr, err) + } + return err + } + if err := d.deleteRecycleBinEntry(ctx, recycleEntry.ID); err != nil { + if directDeleteErr != nil { + return fmt.Errorf("failed to permanently delete recycle-bin entry %s after candidate delete error %v: %w", recycleEntry.ID, directDeleteErr, err) + } + return err + } + return nil +} + +func (d *Open115) deleteRecycleBinEntry(ctx context.Context, tid string) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + _, err := d.client.RbDelete(ctx, tid) + return err +} + +func (d *Open115) findRecycleBinEntry(ctx context.Context, obj *Obj) (*sdk.RbListResp_FileInfo, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 200 + } else if pageSize > 1150 { + pageSize = 1150 + } + + offset := int64(0) + for { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + resp, err := d.client.RbList(ctx, pageSize, offset) + if err != nil { + return nil, err + } + if entry := matchRecycleBinEntry(obj, resp.Files); entry != nil { + return entry, nil + } + + count, err := strconv.ParseInt(resp.Count, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse recycle bin count %q: %w", resp.Count, err) + } + offset += pageSize + if offset >= count || len(resp.Files) == 0 { + break + } + } + + return nil, fmt.Errorf("recycle bin entry not found for object id=%s name=%s parent=%s", obj.GetID(), obj.GetName(), obj.Pid) +} + +func isRecycleBinEntryNotFoundErr(err error) bool { + return err != nil && strings.Contains(err.Error(), "recycle bin entry not found") +} + +func (d *Open115) findRecycleBinEntryWithRetry(ctx context.Context, obj *Obj) (*sdk.RbListResp_FileInfo, error) { + attempts := recycleBinLookupMaxAttempts + if attempts < 1 { + attempts = 1 + } + + var lastErr error + for i := 0; i < attempts; i++ { + entry, err := d.findRecycleBinEntry(ctx, obj) + if err == nil { + return entry, nil + } + + lastErr = err + if !isRecycleBinEntryNotFoundErr(err) || i == attempts-1 { + break + } + + wait := recycleBinLookupRetryDelay * time.Duration(i+1) + if wait <= 0 { + continue + } + + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + + return nil, lastErr +} + +func matchRecycleBinEntry(obj *Obj, files map[string]sdk.RbListResp_FileInfo) *sdk.RbListResp_FileInfo { + if len(files) == 0 { + return nil + } + if entry, ok := files[obj.GetID()]; ok { + matched := entry + return &matched + } + + size := strconv.FormatInt(obj.GetSize(), 10) + for _, entry := range files { + if entry.ID == obj.GetID() { + matched := entry + return &matched + } + cid := string(entry.CID) + if obj.IsDir() { + if entry.FileName == obj.GetName() && cid == obj.Pid { + matched := entry + return &matched + } + continue + } + if obj.Sha1 != "" && entry.SHA1 != "" && strings.EqualFold(entry.SHA1, obj.Sha1) { + if entry.FileName == obj.GetName() || cid == obj.Pid { + matched := entry + return &matched + } + } + if entry.FileName == obj.GetName() && cid == obj.Pid && entry.FileSize == size { + matched := entry + return &matched + } + } return nil } @@ -226,27 +415,97 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } + sha1 := file.GetHash().GetHash(utils.SHA1) - if len(sha1) != utils.SHA1.Width { - _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + sha1128k := file.GetHash().GetHash(utils.SHA1_128K) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ + FileName: file.GetName(), + FileSize: file.GetSize(), + Target: dstDir.GetID(), + FileID: strings.ToUpper(sha1), + PreID: strings.ToUpper(sha1128k), + }) if err != nil { return err } + if resp.Status == 2 { + up(100) + return nil + } + // 秒传失败，继续后续流程 } - const PreHashSize int64 = 128 * utils.KB - hashSize := PreHashSize - if file.GetSize() < PreHashSize { - hashSize = file.GetSize() - } - reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) - if err != nil { - return err - } - sha1128k, err := utils.HashReader(utils.SHA1, reader) - if err != nil { - return err + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(sha1) != utils.SHA1.Width { + sha1, err = stream.StreamHashFile(file, utils.SHA1, 100, &up) + if err != nil { + return err + } + } + // 计算 sha1_128k（如果没有预计算） + if len(sha1128k) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 如果有预计算的 hash，上面已经尝试过秒传了 + if len(sha1) == utils.SHA1.Width && len(sha1128k) == utils.SHA1_128K.Width { + // 秒传失败，需要缓存文件进行实际上传 + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } else { + // 没有预计算的 hash，缓存整个文件并计算 + if len(sha1) != utils.SHA1.Width { + _, sha1, err = stream.CacheFullAndHash(file, &up, utils.SHA1) + if err != nil { + return err + } + } else if file.GetFile() == nil { + // 有 SHA1 但没有缓存，需要缓存以支持后续 RangeRead + _, err = file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + // 计算 sha1_128k + const PreHashSize int64 = 128 * utils.KB + hashSize := PreHashSize + if file.GetSize() < PreHashSize { + hashSize = file.GetSize() + } + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err != nil { + return err + } + sha1128k, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + } } - // 1. Init + + // 1. Init（SeekableStream 或已缓存的 FileStream） resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ FileName: file.GetName(), FileSize: file.GetSize(), @@ -272,11 +531,11 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre if err != nil { return err } - reader, err = file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) + signReader, err := file.RangeRead(http_range.Range{Start: start, Length: end - start + 1}) if err != nil { return err } - signVal, err := utils.HashReader(utils.SHA1, reader) + signVal, err := utils.HashReader(utils.SHA1, signReader) if err != nil { return err } @@ -314,15 +573,48 @@ func (d *Open115) OfflineDownload(ctx context.Context, uris []string, dstDir mod return d.client.AddOfflineTaskURIs(ctx, uris, dstDir.GetID()) } +func (d *Open115) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + var envelope sdk.Resp[[]sdk.AddOfflineTaskURIsResp] + response, err := d.client.AuthRequestRaw(ctx, sdk.ApiAddOffline, http.MethodPost, nil, sdk.ReqWithForm(sdk.Form{ + "urls": strings.Join(uris, "\n"), + "wp_path_id": dstDir.GetID(), + })) + if response != nil { + _ = json.Unmarshal(response.Bytes(), &envelope) + } + hashes := make([]string, 0, len(envelope.Data)) + for _, item := range envelope.Data { + if item.State && item.InfoHash != "" { + hashes = append(hashes, item.InfoHash) + } + } + rawResponse := "" + if response != nil { + rawResponse = response.String() + } + return hashes, envelope.Data, rawResponse, err +} + func (d *Open115) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { return d.client.DeleteOfflineTask(ctx, infoHash, deleteFiles) } func (d *Open115) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + // 获取第一页 resp, err := d.client.OfflineTaskList(ctx, 1) if err != nil { return nil, err } + // 如果有多页，获取所有页面的任务 + if resp.PageCount > 1 { + for page := 2; page <= resp.PageCount; page++ { + pageResp, err := d.client.OfflineTaskList(ctx, int64(page)) + if err != nil { + return nil, err + } + resp.Tasks = append(resp.Tasks, pageResp.Tasks...) + } + } return resp, nil } diff --git a/drivers/115_open/driver_test.go b/drivers/115_open/driver_test.go new file mode 100644 index 000000000..1d9664695 --- /dev/null +++ b/drivers/115_open/driver_test.go @@ -0,0 +1,639 @@ +package _115_open + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "testing" + "time" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type recordedRequest struct { + Path string + Form url.Values +} + +type rewriteTransport struct { + target *url.URL + base http.RoundTripper +} + +func (t *rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + cloned := req.Clone(req.Context()) + cloned.URL.Scheme = t.target.Scheme + cloned.URL.Host = t.target.Host + return t.base.RoundTrip(cloned) +} + +func TestOpen115RemoveTrashUsesDelFileOnly(t *testing.T) { + driver, requests := newTestOpen115(t, "trash", func(w http.ResponseWriter, r *http.Request) { + writeSDKSuccess(t, w, []string{"rb-123"}) + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete") + assertFormValue(t, requests()[0].Form, "file_ids", "file-1") + assertFormValue(t, requests()[0].Form, "parent_id", "dir-1") +} + +func TestOpen115RemoveDeleteUsesDelFileResponseIDWhenAvailable(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/del": + writeSDKSuccess(t, w, []string{"rb-123"}) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del") + assertFormValue(t, requests()[1].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteFallsBackToRecycleBinLookup(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "file_size": "123", + "cid": "dir-1", + "sha1": "sha-demo", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteReturnsErrorWhenRecycleEntryMissing(t *testing.T) { + driver, _ := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{}) + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", FS: 123, Sha1: "sha-demo"} + err := driver.Remove(context.Background(), obj) + if err == nil { + t.Fatalf("expected Remove to fail when recycle-bin entry is missing") + } + if !strings.Contains(err.Error(), "recycle bin entry not found") { + t.Fatalf("expected recycle-bin lookup error, got: %v", err) + } +} + +func TestOpen115DriverInfoIncludesRemoveWay(t *testing.T) { + info, ok := op.GetDriverInfoMap()["115 Open"] + if !ok { + t.Fatalf("115 Open driver info was not registered") + } + + for _, item := range info.Additional { + if item.Name != "remove_way" { + continue + } + if item.Type != "select" { + t.Fatalf("unexpected remove_way type: %q", item.Type) + } + if item.Options != "trash,delete" { + t.Fatalf("unexpected remove_way options: %q", item.Options) + } + if item.Default != "trash" { + t.Fatalf("unexpected remove_way default: %q", item.Default) + } + if !item.Required { + t.Fatalf("expected remove_way to be required") + } + return + } + + t.Fatalf("remove_way item not found in 115 Open driver info") +} + +func newTestOpen115(t *testing.T, removeWay string, responder http.HandlerFunc) (*Open115, func() []recordedRequest) { + t.Helper() + + var ( + mu sync.Mutex + requests []recordedRequest + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Fatalf("ParseForm failed: %v", err) + } + mu.Lock() + requests = append(requests, recordedRequest{ + Path: r.URL.Path, + Form: cloneValues(r.Form), + }) + mu.Unlock() + responder(w, r) + })) + t.Cleanup(server.Close) + + target, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("Parse server URL failed: %v", err) + } + + client := sdk.New(sdk.WithAccessToken("test-token")) + client.SetHttpClient(&http.Client{ + Transport: &rewriteTransport{ + target: target, + base: http.DefaultTransport, + }, + }) + + return &Open115{ + Addition: Addition{ + RemoveWay: removeWay, + PageSize: 1, + }, + client: client, + }, func() []recordedRequest { + mu.Lock() + defer mu.Unlock() + return append([]recordedRequest(nil), requests...) + } +} + +func writeSDKSuccess(t *testing.T, w http.ResponseWriter, data any) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": true, + "data": data, + }) +} + +func writeSDKError(t *testing.T, w http.ResponseWriter, code int64, message string) { + t.Helper() + writeSDKResponse(t, w, map[string]any{ + "state": false, + "code": code, + "message": message, + }) +} + +func writeSDKResponse(t *testing.T, w http.ResponseWriter, payload map[string]any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(payload); err != nil { + t.Fatalf("Encode response failed: %v", err) + } +} + +func assertRequestPaths(t *testing.T, requests []recordedRequest, want ...string) { + t.Helper() + got := make([]string, 0, len(requests)) + for _, req := range requests { + got = append(got, req.Path) + } + if !slices.Equal(got, want) { + t.Fatalf("unexpected request paths: got %v want %v", got, want) + } +} + +func assertFormValue(t *testing.T, form url.Values, key, want string) { + t.Helper() + if got := form.Get(key); got != want { + t.Fatalf("unexpected form value for %s: got %q want %q", key, got, want) + } +} + +func cloneValues(src url.Values) url.Values { + dst := make(url.Values, len(src)) + for key, values := range src { + dst[key] = append([]string(nil), values...) + } + return dst +} + +// --- FlexString / numeric CID tests --- + +func TestFlexStringUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {"string value", `{"cid":"dir-1"}`, "dir-1"}, + {"integer value", `{"cid":3383942108160578280}`, "3383942108160578280"}, + {"large integer", `{"cid":9999999999999999999}`, "9999999999999999999"}, + {"zero", `{"cid":0}`, "0"}, + {"negative", `{"cid":-123}`, "-123"}, + {"float", `{"cid":1.5}`, "1.5"}, + {"empty string", `{"cid":""}`, ""}, + {"null", `{"cid":null}`, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v struct { + CID sdk.FlexString `json:"cid"` + } + if err := json.Unmarshal([]byte(tt.input), &v); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if got := string(v.CID); got != tt.want { + t.Fatalf("got %q, want %q", got, tt.want) + } + }) + } +} + +func TestFlexStringUnmarshalInvalid(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"boolean", `{"cid":true}`}, + {"array", `{"cid":[1]}`}, + {"object", `{"cid":{}}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var v struct { + CID sdk.FlexString `json:"cid"` + } + if err := json.Unmarshal([]byte(tt.input), &v); err == nil { + t.Fatalf("expected error for input %s", tt.input) + } + }) + } +} + +func TestRbListRespUnmarshalCIDAsString(t *testing.T) { + raw := `{"id":"rb-1","file_name":"demo.txt","cid":"dir-1","file_size":"123"}` + var info sdk.RbListResp_FileInfo + if err := json.Unmarshal([]byte(raw), &info); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if string(info.CID) != "dir-1" { + t.Fatalf("got CID %q, want %q", string(info.CID), "dir-1") + } +} + +func TestRbListRespUnmarshalCIDAsNumber(t *testing.T) { + raw := `{"id":"rb-1","file_name":"MyFolder","cid":3383942108160578280,"file_size":"0"}` + var info sdk.RbListResp_FileInfo + if err := json.Unmarshal([]byte(raw), &info); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if string(info.CID) != "3383942108160578280" { + t.Fatalf("got CID %q, want %q", string(info.CID), "3383942108160578280") + } +} + +// --- matchRecycleBinEntry with numeric CID --- + +func TestMatchRecycleBinEntryDirMatchWithNumericCID(t *testing.T) { + // obj represents a directory with Pid (parent ID) as a large number + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-folder-1", + FileName: "MyFolder", + CID: sdk.FlexString("3383942108160578280"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match for directory with numeric CID, got nil") + } + if result.ID != "rb-folder-1" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-folder-1") + } +} + +func TestMatchRecycleBinEntryDirNoMatchWhenCIDWrong(t *testing.T) { + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-folder-1", + FileName: "MyFolder", + CID: sdk.FlexString("9999999999"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result != nil { + t.Fatalf("expected no match, got %+v", result) + } +} + +func TestMatchRecycleBinEntryFileSHA1MatchWithNumericCID(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "3383942108160578280", Fn: "video.mp4", Fc: "1", FS: 1024, Sha1: "abc123"} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "video.mp4", + CID: sdk.FlexString("3383942108160578280"), + SHA1: "ABC123", + FileSize: "1024", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via SHA1+CID, got nil") + } + if result.ID != "rb-file-1" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-file-1") + } +} + +func TestMatchRecycleBinEntryFileSHA1MatchByNameOnly(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "wrong-pid", Fn: "video.mp4", Fc: "1", FS: 1024, Sha1: "abc123"} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "video.mp4", + CID: sdk.FlexString("3383942108160578280"), + SHA1: "ABC123", + FileSize: "1024", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via SHA1+name, got nil") + } +} + +func TestMatchRecycleBinEntryFileNameSizeCIDMatch(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "3383942108160578280", Fn: "doc.pdf", Fc: "1", FS: 500} + files := map[string]sdk.RbListResp_FileInfo{ + "rb-1": { + ID: "rb-file-1", + FileName: "doc.pdf", + CID: sdk.FlexString("3383942108160578280"), + FileSize: "500", + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected match via name+size+CID, got nil") + } +} + +func TestMatchRecycleBinEntryDirectIDMatch(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123} + files := map[string]sdk.RbListResp_FileInfo{ + "file-1": { + ID: "rb-123", + FileName: "demo.txt", + CID: sdk.FlexString("dir-1"), + }, + } + result := matchRecycleBinEntry(obj, files) + if result == nil { + t.Fatal("expected direct ID match, got nil") + } + if result.ID != "rb-123" { + t.Fatalf("got ID %q, want %q", result.ID, "rb-123") + } +} + +func TestMatchRecycleBinEntryEmptyFiles(t *testing.T) { + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123} + result := matchRecycleBinEntry(obj, nil) + if result != nil { + t.Fatalf("expected nil for nil files, got %+v", result) + } + result = matchRecycleBinEntry(obj, map[string]sdk.RbListResp_FileInfo{}) + if result != nil { + t.Fatalf("expected nil for empty files, got %+v", result) + } +} + +// --- Full Remove flow with numeric CID in recycle bin --- + +func TestOpen115RemoveDeleteWithNumericCID(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"folder-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "folder-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-folder-1"}) + case "/open/rb/list": + // CID returned as number (the real bug scenario) + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-folder-1": map[string]any{ + "id": "rb-folder-1", + "file_name": "MyFolder", + "cid": 3383942108160578280, + "file_size": "0", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "folder-1", Pid: "3383942108160578280", Fn: "MyFolder", Fc: "0", FS: 0} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-folder-1") +} + +func TestOpen115RemoveDeleteWithStringCIDStillWorks(t *testing.T) { + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + // CID returned as string (normal case) + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "cid": "dir-1", + "sha1": "sha-demo", + "file_size": "123", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[3].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteRetriesRecycleBinLookupUntilVisible(t *testing.T) { + oldAttempts, oldDelay := recycleBinLookupMaxAttempts, recycleBinLookupRetryDelay + recycleBinLookupMaxAttempts = 3 + recycleBinLookupRetryDelay = time.Millisecond + t.Cleanup(func() { + recycleBinLookupMaxAttempts = oldAttempts + recycleBinLookupRetryDelay = oldDelay + }) + + rbListCalls := 0 + driver, requests := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + if r.FormValue("tid") == "file-1" { + writeSDKError(t, w, 404, "not found") + return + } + writeSDKSuccess(t, w, []string{"rb-123"}) + case "/open/rb/list": + rbListCalls++ + if rbListCalls < 3 { + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + return + } + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "1", + "rb_pass": 0, + "rb-123": map[string]any{ + "id": "rb-123", + "file_name": "demo.txt", + "cid": "dir-1", + "sha1": "sha-demo", + "file_size": "123", + }, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + if err := driver.Remove(context.Background(), obj); err != nil { + t.Fatalf("Remove returned error: %v", err) + } + + if rbListCalls != 3 { + t.Fatalf("rbListCalls = %d, want 3", rbListCalls) + } + assertRequestPaths(t, requests(), "/open/ufile/delete", "/open/rb/del", "/open/rb/list", "/open/rb/list", "/open/rb/list", "/open/rb/del") + assertFormValue(t, requests()[5].Form, "tid", "rb-123") +} + +func TestOpen115RemoveDeleteStopsRetryWhenContextCancelled(t *testing.T) { + oldAttempts, oldDelay := recycleBinLookupMaxAttempts, recycleBinLookupRetryDelay + recycleBinLookupMaxAttempts = 5 + recycleBinLookupRetryDelay = 50 * time.Millisecond + t.Cleanup(func() { + recycleBinLookupMaxAttempts = oldAttempts + recycleBinLookupRetryDelay = oldDelay + }) + + driver, _ := newTestOpen115(t, "delete", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/open/ufile/delete": + writeSDKSuccess(t, w, []string{"file-1"}) + case "/open/rb/del": + writeSDKError(t, w, 404, "not found") + case "/open/rb/list": + writeSDKSuccess(t, w, map[string]any{ + "offset": 0, + "limit": 1, + "count": "0", + "rb_pass": 0, + }) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + }) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + obj := &Obj{Fid: "file-1", Pid: "dir-1", Fn: "demo.txt", Fc: "1", FS: 123, Sha1: "sha-demo"} + err := driver.Remove(ctx, obj) + if err == nil { + t.Fatalf("expected Remove to fail due to context cancellation") + } + if !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + t.Fatalf("expected context deadline exceeded, got: %v", err) + } +} diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go index ed908e2e6..0479ac1fd 100644 --- a/drivers/115_open/meta.go +++ b/drivers/115_open/meta.go @@ -11,6 +11,7 @@ type Addition struct { // define other OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete" default:"trash"` LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` PageSize int64 `json:"page_size" type:"number" default:"200" help:"list api per page size of 115open driver"` AccessToken string `json:"access_token" required:"true"` @@ -18,9 +19,10 @@ type Addition struct { } var config = driver.Config{ - Name: "115 Open", - DefaultRoot: "0", - LinkCacheMode: driver.LinkCacheUA, + Name: "115 Open", + DefaultRoot: "0", + ProxyRangeOption: true, + LinkCacheMode: driver.LinkCacheUA, } func init() { diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go index 3575678c2..6af4403cf 100644 --- a/drivers/115_open/upload.go +++ b/drivers/115_open/upload.go @@ -4,17 +4,30 @@ import ( "context" "encoding/base64" "io" + "strings" "time" sdk "github.com/OpenListTeam/115-sdk-go" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + netutil "github.com/OpenListTeam/OpenList/v4/internal/net" streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" ) +// isTokenExpiredError 检测是否为OSS凭证过期错误 +func isTokenExpiredError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "SecurityTokenExpired") || + strings.Contains(errStr, "InvalidAccessKeyId") +} + func calPartSize(fileSize int64) int64 { var partSize int64 = 20 * utils.MB if fileSize > partSize { @@ -36,7 +49,7 @@ func calPartSize(fileSize int64) int64 { } func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { - ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) + ossClient, err := netutil.NewOSSClient(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) if err != nil { return err } @@ -70,11 +83,16 @@ func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp // } func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { - ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) - if err != nil { - return err + // 创建OSS客户端的辅助函数 + createBucket := func(token *sdk.UploadGetTokenResp) (*oss.Bucket, error) { + ossClient, err := netutil.NewOSSClient(token.Endpoint, token.AccessKeyId, token.AccessKeySecret, oss.SecurityToken(token.SecurityToken)) + if err != nil { + return nil, err + } + return ossClient.Bucket(initResp.Bucket) } - bucket, err := ossClient.Bucket(initResp.Bucket) + + bucket, err := createBucket(tokenResp) if err != nil { return err } @@ -119,7 +137,24 @@ func (d *Open115) multpartUpload(ctx context.Context, stream model.FileStreamer, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), - retry.Delay(time.Second)) + retry.Delay(time.Second), + retry.OnRetry(func(n uint, err error) { + // 如果是凭证过期错误，在重试前刷新凭证并重建bucket + if isTokenExpiredError(err) { + log.Warnf("115 OSS token expired, refreshing token...") + if newToken, refreshErr := d.client.UploadGetToken(ctx); refreshErr == nil { + tokenResp = newToken + if newBucket, bucketErr := createBucket(tokenResp); bucketErr == nil { + bucket = newBucket + log.Infof("115 OSS token refreshed successfully") + } else { + log.Errorf("Failed to create new bucket with refreshed token: %v", bucketErr) + } + } else { + log.Errorf("Failed to refresh 115 OSS token: %v", refreshErr) + } + } + })) ss.FreeSectionReader(rd) if err != nil { return err diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index 00fa623e6..fe8b7733a 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -3,6 +3,7 @@ package _115_share import ( "context" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -49,9 +50,16 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr if err := d.WaitLimit(ctx); err != nil { return nil, err } - + var ua string + // TODO: will use user agent from header + // if args.Header != nil { + // ua = args.Header.Get("User-Agent") + // } + if ua == "" { + ua = base.UserAgentNT + } files := make([]driver115.ShareFile, 0) - fileResp, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) + fileResp, err := d.client.GetShareSnapWithUA(ua, d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) if err != nil { return nil, err } @@ -77,7 +85,14 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if err := d.WaitLimit(ctx); err != nil { return nil, err } - downloadInfo, err := d.client.DownloadByShareCode(d.ShareCode, d.ReceiveCode, file.GetID()) + var ua string + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + downloadInfo, err := d.client.DownloadByShareCodeWithUA(ua, d.ShareCode, d.ReceiveCode, file.GetID()) if err != nil { return nil, err } diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go index 082d9d462..f9575d493 100644 --- a/drivers/115_share/utils.go +++ b/drivers/115_share/utils.go @@ -5,6 +5,7 @@ import ( "strconv" "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/pkg/utils" driver115 "github.com/SheltonZhu/115driver/pkg/driver" @@ -20,6 +21,7 @@ type FileObj struct { FileName string isDir bool FileID string + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -54,6 +56,10 @@ func (f *FileObj) GetPath() string { return "" } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + func transFunc(sf driver115.ShareFile) (model.Obj, error) { timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) if err != nil { @@ -74,15 +80,14 @@ func transFunc(sf driver115.ShareFile) (model.Obj, error) { FileName: string(sf.FileName), isDir: isDir, FileID: fileID, + ThumbURL: sf.ThumbURL, }, nil } -var UserAgent = driver115.UA115Browser - func (d *Pan115Share) login() error { var err error opts := []driver115.Option{ - driver115.UA(UserAgent), + driver115.UA(base.UserAgentNT), } d.client = driver115.New(opts...) if _, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, ""); err != nil { diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 9608cedf9..9adb1aed0 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -34,8 +34,8 @@ func (d *Open123) Init(ctx context.Context) error { d.UploadThread = 3 } - if d.RefreshToken != "" { - // refresh token 直接主动刷新 + if (d.UseOnlineAPI && d.RefreshToken != "" && len(d.APIAddress) > 0) || (d.ClientID != "" && d.ClientSecret != "") { + // proactive refresh by renewapi or client credentials d.AccessToken = "" d.tm = &tokenManager{} } else { @@ -175,20 +175,71 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { } func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // 1. 创建文件 + // 1. 准备参数 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { return nil, fmt.Errorf("parse parentFileID error: %v", err) } + + // 尝试 SHA1 秒传 + sha1Hash := file.GetHash().GetHash(utils.SHA1) + if len(sha1Hash) == utils.SHA1.Width { + resp, err := d.sha1Reuse(parentFileId, file.GetName(), sha1Hash, file.GetSize(), 2) + if err == nil && resp.Data.Reuse { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: resp.Data.FileID, + Type: 2, + SHA1: sha1Hash, + }, nil + } + } + + // etag 文件md5 etag := file.GetHash().GetHash(utils.MD5) - if len(etag) < utils.MD5.Width { + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + // 如果有预计算的 hash，先尝试秒传 + if len(etag) >= utils.MD5.Width { + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return nil, err + } + if createResp.Data.Reuse && createResp.Data.FileID != 0 { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: createResp.Data.FileID, + Type: 2, + Etag: etag, + }, nil + } + // 秒传失败，继续后续流程 + } + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(etag) < utils.MD5.Width { + etag, err = stream.StreamHashFile(file, utils.MD5, 100, &up) + if err != nil { + return nil, err + } + } + } else { + // 不可重复读取的流（如 HTTP body） + // 秒传失败或没有 hash，缓存整个文件并计算 MD5 _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) if err != nil { return nil, err } } + + // 2. 创建上传任务（或再次尝试秒传） createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err @@ -207,13 +258,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } - // 2. 上传分片 - err = d.Upload(ctx, file, createResp, up) + // 3. 上传分片 + uploadProgress := func(p float64) { + up(40 + p*0.6) + } + err = d.Upload(ctx, file, createResp, uploadProgress) if err != nil { return nil, err } - // 3. 上传完毕 + // 4. 合并分片/完成上传 for range 60 { uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) // 返回错误代码未知，如：20103，文档也没有具体说 diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go index 5481ef356..d23f8eec6 100644 --- a/drivers/123_open/meta.go +++ b/drivers/123_open/meta.go @@ -6,9 +6,6 @@ import ( ) type Addition struct { - // refresh_token方式的AccessToken 【对个人开发者暂未开放】 - RefreshToken string `json:"RefreshToken" required:"false"` - // 通过 https://www.123pan.com/developer 申请 ClientID string `json:"ClientID" required:"false"` ClientSecret string `json:"ClientSecret" required:"false"` @@ -16,6 +13,13 @@ type Addition struct { // 直接写入AccessToken, AccessToken有过期时间，不建议直接填写 AccessToken string `json:"AccessToken" required:"false"` + // refresh_token方式的AccessToken 【对个人开发者暂未开放】 + RefreshToken string `json:"RefreshToken" required:"false"` + + // 使用在线API + UseOnlineAPI bool `json:"use_online_api" default:"true"` + APIAddress string `json:"api_url_address" default:"https://api.oplist.org/123cloud/renewapi"` + // 用户名+密码方式登录的AccessToken可以兼容 //Username string `json:"username" required:"false"` //Password string `json:"password" required:"false"` diff --git a/drivers/123_open/token.go b/drivers/123_open/token.go index 3c5c416c9..a628d22fe 100644 --- a/drivers/123_open/token.go +++ b/drivers/123_open/token.go @@ -1,7 +1,6 @@ package _123_open import ( - "encoding/json" "errors" "fmt" "net/http" @@ -13,10 +12,16 @@ import ( ) var ( - AccessToken = "https://open-api.123pan.com/api/v1/access_token" - RefreshToken = "https://open-api.123pan.com/api/v1/oauth2/access_token" + AccessToken = "https://open-api.123pan.com/api/v1/access_token" ) +func expiresInToExpiredAt(expiresIn int64) (time.Time, error) { + if expiresIn <= 0 { + return time.Time{}, errors.New("invalid expires_in from official API") + } + return time.Now().UTC().Add(time.Duration(expiresIn) * time.Second), nil +} + type tokenManager struct { // accessToken string expiredAt time.Time @@ -43,73 +48,82 @@ func (d *Open123) getAccessToken(forceRefresh bool) (string, error) { } func (d *Open123) flushAccessToken() error { - // directly send request to avoid deadlock - req := base.RestyClient.R() - req.SetHeaders(map[string]string{ - "authorization": "Bearer " + d.AccessToken, - "platform": "open_platform", - "Content-Type": "application/json", - }) + // Official app renewapi response contains access_token, refresh_token and expires_in. + if d.UseOnlineAPI && d.RefreshToken != "" && len(d.APIAddress) > 0 { + var resp RefreshTokenResp + _, err := base.RestyClient.R(). + SetResult(&resp). + SetQueryParams(map[string]string{ + "refresh_ui": d.RefreshToken, + "server_use": "true", + "driver_txt": "123cloud_oa", + }). + Get(d.APIAddress) + if err != nil { + return err + } - if d.ClientID != "" { - if d.RefreshToken != "" { - var resp RefreshTokenResp - req.SetQueryParam("client_id", d.ClientID) - if d.ClientSecret != "" { - req.SetQueryParam("client_secret", d.ClientSecret) + if resp.AccessToken == "" || resp.RefreshToken == "" { + errMessage := resp.ErrorDescription + if errMessage == "" { + errMessage = resp.Text } - req.SetQueryParam("grant_type", "refresh_token") - req.SetQueryParam("refresh_token", d.RefreshToken) - req.SetResult(&resp) - res, err := req.Execute(http.MethodPost, RefreshToken) - if err != nil { - return err + if errMessage == "" { + errMessage = resp.Message } - body := res.Body() - var baseResp BaseResp - if err = json.Unmarshal(body, &baseResp); err != nil { - return err + if errMessage == "" { + errMessage = resp.Error } - if baseResp.Code != 0 { - return fmt.Errorf("get access token failed: %s", baseResp.Message) + if errMessage != "" { + return fmt.Errorf("failed to refresh token: %s", errMessage) } + return fmt.Errorf("empty access_token or refresh_token returned from official API") + } + expiredAt, err := expiresInToExpiredAt(resp.ExpiresIn) + if err != nil { + return err + } - d.AccessToken = resp.AccessToken - // add token expire time - d.tm.expiredAt = time.Now().Add(time.Duration(resp.ExpiresIn) * time.Second) - d.RefreshToken = resp.RefreshToken - op.MustSaveDriverStorage(d) - d.tm.blockRefresh = false - return nil - } else if d.ClientSecret != "" { - var resp AccessTokenResp - req.SetBody(base.Json{ - "clientID": d.ClientID, - "clientSecret": d.ClientSecret, - }) - req.SetResult(&resp) - res, err := req.Execute(http.MethodPost, AccessToken) - if err != nil { - return err - } - body := res.Body() - var baseResp BaseResp - if err = json.Unmarshal(body, &baseResp); err != nil { - return err - } - if baseResp.Code != 0 { - return fmt.Errorf("get access token failed: %s", baseResp.Message) - } - d.AccessToken = resp.Data.AccessToken - // parse token expire time - d.tm.expiredAt, err = time.Parse(time.RFC3339, resp.Data.ExpiredAt) - if err != nil { - return fmt.Errorf("parse expire time failed: %w", err) - } - op.MustSaveDriverStorage(d) - d.tm.blockRefresh = false - return nil + d.AccessToken = resp.AccessToken + d.RefreshToken = resp.RefreshToken + d.tm.expiredAt = expiredAt + op.MustSaveDriverStorage(d) + d.tm.blockRefresh = false + return nil + } + + // Developer API response contains code/message/data(accessToken, expiredAt). + if d.ClientID != "" && d.ClientSecret != "" { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "platform": "open_platform", + "Content-Type": "application/json", + }) + var resp AccessTokenResp + req.SetBody(base.Json{ + "clientID": d.ClientID, + "clientSecret": d.ClientSecret, + }) + req.SetResult(&resp) + _, err := req.Execute(http.MethodPost, AccessToken) + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("get access token failed: %s", resp.Message) + } + if resp.Data.AccessToken == "" || resp.Data.ExpiredAt == "" { + return errors.New("invalid token payload from developer API") + } + expiredAt, err := time.Parse(time.RFC3339, resp.Data.ExpiredAt) + if err != nil { + return fmt.Errorf("parse expire time failed: %w", err) } + d.AccessToken = resp.Data.AccessToken + d.tm.expiredAt = expiredAt.UTC() + op.MustSaveDriverStorage(d) + d.tm.blockRefresh = false + return nil } return errors.New("no valid authentication method available") } diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go index 8745ff795..b6e507ac4 100644 --- a/drivers/123_open/types.go +++ b/drivers/123_open/types.go @@ -58,9 +58,13 @@ type File struct { Category int `json:"category"` Status int `json:"status"` Trashed int `json:"trashed"` + SHA1 string } func (f File) GetHash() utils.HashInfo { + if len(f.SHA1) == utils.SHA1.Width && len(f.Etag) != utils.MD5.Width { + return utils.NewHashInfo(utils.SHA1, f.SHA1) + } return utils.NewHashInfo(utils.MD5, f.Etag) } @@ -121,11 +125,14 @@ type AccessTokenResp struct { } type RefreshTokenResp struct { - AccessToken string `json:"access_token"` - ExpiresIn int `json:"expires_in"` - RefreshToken string `json:"refresh_token"` - Scope string `json:"scope"` - TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int64 `json:"expires_in"` + Code int `json:"code"` + Message string `json:"message"` + ErrorDescription string `json:"error_description"` + Error string `json:"error"` + Text string `json:"text"` } type UserInfoResp struct { @@ -190,6 +197,14 @@ type UploadCompleteResp struct { } `json:"data"` } +type SHA1ReuseResp struct { + BaseResp + Data struct { + FileID int64 `json:"fileID"` + Reuse bool `json:"reuse"` + } `json:"data"` +} + type OfflineDownloadResp struct { BaseResp Data struct { diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go index 90cff90d7..0e03684e9 100644 --- a/drivers/123_open/upload.go +++ b/drivers/123_open/upload.go @@ -183,3 +183,21 @@ func (d *Open123) complete(preuploadID string) (*UploadCompleteResp, error) { } return &resp, nil } + +// SHA1 秒传 +func (d *Open123) sha1Reuse(parentFileID int64, filename string, sha1Hash string, size int64, duplicate int) (*SHA1ReuseResp, error) { + var resp SHA1ReuseResp + _, err := d.Request(UploadSHA1Reuse, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "parentFileID": parentFileID, + "filename": filename, + "sha1": strings.ToLower(sha1Hash), + "size": size, + "duplicate": duplicate, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} diff --git a/drivers/123_open/util.go b/drivers/123_open/util.go index 5d961d5c2..1b6eea2da 100644 --- a/drivers/123_open/util.go +++ b/drivers/123_open/util.go @@ -21,16 +21,17 @@ import ( var ( // 不同情况下获取的AccessTokenQPS限制不同 如下模块化易于拓展 Api = "https://open-api.123pan.com" - UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) - FileList = InitApiInfo(Api+"/api/v2/file/list", 3) - DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 5) - DirectLink = InitApiInfo(Api+"/api/v1/direct-link/url", 5) - Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) - Move = InitApiInfo(Api+"/api/v1/file/move", 1) - Rename = InitApiInfo(Api+"/api/v1/file/name", 1) - Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) - UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) - UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) + UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) + FileList = InitApiInfo(Api+"/api/v2/file/list", 3) + DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 5) + DirectLink = InitApiInfo(Api+"/api/v1/direct-link/url", 5) + Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) + Move = InitApiInfo(Api+"/api/v1/file/move", 1) + Rename = InitApiInfo(Api+"/api/v1/file/name", 1) + Trash = InitApiInfo(Api+"/api/v1/file/trash", 2) + UploadCreate = InitApiInfo(Api+"/upload/v2/file/create", 2) + UploadComplete = InitApiInfo(Api+"/upload/v2/file/upload_complete", 0) + UploadSHA1Reuse = InitApiInfo(Api+"/upload/v2/file/sha1_reuse", 2) OfflineDownload = InitApiInfo(Api+"/api/v1/offline/download", 1) OfflineDownloadProcess = InitApiInfo(Api+"/api/v1/offline/download/process", 5) diff --git a/drivers/139/util.go b/drivers/139/util.go index 2c7ac242d..85c798cc7 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -655,10 +655,12 @@ func (d *Yun139) personalGetLink(fileId string) (string, error) { } cdnUrl := jsoniter.Get(res, "data", "cdnUrl").ToString() if cdnUrl != "" { - return cdnUrl, nil - } else { - return jsoniter.Get(res, "data", "url").ToString(), nil + cdnSwitch := jsoniter.Get(res, "data", "cdnSwitch").ToBool() + if cdnSwitch { + return cdnUrl, nil + } } + return jsoniter.Get(res, "data", "url").ToString(), nil } func (d *Yun139) getAuthorization() string { diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 64376957e..e1ba41eb6 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -229,6 +229,15 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ for _, obj := range objMap { objs = append(objs, obj) } + if d.OrderBy == "" { + sort := getAllSort(dirs) + if sort.OrderBy != "" { + model.SortFiles(objs, sort.OrderBy, sort.OrderDirection) + } + if d.ExtractFolder == "" && sort.ExtractFolder != "" { + model.ExtractFolder(objs, sort.ExtractFolder) + } + } return objs, nil } @@ -276,21 +285,38 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( }, nil } - reqPath := d.getBalancedPath(ctx, file) - link, fi, err := d.link(ctx, reqPath, args) + var link *model.Link + var fi model.Obj + var err error + files := file.(BalancedObjs) + if d.ReadConflictPolicy == RandomBalancedRP || d.ReadConflictPolicy == AllRWP { + rand.Shuffle(len(files), func(i, j int) { + files[i], files[j] = files[j], files[i] + }) + } + for _, f := range files { + if f == nil { + continue + } + link, fi, err = d.link(ctx, f.GetPath(), args) + if err == nil { + if link == nil { + // 重定向且需要通过代理 + return &model.Link{ + URL: fmt.Sprintf("%s/p%s?sign=%s", + common.GetApiUrl(ctx), + utils.EncodePath(f.GetPath(), true), + sign.Sign(f.GetPath())), + }, nil + } + break + } + } if err != nil { return nil, err } - if link == nil { - // 重定向且需要通过代理 - return &model.Link{ - URL: fmt.Sprintf("%s/p%s?sign=%s", - common.GetApiUrl(ctx), - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)), - }, nil - } resultLink := *link // 复制一份，避免修改到原始link + resultLink.Expiration = nil resultLink.SyncClosers = utils.NewSyncClosers(link) if args.Redirect { return &resultLink, nil diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 7336b9ba9..b37854394 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -40,7 +40,7 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model if !withDetails || len(v) != 1 { continue } - remoteDriver, err := op.GetStorageByMountPath(v[0]) + remoteDriver, err := fs.GetStorage(v[0], &fs.GetStoragesArgs{}) if err != nil { continue } @@ -490,3 +490,43 @@ func (d *Alias) extract(ctx context.Context, reqPath string, args model.ArchiveI link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) return link, err } + +func getAllSort(dirs []model.Obj) model.Sort { + ret := model.Sort{} + noSort := false + noExtractFolder := false + for _, dir := range dirs { + if dir == nil { + continue + } + storage, err := fs.GetStorage(dir.GetPath(), &fs.GetStoragesArgs{}) + if err != nil { + continue + } + if !noSort && storage.GetStorage().OrderBy != "" { + if ret.OrderBy == "" { + ret.OrderBy = storage.GetStorage().OrderBy + ret.OrderDirection = storage.GetStorage().OrderDirection + if ret.OrderDirection == "" { + ret.OrderDirection = "asc" + } + } else if ret.OrderBy != storage.GetStorage().OrderBy || ret.OrderDirection != storage.GetStorage().OrderDirection { + ret.OrderBy = "" + ret.OrderDirection = "" + noSort = true + } + } + if !noExtractFolder && storage.GetStorage().ExtractFolder != "" { + if ret.ExtractFolder == "" { + ret.ExtractFolder = storage.GetStorage().ExtractFolder + } else if ret.ExtractFolder != storage.GetStorage().ExtractFolder { + ret.ExtractFolder = "" + noExtractFolder = true + } + } + if noSort && noExtractFolder { + break + } + } + return ret +} diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index a4a6c1de1..5f02c75f5 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -163,21 +163,29 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m } count := int(math.Ceil(float64(stream.GetSize()) / float64(partSize))) createData["part_info_list"] = makePartInfos(count) + + // 检查是否是可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + // rapid upload rapidUpload := !stream.IsForceStreamUpload() && stream.GetSize() > 100*utils.KB && d.RapidUpload if rapidUpload { log.Debugf("[aliyundrive_open] start cal pre_hash") - // read 1024 bytes to calculate pre hash - reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) - if err != nil { - return nil, err - } - hash, err := utils.HashReader(utils.SHA1, reader) - if err != nil { - return nil, err + // 优先使用预计算的 pre_hash + preHash := stream.GetHash().GetHash(utils.PRE_HASH) + if len(preHash) != utils.PRE_HASH.Width { + // 没有预计算的 pre_hash，使用 RangeRead 计算 + reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: 1024}) + if err != nil { + return nil, err + } + preHash, err = utils.HashReader(utils.SHA1, reader) + if err != nil { + return nil, err + } } createData["size"] = stream.GetSize() - createData["pre_hash"] = hash + createData["pre_hash"] = preHash } var createResp CreateResp _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { @@ -191,9 +199,18 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m hash := stream.GetHash().GetHash(utils.SHA1) if len(hash) != utils.SHA1.Width { - _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) - if err != nil { - return nil, err + if isSeekable { + // 可重复读取的流，使用 StreamHashFile（RangeRead），不缓存 + hash, err = streamPkg.StreamHashFile(stream, utils.SHA1, 100, &up) + if err != nil { + return nil, err + } + } else { + // 不可重复读取的流，缓存并计算 + _, hash, err = streamPkg.CacheFullAndHash(stream, &up, utils.SHA1) + if err != nil { + return nil, err + } } } diff --git a/drivers/all.go b/drivers/all.go index 7e1c24bba..fb68d0395 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -29,6 +29,7 @@ import ( _ "github.com/OpenListTeam/OpenList/v4/drivers/crypt" _ "github.com/OpenListTeam/OpenList/v4/drivers/degoo" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao" + _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_new" _ "github.com/OpenListTeam/OpenList/v4/drivers/doubao_share" _ "github.com/OpenListTeam/OpenList/v4/drivers/dropbox" _ "github.com/OpenListTeam/OpenList/v4/drivers/febbox" diff --git a/drivers/azure_blob/driver.go b/drivers/azure_blob/driver.go index ddfe3ff68..1c0bb9a9c 100644 --- a/drivers/azure_blob/driver.go +++ b/drivers/azure_blob/driver.go @@ -85,6 +85,9 @@ func (d *AzureBlob) Drop(ctx context.Context) error { // List retrieves blobs and directories under the specified path. func (d *AzureBlob) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { prefix := ensureTrailingSlash(dir.GetPath()) + if prefix == "/" { + prefix = "" + } pager := d.containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{ Prefix: &prefix, @@ -100,10 +103,11 @@ func (d *AzureBlob) List(ctx context.Context, dir model.Obj, args model.ListArgs // Process directories for _, blobPrefix := range page.Segment.BlobPrefixes { objs = append(objs, &model.Object{ - Name: path.Base(strings.TrimSuffix(*blobPrefix.Name, "/")), - Path: *blobPrefix.Name, - Modified: *blobPrefix.Properties.LastModified, - Ctime: *blobPrefix.Properties.CreationTime, + Name: path.Base(strings.TrimSuffix(*blobPrefix.Name, "/")), + Path: *blobPrefix.Name, + // Azure does not support properties now. + //Modified: *blobPrefix.Properties.LastModified, + //Ctime: *blobPrefix.Properties.CreationTime, IsFolder: true, }) } diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index fe77aca38..474dd2b98 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -1,30 +1,18 @@ package baidu_netdisk import ( - "bytes" "context" - "crypto/md5" - "encoding/hex" "errors" - "io" - "mime/multipart" - "net/http" "net/url" - "os" stdpath "path" "strconv" - "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" - "github.com/OpenListTeam/OpenList/v4/internal/net" - "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" "github.com/OpenListTeam/OpenList/v4/pkg/utils" - "github.com/avast/retry-go" log "github.com/sirupsen/logrus" ) @@ -37,6 +25,7 @@ type BaiduNetdisk struct { } var ErrUploadIDExpired = errors.New("uploadid expired") +var ErrUploadURLExpired = errors.New("upload url expired or unavailable") func (d *BaiduNetdisk) Config() driver.Config { return config @@ -199,80 +188,26 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return newObj, nil } - var ( - cache = stream.GetFile() - tmpF *os.File - err error - ) - if cache == nil { - tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - defer func() { - _ = tmpF.Close() - _ = os.Remove(tmpF.Name()) - }() - cache = tmpF - } - streamSize := stream.GetSize() sliceSize := d.getSliceSize(streamSize) count := 1 if streamSize > sliceSize { count = int((streamSize + sliceSize - 1) / sliceSize) } - lastBlockSize := streamSize % sliceSize - if lastBlockSize == 0 { - lastBlockSize = sliceSize - } - - // cal md5 for first 256k data - const SliceSize int64 = 256 * utils.KB - blockList := make([]string, 0, count) - byteSize := sliceSize - fileMd5H := md5.New() - sliceMd5H := md5.New() - sliceMd5H2 := md5.New() - slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) - writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} - if tmpF != nil { - writers = append(writers, tmpF) - } - written := int64(0) - for i := 1; i <= count; i++ { - if utils.IsCanceled(ctx) { - return nil, ctx.Err() - } - if i == count { - byteSize = lastBlockSize - } - n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) - written += n - if err != nil && err != io.EOF { - return nil, err - } - blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) - sliceMd5H.Reset() - } - if tmpF != nil { - if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) - } - _, err = tmpF.Seek(0, io.SeekStart) - if err != nil { - return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") - } - } - contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) - sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) - blockListStr, _ := utils.Json.MarshalToString(blockList) path := stdpath.Join(dstDir.GetPath(), stream.GetName()) mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 尝试读取已保存进度 + // step.1 流式计算MD5哈希值（使用 RangeRead，不会消耗流） + contentMd5, sliceMd5, blockList, err := d.calculateHashesStream(ctx, stream, sliceSize, &up) + if err != nil { + return nil, err + } + + blockListStr, _ := utils.Json.MarshalToString(blockList) + + // step.2 尝试读取已保存进度或执行预上传 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { // 没有进度，走预上传 @@ -288,6 +223,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return fileToObj(precreateResp.File), nil } } + ensureUploadURL := func() { if precreateResp.UploadURL != "" { return @@ -295,58 +231,20 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) } - // step.2 上传分片 + // step.3 流式上传分片 + // 创建 StreamSectionReader 用于上传 + ss, err := streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } + uploadLoop: for range 2 { // 获取上传域名 ensureUploadURL() - // 并发上传 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(UPLOAD_RETRY_COUNT), - retry.Delay(UPLOAD_RETRY_WAIT_TIME), - retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), - retry.DelayType(retry.BackOffDelay), - retry.RetryIf(func(err error) bool { - return !errors.Is(err, ErrUploadIDExpired) - }), - retry.LastErrorOnly(true)) - - totalParts := len(precreateResp.BlockList) - - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break - } - if partseq < 0 { - continue - } - i, partseq := i, partseq - offset, size := int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - size = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), - } - section := io.NewSectionReader(cache, offset, size) - err := d.uploadSlice(ctx, precreateResp.UploadURL, params, stream.GetName(), section) - if err != nil { - return err - } - precreateResp.BlockList[i] = -1 - progress := float64(threadG.Success()+1) * 100 / float64(totalParts+1) - up(progress) - return nil - }) - } - err = threadG.Wait() + // 流式并发上传 + err = d.uploadChunksStream(ctx, ss, stream, precreateResp, path, sliceSize, count, up) if err == nil { break uploadLoop } @@ -372,13 +270,19 @@ uploadLoop: precreateResp.UploadURL = "" // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + + // 尝试重新创建 StreamSectionReader（如果流支持重新读取） + ss, err = streamPkg.NewStreamSectionReader(stream, int(sliceSize), &up) + if err != nil { + return nil, err + } continue uploadLoop } return nil, err } defer up(100) - // step.3 创建文件 + // step.4 创建文件 var newFile File _, err = d.create(path, streamSize, 0, precreateResp.Uploadid, blockListStr, &newFile, mtime, ctime) if err != nil { @@ -427,68 +331,6 @@ func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize in return &precreateResp, nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file *io.SectionReader) error { - b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) - mw := multipart.NewWriter(b) - _, err := mw.CreateFormFile("file", fileName) - if err != nil { - return err - } - headSize := b.Len() - err = mw.Close() - if err != nil { - return err - } - head := bytes.NewReader(b.Bytes()[:headSize]) - tail := bytes.NewReader(b.Bytes()[headSize:]) - rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, file, tail)) - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) - if err != nil { - return err - } - query := req.URL.Query() - for k, v := range params { - query.Set(k, v) - } - req.URL.RawQuery = query.Encode() - req.Header.Set("Content-Type", mw.FormDataContentType()) - req.ContentLength = int64(b.Len()) + file.Size() - - client := net.NewHttpClient() - if d.UploadSliceTimeout > 0 { - client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) - } else { - client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT - } - resp, err := client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - b.Reset() - _, err = b.ReadFrom(resp.Body) - if err != nil { - return err - } - body := b.Bytes() - respStr := string(body) - log.Debugln(respStr) - lower := strings.ToLower(respStr) - // 合并 uploadid 过期检测逻辑 - if strings.Contains(lower, "uploadid") && - (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { - return ErrUploadIDExpired - } - - errCode := utils.Json.Get(body, "error_code").ToInt() - errNo := utils.Json.Get(body, "errno").ToInt() - if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) - } - return nil -} - func (d *BaiduNetdisk) GetDetails(ctx context.Context) (*model.StorageDetails, error) { du, err := d.quota(ctx) if err != nil { diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 3f3bed022..499fcd8a8 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -31,8 +31,8 @@ type Addition struct { const ( UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) - DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 60 // 上传分片请求默认超时时间 - UPLOAD_RETRY_COUNT = 3 + DEFAULT_UPLOAD_SLICE_TIMEOUT = time.Second * 180 // 上传分片请求默认超时时间（增加到3分钟以应对慢速网络） + UPLOAD_RETRY_COUNT = 5 // 增加重试次数以提高成功率 UPLOAD_RETRY_WAIT_TIME = time.Second * 1 UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 ) diff --git a/drivers/baidu_netdisk/upload.go b/drivers/baidu_netdisk/upload.go new file mode 100644 index 000000000..c160c3a9e --- /dev/null +++ b/drivers/baidu_netdisk/upload.go @@ -0,0 +1,311 @@ +package baidu_netdisk + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "errors" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/net" + streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/errgroup" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" + log "github.com/sirupsen/logrus" +) + +// calculateHashesStream 流式计算文件的MD5哈希值 +// 返回：文件MD5、前256KB的MD5、每个分片的MD5列表 +// 注意：此函数使用 RangeRead 读取数据，不会消耗流 +func (d *BaiduNetdisk) calculateHashesStream( + ctx context.Context, + stream model.FileStreamer, + sliceSize int64, + up *driver.UpdateProgress, +) (contentMd5 string, sliceMd5 string, blockList []string, err error) { + streamSize := stream.GetSize() + count := 1 + if streamSize > sliceSize { + count = int((streamSize + sliceSize - 1) / sliceSize) + } + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 前256KB的MD5 + const SliceSize int64 = 256 * utils.KB + blockList = make([]string, 0, count) + fileMd5H := md5.New() + sliceMd5H2 := md5.New() + sliceWritten := int64(0) + + // 使用固定大小的缓冲区进行流式哈希计算 + // 这样可以利用 readFullWithRangeRead 的链接刷新逻辑 + const chunkSize = 10 * 1024 * 1024 // 10MB per chunk + buf := make([]byte, chunkSize) + + for i := 0; i < count; i++ { + if utils.IsCanceled(ctx) { + return "", "", nil, ctx.Err() + } + + offset := int64(i) * sliceSize + length := sliceSize + if i == count-1 { + length = lastBlockSize + } + + // 计算分片MD5 + sliceMd5Calc := md5.New() + + // 分块读取并计算哈希 + var sliceOffset int64 = 0 + for sliceOffset < length { + readSize := chunkSize + if length-sliceOffset < int64(chunkSize) { + readSize = int(length - sliceOffset) + } + + // 使用 readFullWithRangeRead 读取数据，自动处理链接刷新 + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset+sliceOffset) + if err != nil { + return "", "", nil, err + } + + // 同时写入多个哈希计算器 + fileMd5H.Write(buf[:n]) + sliceMd5Calc.Write(buf[:n]) + if sliceWritten < SliceSize { + remaining := SliceSize - sliceWritten + if int64(n) > remaining { + sliceMd5H2.Write(buf[:remaining]) + sliceWritten += remaining + } else { + sliceMd5H2.Write(buf[:n]) + sliceWritten += int64(n) + } + } + + sliceOffset += int64(n) + } + + blockList = append(blockList, hex.EncodeToString(sliceMd5Calc.Sum(nil))) + + // 更新进度（哈希计算占总进度的一小部分） + if up != nil { + progress := float64(i+1) * 10 / float64(count) + (*up)(progress) + } + } + + return hex.EncodeToString(fileMd5H.Sum(nil)), + hex.EncodeToString(sliceMd5H2.Sum(nil)), + blockList, nil +} + +// uploadChunksStream 流式上传所有分片 +func (d *BaiduNetdisk) uploadChunksStream( + ctx context.Context, + ss streamPkg.StreamSectionReaderIF, + stream model.FileStreamer, + precreateResp *PrecreateResp, + path string, + sliceSize int64, + count int, + up driver.UpdateProgress, +) error { + streamSize := stream.GetSize() + lastBlockSize := streamSize % sliceSize + if lastBlockSize == 0 { + lastBlockSize = sliceSize + } + + // 使用 OrderedGroup 保证 Before 阶段有序 + thread := min(d.uploadThread, len(precreateResp.BlockList)) + threadG, upCtx := errgroup.NewOrderedGroupWithContext(ctx, thread, + retry.Attempts(UPLOAD_RETRY_COUNT), + retry.Delay(UPLOAD_RETRY_WAIT_TIME), + retry.MaxDelay(UPLOAD_RETRY_MAX_WAIT_TIME), + retry.DelayType(retry.BackOffDelay), + retry.RetryIf(func(err error) bool { + return !errors.Is(err, ErrUploadIDExpired) + }), + retry.OnRetry(func(n uint, err error) { + // 重试前检测是否需要刷新上传 URL + if errors.Is(err, ErrUploadURLExpired) { + log.Infof("[baidu_netdisk] refreshing upload URL due to error: %v", err) + precreateResp.UploadURL = d.getUploadUrl(path, precreateResp.Uploadid) + } + }), + retry.LastErrorOnly(true)) + + totalParts := len(precreateResp.BlockList) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) { + break + } + if partseq < 0 { + continue + } + + i, partseq := i, partseq + offset := int64(partseq) * sliceSize + size := sliceSize + if partseq+1 == count { + size = lastBlockSize + } + + var reader io.ReadSeeker + + threadG.GoWithLifecycle(errgroup.Lifecycle{ + Before: func(ctx context.Context) error { + var err error + reader, err = ss.GetSectionReader(offset, size) + return err + }, + Do: func(ctx context.Context) error { + reader.Seek(0, io.SeekStart) + err := d.uploadSliceStream(ctx, precreateResp.UploadURL, path, + precreateResp.Uploadid, partseq, stream.GetName(), reader, size) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 进度从10%开始（前10%是哈希计算） + progress := 10 + float64(threadG.Success()+1)*90/float64(totalParts+1) + up(progress) + return nil + }, + After: func(err error) { + ss.FreeSectionReader(reader) + }, + }) + } + + return threadG.Wait() +} + +// uploadSliceStream 上传单个分片（接受io.ReadSeeker） +func (d *BaiduNetdisk) uploadSliceStream( + ctx context.Context, + uploadUrl string, + path string, + uploadid string, + partseq int, + fileName string, + reader io.ReadSeeker, + size int64, +) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": uploadid, + "partseq": strconv.Itoa(partseq), + } + + b := bytes.NewBuffer(make([]byte, 0, bytes.MinRead)) + mw := multipart.NewWriter(b) + _, err := mw.CreateFormFile("file", fileName) + if err != nil { + return err + } + headSize := b.Len() + err = mw.Close() + if err != nil { + return err + } + head := bytes.NewReader(b.Bytes()[:headSize]) + tail := bytes.NewReader(b.Bytes()[headSize:]) + rateLimitedRd := driver.NewLimitedUploadStream(ctx, io.MultiReader(head, reader, tail)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, uploadUrl+"/rest/2.0/pcs/superfile2", rateLimitedRd) + if err != nil { + return err + } + query := req.URL.Query() + for k, v := range params { + query.Set(k, v) + } + req.URL.RawQuery = query.Encode() + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(b.Len()) + size + + client := net.NewHttpClient() + if d.UploadSliceTimeout > 0 { + client.Timeout = time.Second * time.Duration(d.UploadSliceTimeout) + } else { + client.Timeout = DEFAULT_UPLOAD_SLICE_TIMEOUT + } + resp, err := client.Do(req) + if err != nil { + // 检测超时或网络错误，标记需要刷新上传 URL + if isUploadURLError(err) { + log.Warnf("[baidu_netdisk] upload slice failed with network error: %v, will refresh upload URL", err) + return errors.Join(err, ErrUploadURLExpired) + } + return err + } + defer resp.Body.Close() + b.Reset() + _, err = b.ReadFrom(resp.Body) + if err != nil { + return err + } + body := b.Bytes() + respStr := string(body) + log.Debugln(respStr) + lower := strings.ToLower(respStr) + // 合并 uploadid 过期检测逻辑 + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() + if errCode != 0 || errNo != 0 { + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", respStr) + } + return nil +} + +// isUploadURLError 判断是否为需要刷新上传 URL 的错误 +// 包括：超时、连接被拒绝、连接重置、DNS 解析失败等网络错误 +func isUploadURLError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + // 超时错误 + if strings.Contains(errStr, "timeout") || + strings.Contains(errStr, "deadline exceeded") { + return true + } + // 连接错误 + if strings.Contains(errStr, "connection refused") || + strings.Contains(errStr, "connection reset") || + strings.Contains(errStr, "no such host") || + strings.Contains(errStr, "network is unreachable") { + return true + } + // EOF 错误（连接被服务器关闭） + if strings.Contains(errStr, "eof") || + strings.Contains(errStr, "broken pipe") { + return true + } + return false +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 0e27fb305..75018a708 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -207,7 +207,24 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Li return nil, err } u := fmt.Sprintf("%s&access_token=%s", resp.List[0].Dlink, d.AccessToken) - res, err := base.NoRedirectClient.R().SetHeader("User-Agent", "pan.baidu.com").Head(u) + + // Retry HEAD request with longer timeout to avoid client-side errors + // Create a client with longer timeout (base.NoRedirectClient doesn't have timeout set) + client := base.NoRedirectClient.SetTimeout(60 * time.Second) + var res *resty.Response + maxRetries := 5 + for i := 0; i < maxRetries; i++ { + res, err = client.R(). + SetHeader("User-Agent", "pan.baidu.com"). + Head(u) + if err == nil { + break + } + if i < maxRetries-1 { + log.Warnf("HEAD request failed (attempt %d/%d): %v, retrying...", i+1, maxRetries, err) + time.Sleep(time.Duration(i+1) * 2 * time.Second) // Exponential backoff: 2s, 4s, 6s, 8s + } + } if err != nil { return nil, err } diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 55462be7e..12c4fc028 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -89,6 +89,10 @@ func (d *Cloudreve) Link(ctx context.Context, file model.Obj, args model.LinkArg } return &model.Link{ URL: dUrl, + Header: http.Header{ + "Referer": {d.Address}, + "User-Agent": {d.getUA()}, + }, }, nil } diff --git a/drivers/cloudreve_v4/driver.go b/drivers/cloudreve_v4/driver.go index a1d301635..afd64d3d5 100644 --- a/drivers/cloudreve_v4/driver.go +++ b/drivers/cloudreve_v4/driver.go @@ -46,6 +46,9 @@ func (d *CloudreveV4) Init(ctx context.Context) error { if d.ref != nil { return nil } + if d.isShare() { + return nil + } if d.canLogin() { return d.login() } @@ -129,15 +132,7 @@ func (d *CloudreveV4) List(ctx context.Context, dir model.Obj, args model.ListAr } } return &model.ObjThumb{ - Object: model.Object{ - ID: src.ID, - Path: src.Path, - Name: src.Name, - Size: src.Size, - Modified: src.UpdatedAt, - Ctime: src.CreatedAt, - IsFolder: src.Type == 1, - }, + Object: *fileToObject(&src), Thumbnail: thumb, }, nil }) @@ -151,14 +146,7 @@ func (d *CloudreveV4) Get(ctx context.Context, path string) (model.Obj, error) { if err != nil { return nil, err } - return &model.Object{ - ID: info.ID, - Path: info.Path, - Name: info.Name, - Size: info.Size, - Modified: info.UpdatedAt, - Ctime: info.CreatedAt, - }, nil + return fileToObject(&info), nil } func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -179,6 +167,10 @@ func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkA return &model.Link{ URL: url.Urls[0].URL, Expiration: &exp, + Header: http.Header{ + "Referer": {d.Address}, + "User-Agent": {d.getUA()}, + }, }, nil } diff --git a/drivers/cloudreve_v4/types.go b/drivers/cloudreve_v4/types.go index 23335042f..b67cfc862 100644 --- a/drivers/cloudreve_v4/types.go +++ b/drivers/cloudreve_v4/types.go @@ -122,6 +122,18 @@ type File struct { PrimaryEntity string `json:"primary_entity"` } +func fileToObject(f *File) *model.Object { + return &model.Object{ + ID: f.ID, + Path: f.Path, + Name: f.Name, + Size: f.Size, + Modified: f.UpdatedAt, + Ctime: f.CreatedAt, + IsFolder: f.Type == 1, + } +} + type StoragePolicy struct { ID string `json:"id"` Name string `json:"name"` diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go index 853df9ad6..5d0157ff1 100644 --- a/drivers/cloudreve_v4/util.go +++ b/drivers/cloudreve_v4/util.go @@ -16,6 +16,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -30,7 +31,9 @@ import ( const ( CodeLoginRequired = http.StatusUnauthorized + CodePathNotExist = 40016 // Path not exist CodeCredentialInvalid = 40020 // Failed to issue token + // IncorrectSharePassword = 40069 // Incorrect share password ) var ( @@ -101,6 +104,9 @@ func (d *CloudreveV4) _request(method string, path string, callback base.ReqCall if r.Code == CodeCredentialInvalid { return ErrorIssueToken } + if r.Code == CodePathNotExist { + return errs.ObjectNotFound + } return fmt.Errorf("%d: %s", r.Code, r.Msg) } @@ -272,9 +278,16 @@ func (d *CloudreveV4) parseJWT(token string, jwt any) error { return nil } +func (d *CloudreveV4) isShare() bool { + return strings.HasSuffix(d.GetRootPath(), "@share") +} + // check if token is expired // https://github.com/cloudreve/frontend/blob/ddfacc1c31c49be03beb71de4cc114c8811038d6/src/session/index.ts#L177-L200 func (d *CloudreveV4) isTokenExpired() bool { + if d.isShare() { + return false + } if d.RefreshToken == "" { // login again if username and password is set if d.canLogin() { diff --git a/drivers/doubao_new/auth.go b/drivers/doubao_new/auth.go new file mode 100644 index 000000000..537b6a6ea --- /dev/null +++ b/drivers/doubao_new/auth.go @@ -0,0 +1,597 @@ +package doubao_new + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "math/big" + "net/url" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/pkg/cookie" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" + "golang.org/x/crypto/pbkdf2" +) + +const ( + defaultAuthRefreshAheadSeconds = int64(120) + defaultDpopRefreshAheadSeconds = int64(5) +) + +type Clock interface { + Now() (int64, error) +} + +type SystemClock struct{} + +func (SystemClock) Now() (int64, error) { return time.Now().Unix(), nil } + +type DPoPTokenInput struct { + KeyPair *ecdsa.PrivateKey + ExpiresIn int64 // 默认 15 + + JTI string + HTM string + HTU string + IAT int64 + Nonce string + Clock Clock +} + +type DPoPTokenOutput struct { + DPoPToken string `json:"dpopToken"` + ExpiredTime int64 `json:"expiredTime"` + ExpiresIn int64 `json:"expiresIn"` +} + +type JWTPayload struct { + Exp int64 `json:"exp,omitempty"` + Iat int64 `json:"iat,omitempty"` + Nbf int64 `json:"nbf,omitempty"` + Jti string `json:"jti,omitempty"` + Htm string `json:"htm,omitempty"` + Htu string `json:"htu,omitempty"` + Nonce string `json:"nonce,omitempty"` + Sub string `json:"sub,omitempty"` +} + +type jwkECPrivateKey struct { + Kty string `json:"kty"` + Crv string `json:"crv"` + X string `json:"x"` + Y string `json:"y"` + D string `json:"d"` +} + +type dpopKeyPairEnvelope struct { + PrivateKey *jwkECPrivateKey `json:"privateKey"` + KeyPair *jwkECPrivateKey `json:"keyPair"` + JWK *jwkECPrivateKey `json:"jwk"` +} + +type encryptedDpopKeyPair struct { + Data string `json:"data"` + Ciphertext string `json:"ciphertext"` + Encrypted string `json:"encrypted"` + Secret string `json:"secret"` + Password string `json:"password"` + Passphrase string `json:"passphrase"` +} + +func GenerateDPoPToken(in DPoPTokenInput) (*DPoPTokenOutput, error) { + if in.KeyPair == nil { + return nil, errors.New("keyPair required") + } + if in.KeyPair.Curve != elliptic.P256() { + return nil, errors.New("ES256 requires P-256 key") + } + if in.Clock == nil { + in.Clock = SystemClock{} + } + if in.ExpiresIn <= 0 { + in.ExpiresIn = 15 + } + + now, err := in.Clock.Now() + if err != nil { + return nil, err + } + + payload := map[string]any{ + "jti": pickStr(in.JTI, uuid.NewString()), + "htm": pickStr(in.HTM, ""), + "htu": pickStr(in.HTU, ""), + "iat": pickI64(in.IAT, now), + "nonce": pickStr(in.Nonce, uuid.NewString()), + } + if in.ExpiresIn > 0 { + payload["exp"] = payload["iat"].(int64) + in.ExpiresIn + } + + pub := in.KeyPair.PublicKey + header := map[string]any{ + "typ": "dpop+jwt", + "alg": "ES256", + "jwk": map[string]string{ + "kty": "EC", + "crv": "P-256", + "x": b64url(pad32(pub.X.Bytes())), + "y": b64url(pad32(pub.Y.Bytes())), + }, + } + + hb, err := json.Marshal(header) + if err != nil { + return nil, err + } + pb, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + hEnc := b64url(hb) + pEnc := b64url(pb) + signingInput := hEnc + "." + pEnc + + sum := sha256.Sum256([]byte(signingInput)) + r, s, err := ecdsa.Sign(rand.Reader, in.KeyPair, sum[:]) + if err != nil { + return nil, err + } + + sig := append(pad32(r.Bytes()), pad32(s.Bytes())...) + token := signingInput + "." + b64url(sig) + + iat := payload["iat"].(int64) + return &DPoPTokenOutput{ + DPoPToken: token, + ExpiredTime: iat + in.ExpiresIn, + ExpiresIn: in.ExpiresIn, + }, nil +} + +func GenerateDPoPKeyPair() (*ecdsa.PrivateKey, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + return validateP256Key(key) +} + +func ParseJWTPayload(token string, out any) error { + token = strings.TrimSpace(trimTokenScheme(token)) + parts := strings.Split(token, ".") + if len(parts) < 2 { + return fmt.Errorf("invalid JWT format") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return fmt.Errorf("failed to decode JWT payload: %w", err) + } + if err := json.Unmarshal(payload, out); err != nil { + return fmt.Errorf("failed to parse JWT payload: %w", err) + } + return nil +} + +func parseECPrivateKeyJWK(raw string) (*ecdsa.PrivateKey, error) { + var jwk jwkECPrivateKey + if err := json.Unmarshal([]byte(raw), &jwk); err != nil { + return nil, err + } + if jwk.D == "" || jwk.X == "" || jwk.Y == "" { + var env dpopKeyPairEnvelope + if err := json.Unmarshal([]byte(raw), &env); err != nil { + return nil, err + } + switch { + case env.PrivateKey != nil: + jwk = *env.PrivateKey + case env.KeyPair != nil: + jwk = *env.KeyPair + case env.JWK != nil: + jwk = *env.JWK + default: + return nil, errors.New("missing private key JWK") + } + } + + if jwk.Kty != "" && jwk.Kty != "EC" { + return nil, errors.New("unsupported JWK kty") + } + if jwk.Crv != "" && jwk.Crv != "P-256" { + return nil, errors.New("unsupported JWK curve") + } + if jwk.D == "" || jwk.X == "" || jwk.Y == "" { + return nil, errors.New("incomplete JWK") + } + + xBytes, err := base64.RawURLEncoding.DecodeString(jwk.X) + if err != nil { + return nil, fmt.Errorf("invalid jwk x: %w", err) + } + yBytes, err := base64.RawURLEncoding.DecodeString(jwk.Y) + if err != nil { + return nil, fmt.Errorf("invalid jwk y: %w", err) + } + dBytes, err := base64.RawURLEncoding.DecodeString(jwk.D) + if err != nil { + return nil, fmt.Errorf("invalid jwk d: %w", err) + } + + key := &ecdsa.PrivateKey{ + PublicKey: ecdsa.PublicKey{ + Curve: elliptic.P256(), + X: new(big.Int).SetBytes(xBytes), + Y: new(big.Int).SetBytes(yBytes), + }, + D: new(big.Int).SetBytes(dBytes), + } + return validateP256Key(key) +} + +func parseEncryptedDPoPKeyPair(raw, secret string) (*ecdsa.PrivateKey, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, errors.New("empty encrypted key pair") + } + + var payload encryptedDpopKeyPair + ciphertext := raw + if strings.HasPrefix(raw, "{") { + if err := json.Unmarshal([]byte(raw), &payload); err != nil { + return nil, err + } + switch { + case strings.TrimSpace(payload.Data) != "": + ciphertext = strings.TrimSpace(payload.Data) + case strings.TrimSpace(payload.Ciphertext) != "": + ciphertext = strings.TrimSpace(payload.Ciphertext) + case strings.TrimSpace(payload.Encrypted) != "": + ciphertext = strings.TrimSpace(payload.Encrypted) + default: + return nil, errors.New("missing encrypted dpop payload") + } + } + + decoded, err := decodeBase64Loose(ciphertext) + if err != nil { + return nil, err + } + if len(decoded) <= 12 { + return nil, errors.New("encrypted dpop payload too short") + } + + plain, err := decryptDoubaoKeyPair(decoded, secret) + if err != nil { + return nil, fmt.Errorf("failed to decrypt with secret: %w", err) + } + return parseECPrivateKeyJWK(string(plain)) +} + +func decryptDoubaoKeyPair(ciphertext []byte, secret string) ([]byte, error) { + key := pbkdf2.Key([]byte(secret), []byte("fixed-salt"), 100000, 32, sha256.New) + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + aead, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonceSize := aead.NonceSize() + if len(ciphertext) <= nonceSize { + return nil, errors.New("ciphertext too short") + } + nonce := ciphertext[:nonceSize] + enc := ciphertext[nonceSize:] + return aead.Open(nil, nonce, enc, nil) +} + +func decodeBase64Loose(raw string) ([]byte, error) { + raw = strings.TrimSpace(raw) + raw = strings.ReplaceAll(raw, "\n", "") + raw = strings.ReplaceAll(raw, "\r", "") + raw = strings.ReplaceAll(raw, "\t", "") + raw = strings.ReplaceAll(raw, " ", "") + + encodings := []*base64.Encoding{ + base64.StdEncoding, + base64.RawStdEncoding, + base64.URLEncoding, + base64.RawURLEncoding, + } + var lastErr error + for _, enc := range encodings { + decoded, err := enc.DecodeString(raw) + if err == nil { + return decoded, nil + } + lastErr = err + } + if lastErr == nil { + lastErr = errors.New("invalid base64") + } + return nil, lastErr +} + +func validateP256Key(key *ecdsa.PrivateKey) (*ecdsa.PrivateKey, error) { + if key == nil { + return nil, errors.New("nil private key") + } + if key.Curve != elliptic.P256() { + return nil, errors.New("ES256 requires P-256 key") + } + if key.PublicKey.X == nil || key.PublicKey.Y == nil || key.D == nil { + return nil, errors.New("invalid private key") + } + if !key.Curve.IsOnCurve(key.PublicKey.X, key.PublicKey.Y) { + return nil, errors.New("public key is not on P-256 curve") + } + return key, nil +} + +func trimTokenScheme(token string) string { + token = strings.TrimSpace(token) + if i := strings.IndexByte(token, ' '); i > 0 { + scheme := strings.ToLower(strings.TrimSpace(token[:i])) + if scheme == "bearer" || scheme == "dpop" { + return strings.TrimSpace(token[i+1:]) + } + } + return token +} + +func b64url(b []byte) string { + return base64.RawURLEncoding.EncodeToString(b) +} + +func pad32(b []byte) []byte { + if len(b) >= 32 { + return b[len(b)-32:] + } + out := make([]byte, 32) + copy(out[32-len(b):], b) + return out +} + +func pickStr(v, def string) string { + if v != "" { + return v + } + return def +} + +func pickI64(v, def int64) int64 { + if v != 0 { + return v + } + return def +} + +func (d *DoubaoNew) resolveAuthorization() string { + auth := trimTokenScheme(d.Authorization) + if auth == "" { + return "" + } + return "DPoP " + auth +} + +func shouldRefreshJWT(token string) bool { + if token == "" { + return true + } + var payload JWTPayload + if err := ParseJWTPayload(token, &payload); err != nil { + return true + } + if payload.Exp <= 0 { + return false + } + return payload.Exp <= time.Now().Unix()+defaultAuthRefreshAheadSeconds +} + +func (d *DoubaoNew) fetchBizAuth(dpop string, public bool) (string, error) { + var reqUrl string + client := base.RestyClient.Clone() + req := client.R() + req.SetHeader("accept", "application/json, text/javascript") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + req.SetHeader("content-type", "application/x-www-form-urlencoded") + if public { + reqUrl = DoubaoURL + "/passport/anonymity_user/biz_auth/" + } else { + reqUrl = DoubaoURL + "/passport/user/biz_auth/" + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + if csrf := strings.TrimSpace(cookie.GetStr(d.Cookie, "passport_csrf_token")); csrf != "" { + req.SetHeader("x-tt-passport-csrf-token", csrf) + } + } + if oldAuth := d.resolveAuthorization(); oldAuth != "" { + req.SetHeader("authorization", oldAuth) + } + } + if dpop != "" { + req.SetHeader("dpop", dpop) + } + values := url.Values{} + values.Set("client_id", d.AuthClientID) + values.Set("client_type", d.AuthClientType) + values.Set("scope", d.AuthScope) + values.Set("d_pop", dpop) + req.SetBody(values.Encode()) + req.SetQueryParam("aid", d.AppID) + req.SetQueryParam("account_sdk_source", d.AuthSDKSource) + req.SetQueryParam("sdk_version", d.AuthSDKVersion) + + res, err := req.Post(reqUrl) + if err != nil { + return "", err + } + var resp bizAuthResp + if err = json.Unmarshal(res.Body(), &resp); err != nil { + return "", err + } + if resp.Message != "success" || resp.Data.AccessToken == "" { + return "", fmt.Errorf("[doubao_new] %s: %s", resp.Message, resp.Data.Description) + } + return resp.Data.AccessToken, nil +} + +func (d *DoubaoNew) refreshAuthorizationWithDPoP(dpop string) (string, error) { + token, err := d.fetchBizAuth(dpop, false) + if err == nil && token != "" { + return token, nil + } + if err == nil { + err = errors.New("biz auth refresh failed") + } + return "", err +} + +func (d *DoubaoNew) resolveDpopForRequest(method, rawURL string) (string, error) { + if d.DPoPKeyPair != nil { + proof, err := GenerateDPoPToken(DPoPTokenInput{ + KeyPair: d.DPoPKeyPair, + HTM: strings.ToUpper(strings.TrimSpace(method)), + HTU: normalizeDPoPURL(rawURL), + }) + if err != nil { + return "", err + } + return proof.DPoPToken, nil + } + + static := d.DPoP + if static == "" { + return "", nil + } + if !d.IgnoreJWTCheck { + if payload, err := parseDPoPPayload(static); err == nil && payload.Exp > 0 { + now := time.Now().Unix() + if payload.Exp <= now+defaultDpopRefreshAheadSeconds { + return "", errors.New("static dpop token expired or near expiry; configure dpop_key_pair for automatic refresh") + } + } + } + return static, nil +} + +func (d *DoubaoNew) ensureAuthAdditons() bool { + return d.DPoPKeySecret != "" && d.AuthClientID != "" && d.AuthClientType != "" && + d.AuthScope != "" && d.AuthSDKSource != "" && d.AuthSDKVersion != "" +} + +func (d *DoubaoNew) resolveAuthorizationForRequest(method, rawURL string) (string, error) { + if !shouldRefreshJWT(d.Authorization) { + return d.resolveAuthorization(), nil + } + + if d.DPoPKeyPair == nil || strings.TrimSpace(d.Cookie) == "" || !d.ensureAuthAdditons() { + return d.resolveAuthorization(), nil + } + + d.authRefreshMu.Lock() + defer d.authRefreshMu.Unlock() + + if !shouldRefreshJWT(d.Authorization) { + return d.resolveAuthorization(), nil + } + + refreshDpop, err := d.resolveDpopForRequest(method, rawURL) + if err != nil || refreshDpop == "" { + return "", err + } + + newToken, err := d.refreshAuthorizationWithDPoP(refreshDpop) + if err != nil { + if auth := d.resolveAuthorization(); auth != "" { + return auth, nil + } + return "", err + } + d.Authorization = trimTokenScheme(newToken) + return d.resolveAuthorization(), nil +} + +func (d *DoubaoNew) resolveAuthorizationForPublic() (dpop string, auth string, err error) { + if d.DPoPPublic != "" && !shouldRefreshJWT(d.AuthorizationPublic) { + return d.DPoPPublic, "DPoP " + d.AuthorizationPublic, nil + } + + if !d.ensureAuthAdditons() { + return "", "", fmt.Errorf("[doubao_new] missing auth additions, please fill them all") + } + + d.authRefreshPublicMu.Lock() + defer d.authRefreshPublicMu.Unlock() + + if d.DPoPPublic != "" && !shouldRefreshJWT(d.AuthorizationPublic) { + return d.DPoPPublic, "DPoP " + d.AuthorizationPublic, nil + } + + // generate new public dpop + keypair, err := GenerateDPoPKeyPair() + if err != nil { + return "", "", err + } + proof, err := GenerateDPoPToken(DPoPTokenInput{ + KeyPair: keypair, + }) + d.DPoPPublic = proof.DPoPToken + + // get authorization token + d.AuthorizationPublic, err = d.fetchBizAuth(proof.DPoPToken, true) + if err != nil { + return "", "", err + } + + return d.DPoPPublic, "DPoP " + d.AuthorizationPublic, nil +} + +func (d *DoubaoNew) applyAuthHeaders(req *resty.Request, method, rawURL string) error { + auth, err := d.resolveAuthorizationForRequest(method, rawURL) + if err != nil { + return err + } + if auth != "" { + req.SetHeader("authorization", auth) + } + dpop, err := d.resolveDpopForRequest(method, rawURL) + if err != nil { + return err + } + if dpop != "" { + req.SetHeader("dpop", dpop) + } + return nil +} + +func normalizeDPoPURL(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + u.Fragment = "" + return u.String() +} + +func parseDPoPPayload(token string) (*JWTPayload, error) { + var payload JWTPayload + if err := ParseJWTPayload(token, &payload); err != nil { + return nil, err + } + return &payload, nil +} diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go new file mode 100644 index 000000000..2a90cf16c --- /dev/null +++ b/drivers/doubao_new/driver.go @@ -0,0 +1,529 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/cookie" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +type DoubaoNew struct { + model.Storage + Addition + TtLogid string + + // DPoP access token (Authorization header value, without DPoP prefix) + Authorization string + AuthorizationPublic string + // DPoP header value + DPoP string + DPoPPublic string + // DPoP key pair for generating DPoP + DPoPKeyPairStr string + DPoPKeyPair *ecdsa.PrivateKey + + authRefreshMu sync.Mutex + authRefreshPublicMu sync.Mutex +} + +func (d *DoubaoNew) Config() driver.Config { + return config +} + +func (d *DoubaoNew) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *DoubaoNew) Init(ctx context.Context) error { + if cookieStr := strings.TrimSpace(d.Cookie); cookieStr != "" { + d.Cookie = cookieStr + auth := trimTokenScheme(cookie.GetStr(d.Cookie, "LARK_SUITE_ACCESS_TOKEN")) + if auth != "" { + d.Authorization = auth + } + dpop := strings.TrimSpace(cookie.GetStr(d.Cookie, "LARK_SUITE_DPOP")) + if dpop != "" { + d.DPoP = dpop + } + keypair := strings.TrimSpace(cookie.GetStr(d.Cookie, "feishu_dpop_keypair")) + if keypair != "" && d.DPoPKeySecret != "" { + d.DPoPKeyPairStr = keypair + d.DPoPKeyPair, _ = parseEncryptedDPoPKeyPair(keypair, d.DPoPKeySecret) + } + } + return nil +} + +func (d *DoubaoNew) Drop(ctx context.Context) error { + if d.Authorization != "" { + d.Cookie = cookie.SetStr(d.Cookie, "LARK_SUITE_ACCESS_TOKEN", d.Authorization) + } + if d.DPoP != "" { + d.Cookie = cookie.SetStr(d.Cookie, "LARK_SUITE_DPOP", d.DPoP) + } + if d.DPoPKeyPairStr != "" { + d.Cookie = cookie.SetStr(d.Cookie, "feishu_dpop_keypair", d.DPoPKeyPairStr) + } + op.MustSaveDriverStorage(d) + return nil +} + +func (d *DoubaoNew) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + nodes, err := d.listAllChildren(ctx, dir.GetID()) + if err != nil { + return nil, err + } + + objs := make([]model.Obj, 0, len(nodes)) + for _, node := range nodes { + if node.NodeToken == "" || node.ObjToken == "" { + continue + } + + size := parseSize(node.Extra.Size) + isFolder := node.Type == 0 + if isFolder && node.NodeToken == dir.GetID() { + continue + } + + obj := &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: dir.GetID(), + Name: node.Name, + Size: size, + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: isFolder, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + } + objs = append(objs, obj) + } + + return objs, nil +} + +func (d *DoubaoNew) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj, ok := file.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + if obj.IsFolder { + return nil, fmt.Errorf("link is directory") + } + var ( + err error + auth, dpop string + ) + if d.ShareLink { + err := d.createShare(ctx, obj) + if err != nil { + return nil, err + } + dpop, auth, err = d.resolveAuthorizationForPublic() + } else { + // TODO: append previewLink() with auth args to support ShareLink + if args.Type == "preview" || args.Type == "thumb" { + if link, err := d.previewLink(ctx, obj, args); err == nil { + return link, nil + } + } + auth = d.resolveAuthorization() + dpop, err = d.resolveDpopForRequest(http.MethodGet, DownloadBaseURL+"/space/api/box/stream/download/all/"+obj.ObjToken+"/") + } + if err != nil { + return nil, err + } + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + + query := url.Values{} + query.Set("authorization", auth) + query.Set("dpop", dpop) + + downloadURL := DownloadBaseURL + "/space/api/box/stream/download/all/" + obj.ObjToken + "/?" + query.Encode() + + headers := http.Header{ + "Referer": []string{DoubaoURL + "/"}, + "User-Agent": []string{base.UserAgent}, + } + + return &model.Link{ + URL: downloadURL, + Header: headers, + }, nil +} + +func (d *DoubaoNew) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + node, err := d.createFolder(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + return &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: parentDir.GetID(), + Name: node.Name, + Size: parseSize(node.Extra.Size), + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: true, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + }, nil +} + +func (d *DoubaoNew) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if dstDir == nil { + return nil, errors.New("nil destination dir") + } + srcToken := srcObj.GetID() + if srcToken == "" { + if obj, ok := srcObj.(*Object); ok { + srcToken = obj.ObjToken + } + } + if srcToken == "" { + return nil, errors.New("missing source token") + } + if err := d.moveObj(ctx, srcToken, dstDir.GetID()); err != nil { + return nil, err + } + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Path = dstDir.GetID() + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if srcObj.IsDir() { + if err := d.renameFolder(ctx, srcObj.GetID(), newName); err != nil { + return nil, err + } + } else { + fileToken := "" + if obj, ok := srcObj.(*Object); ok { + fileToken = obj.ObjToken + } + if fileToken == "" { + fileToken = srcObj.GetID() + } + if err := d.renameFile(ctx, fileToken, newName); err != nil { + return nil, err + } + } + + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Name = newName + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Remove(ctx context.Context, obj model.Obj) error { + if obj == nil { + return errors.New("nil object") + } + token := obj.GetID() + if token == "" { + if o, ok := obj.(*Object); ok { + token = o.ObjToken + } + } + if token == "" { + return errors.New("missing object token") + } + return d.removeObj(ctx, []string{token}) +} + +func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if file == nil { + return nil, errors.New("nil file") + } + if file.GetSize() <= 0 { + return nil, errors.New("invalid file size") + } + + uploadPrep, err := d.prepareUpload(ctx, file.GetName(), file.GetSize(), dstDir.GetID()) + if err != nil { + return nil, err + } + if uploadPrep.BlockSize <= 0 { + return nil, errors.New("invalid block size from prepare") + } + + tmpFile, err := utils.CreateTempFile(file, file.GetSize()) + if err != nil { + return nil, err + } + defer tmpFile.Close() + + blockSize := uploadPrep.BlockSize + totalSize := file.GetSize() + numBlocks := int((totalSize + blockSize - 1) / blockSize) + blocks := make([]UploadBlock, 0, numBlocks) + blockMeta := make(map[int]UploadBlock, numBlocks) + + for seq := 0; seq < numBlocks; seq++ { + offset := int64(seq) * blockSize + length := blockSize + if remain := totalSize - offset; remain < length { + length = remain + } + buf := make([]byte, int(length)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + buf = buf[:n] + sum := sha256.Sum256(buf) + hash := base64.StdEncoding.EncodeToString(sum[:]) + checksum := adler32String(buf) + + block := UploadBlock{ + Hash: hash, + Seq: seq, + Size: int64(n), + Checksum: checksum, + IsUploaded: true, + } + blocks = append(blocks, block) + blockMeta[seq] = block + } + + needed, err := d.uploadBlocks(ctx, uploadPrep.UploadID, blocks, "explorer") + if err != nil { + return nil, err + } + + if len(needed.NeededUploadBlocks) > 0 { + sort.Slice(needed.NeededUploadBlocks, func(i, j int) bool { + return needed.NeededUploadBlocks[i].Seq < needed.NeededUploadBlocks[j].Seq + }) + const maxMergeBlockCount = 20 + var ( + groupSeqs []int + groupChecksums []string + groupSizes []int64 + groupRealSize int64 + groupExpectSum int64 + groupBuf bytes.Buffer + uploadedBytes int64 + ) + + flushGroup := func() error { + if len(groupSeqs) == 0 { + return nil + } + data := groupBuf.Bytes() + expectLen := groupExpectSum + if int64(len(data)) != expectLen { + return fmt.Errorf("[doubao_new] merge blocks invalid body len: got=%d expect=%d seqs=%v", len(data), expectLen, groupSeqs) + } + mergeResp, err := d.mergeUploadBlocks(ctx, uploadPrep.UploadID, groupSeqs, groupChecksums, groupSizes, blockSize, data) + if err != nil { + return err + } + if len(mergeResp.SuccessSeqList) != len(groupSeqs) { + return fmt.Errorf("[doubao_new] merge blocks incomplete: %v", mergeResp.SuccessSeqList) + } + success := make(map[int]bool, len(mergeResp.SuccessSeqList)) + for _, seq := range mergeResp.SuccessSeqList { + success[seq] = true + } + for _, seq := range groupSeqs { + if !success[seq] { + return fmt.Errorf("[doubao_new] merge blocks missing seq %d", seq) + } + } + + uploadedBytes += groupRealSize + groupSeqs = groupSeqs[:0] + groupChecksums = groupChecksums[:0] + groupSizes = groupSizes[:0] + groupRealSize = 0 + groupExpectSum = 0 + groupBuf.Reset() + if up != nil { + percent := float64(uploadedBytes) / float64(totalSize) * 100 + up(percent) + } + return nil + } + + for _, item := range needed.NeededUploadBlocks { + if _, ok := blockMeta[item.Seq]; !ok { + return nil, fmt.Errorf("[doubao_new] missing block meta for seq %d", item.Seq) + } + if item.Size <= 0 { + return nil, fmt.Errorf("[doubao_new] invalid block size from needed list: seq=%d size=%d", item.Seq, item.Size) + } + offset := int64(item.Seq) * blockSize + buf := make([]byte, int(item.Size)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + if n != len(buf) { + return nil, fmt.Errorf("[doubao_new] short read: seq=%d want=%d got=%d", item.Seq, len(buf), n) + } + buf = buf[:n] + realAdler := adler32String(buf) + if realAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] block checksum mismatch: seq=%d offset=%d adler32=%s step2=%s", item.Seq, offset, realAdler, item.Checksum) + } + payloadStart := groupBuf.Len() + groupBuf.Write(buf) + payloadEnd := groupBuf.Len() + payloadAdler := adler32String(groupBuf.Bytes()[payloadStart:payloadEnd]) + if payloadAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] payload checksum mismatch: seq=%d start=%d end=%d adler32=%s step2=%s", item.Seq, payloadStart, payloadEnd, payloadAdler, item.Checksum) + } + groupSeqs = append(groupSeqs, item.Seq) + groupChecksums = append(groupChecksums, item.Checksum) + groupSizes = append(groupSizes, item.Size) + groupRealSize += int64(n) + groupExpectSum += item.Size + if len(groupSeqs) >= maxMergeBlockCount { + if err := flushGroup(); err != nil { + return nil, err + } + } + } + + if err := flushGroup(); err != nil { + return nil, err + } + if up != nil { + up(100) + } + } else if up != nil { + up(100) + } + + numBlocksFinish := uploadPrep.NumBlocks + if numBlocksFinish <= 0 { + numBlocksFinish = numBlocks + } + finish, err := d.finishUpload(ctx, uploadPrep.UploadID, numBlocksFinish, "explorer") + if err != nil { + return nil, err + } + + nodeToken := finish.Extra.NodeToken + if nodeToken == "" { + nodeToken = finish.FileToken + } + now := time.Now() + return &Object{ + Object: model.Object{ + ID: nodeToken, + Path: dstDir.GetID(), + Name: file.GetName(), + Size: file.GetSize(), + Modified: now, + Ctime: now, + IsFolder: false, + }, + ObjToken: finish.FileToken, + }, nil +} + +func (d *DoubaoNew) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + data, err := d.getUserStorage(ctx) + if err != nil { + return nil, err + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: data.TotalSizeLimitBytes, + UsedSpace: data.UsedSizeBytes, + }, + }, nil +} + +func (d *DoubaoNew) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch args.Method { + case "doubao_preview", "preview": + obj, ok := args.Obj.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errs.NotSupport + } + + imgExt := ".webp" + pageNums := 1 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + if extra.PageNums > 0 { + pageNums = extra.PageNums + } + } + } + + return base.Json{ + "version": info.Version, + "img_ext": imgExt, + "page_nums": pageNums, + }, nil + default: + return nil, errs.NotSupport + } +} + +var _ driver.Driver = (*DoubaoNew)(nil) diff --git a/drivers/doubao_new/meta.go b/drivers/doubao_new/meta.go new file mode 100644 index 000000000..2345357bf --- /dev/null +++ b/drivers/doubao_new/meta.go @@ -0,0 +1,39 @@ +package doubao_new + +import ( + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + Cookie string `json:"cookie" required:"true" help:"Web Cookie"` + AppID string `json:"app_id" required:"true" default:"497858" help:"Doubao App ID"` + DPoPKeySecret string `json:"dpop_key_secret" help:"DPoP Key Secret for generating DPoP token"` + AuthClientID string `json:"auth_client_id" help:"Doubao Biz Auth Client ID"` + AuthClientType string `json:"auth_client_type" help:"Doubao Biz Auth Client Type"` + AuthScope string `json:"auth_scope" help:"Doubao Biz Auth Scope"` + AuthSDKSource string `json:"auth_sdk_source" help:"Doubao Biz Auth SDK Source"` + AuthSDKVersion string `json:"auth_sdk_version" help:"Doubao Biz Auth SDK Version"` + ShareLink bool `json:"share_link" help:"Whether to use share link for download"` + IgnoreJWTCheck bool `json:"ignore_jwt_check" help:"Whether to ignore JWT check to prevent time issue"` +} + +var config = driver.Config{ + Name: "DoubaoNew", + LocalSort: true, + DefaultRoot: "", + Alert: `danger|Do not use 302 if the storage is public accessible. +Otherwise, the download link may leak sensitive information such as access token or signature. +Others may use the leaked link to access all your files.`, + NoOverwriteUpload: false, + PreferProxy: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &DoubaoNew{} + }) +} diff --git a/drivers/doubao_new/types.go b/drivers/doubao_new/types.go new file mode 100644 index 000000000..3b64a2a20 --- /dev/null +++ b/drivers/doubao_new/types.go @@ -0,0 +1,192 @@ +package doubao_new + +import "github.com/OpenListTeam/OpenList/v4/internal/model" + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` +} + +type ListResp struct { + BaseResp + Data ListData `json:"data"` +} + +type ListData struct { + HasMore bool `json:"has_more"` + LastLabel string `json:"last_label"` + NodeList []string `json:"node_list"` + Entities struct { + Nodes map[string]Node `json:"nodes"` + Users map[string]User `json:"users"` + } `json:"entities"` +} + +type Node struct { + Token string `json:"token"` + NodeToken string `json:"node_token"` + ObjToken string `json:"obj_token"` + Name string `json:"name"` + Type int `json:"type"` + NodeType int `json:"node_type"` + OwnerID string `json:"owner_id"` + EditUID string `json:"edit_uid"` + CreateTime int64 `json:"create_time"` + EditTime int64 `json:"edit_time"` + URL string `json:"url"` + Extra struct { + Size string `json:"size"` + } `json:"extra"` +} + +type User struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type Object struct { + model.Object + ObjToken string + NodeType int + ObjType int + URL string +} + +type CreateFolderResp struct { + BaseResp + Data struct { + Entities struct { + Nodes map[string]Node `json:"nodes"` + } `json:"entities"` + NodeList []string `json:"node_list"` + } `json:"data"` +} + +type FileInfoResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileInfo `json:"data"` +} + +type FileInfo struct { + Name string `json:"name"` + NumBlocks int `json:"num_blocks"` + Version string `json:"version"` + MimeType string `json:"mime_type"` + MountPoint string `json:"mount_point"` + PreviewMeta PreviewMeta `json:"preview_meta"` +} + +type PreviewMeta struct { + Data map[string]PreviewMetaEntry `json:"data"` +} + +type PreviewMetaEntry struct { + Status int `json:"status"` + Extra string `json:"extra"` + PreviewFileSize int64 `json:"preview_file_size"` +} + +type PreviewImageExtra struct { + ImgExt string `json:"img_ext"` + PageNums int `json:"page_nums"` +} + +type UserStorageResp struct { + BaseResp + Data UserStorageData `json:"data"` +} + +type UserStorageData struct { + ShowSizeLimit bool `json:"show_size_limit"` + TotalSizeLimitBytes int64 `json:"total_size_limit_bytes"` + UsedSizeBytes int64 `json:"used_size_bytes"` +} + +type UploadPrepareResp struct { + BaseResp + Data UploadPrepareData `json:"data"` +} + +type UploadPrepareData struct { + BlockSize int64 `json:"block_size"` + NumBlocks int `json:"num_blocks"` + OptionBlockSize int64 `json:"option_block_size"` + DedupeSupport bool `json:"dedupe_support"` + UploadID string `json:"upload_id"` +} + +type UploadBlock struct { + Hash string `json:"hash"` + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + IsUploaded bool `json:"isUploaded"` +} + +type UploadBlocksResp struct { + BaseResp + Data UploadBlocksData `json:"data"` +} + +type UploadBlocksData struct { + NeededUploadBlocks []UploadBlockNeed `json:"needed_upload_blocks"` +} + +type UploadBlockNeed struct { + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + Hash string `json:"hash"` +} + +type UploadMergeResp struct { + BaseResp + Data UploadMergeData `json:"data"` +} + +type UploadMergeData struct { + SuccessSeqList []int `json:"success_seq_list"` +} + +type UploadFinishResp struct { + BaseResp + Data UploadFinishData `json:"data"` +} + +type UploadFinishData struct { + Version string `json:"version"` + DataVersion string `json:"data_version"` + Extra struct { + NodeToken string `json:"node_token"` + } `json:"extra"` + FileToken string `json:"file_token"` +} + +type RemoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + } `json:"data"` +} + +type TaskStatusResp struct { + BaseResp + Data TaskStatusData `json:"data"` +} + +type TaskStatusData struct { + IsFinish bool `json:"is_finish"` + IsFail bool `json:"is_fail"` +} + +type bizAuthResp struct { + Data struct { + AccessToken string `json:"access_token"` + AuthScheme string `json:"auth_scheme"` + ExpiresIn int64 `json:"expires_in"` + Description string `json:"description,omitempty"` + } `json:"data"` + Message string `json:"message"` +} diff --git a/drivers/doubao_new/upload.go b/drivers/doubao_new/upload.go new file mode 100644 index 000000000..6f3a8b068 --- /dev/null +++ b/drivers/doubao_new/upload.go @@ -0,0 +1,283 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/go-resty/resty/v2" +) + +func (d *DoubaoNew) prepareUpload(ctx context.Context, name string, size int64, mountNodeToken string) (UploadPrepareData, error) { + var resp UploadPrepareResp + _, err := d.request(ctx, "/space/api/box/upload/prepare/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.prepare") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + body := base.Json{ + "mount_point": "explorer", + "mount_node_token": "", + "name": name, + "size": size, + "size_checker": true, + } + if mountNodeToken != "" { + body["mount_node_token"] = mountNodeToken + } + req.SetBody(body) + }, &resp) + if err != nil { + return UploadPrepareData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlocks(ctx context.Context, uploadID string, blocks []UploadBlock, mountPoint string) (UploadBlocksData, error) { + if uploadID == "" { + return UploadBlocksData{}, fmt.Errorf("[doubao_new] upload blocks missing upload_id") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadBlocksResp + _, err := d.request(ctx, "/space/api/box/upload/blocks/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.blocks") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetBody(base.Json{ + "blocks": blocks, + "upload_id": uploadID, + "mount_point": mountPoint, + }) + }, &resp) + if err != nil { + return UploadBlocksData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) mergeUploadBlocks(ctx context.Context, uploadID string, seqList []int, checksumList []string, sizeList []int64, blockOriginSize int64, data []byte) (UploadMergeData, error) { + if uploadID == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing upload_id") + } + if len(seqList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty seq list") + } + if len(checksumList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty checksum list") + } + if len(sizeList) != len(seqList) { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks size list mismatch") + } + if blockOriginSize <= 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks invalid block origin size") + } + if len(data) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty data") + } + + seqHeader := joinIntComma(seqList) + checksumHeader := buildCommaHeader(checksumList) + + client := base.NewRestyClient() + client.SetCookieJar(nil) + req := client.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("content-type", "application/octet-stream") + req.SetHeader("x-block-list-checksum", checksumHeader) + req.SetHeader("x-seq-list", seqHeader) + req.SetHeader("x-block-origin-size", strconv.FormatInt(blockOriginSize, 10)) + req.SetHeader("x-command", "space.api.box.stream.upload.merge_block") + req.SetHeader("x-csrftoken", "") + reqID := "" + if buf := make([]byte, 16); true { + if _, err := rand.Read(buf); err == nil { + reqID = hex.EncodeToString(buf) + } + } + if reqID != "" { + req.SetHeader("x-request-id", reqID) + } + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + urlStr := DownloadBaseURL + "/space/api/box/stream/upload/merge_block/?" + values.Encode() + if err := d.applyAuthHeaders(req, http.MethodPost, urlStr); err != nil { + return UploadMergeData{}, err + } + req.Header.Del("cookie") + if req.Header.Get("x-command") == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing x-command header") + } + req.SetBody(data) + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return UploadMergeData{}, err + } + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + body := res.Body() + var resp UploadMergeResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UploadMergeData{}, fmt.Errorf("%s", msg) + } + if resp.Code != 0 { + if res != nil && res.StatusCode() == http.StatusBadRequest && resp.Code == 2 { + success := make([]int, 0, len(seqList)) + offset := 0 + for i, seq := range seqList { + size := sizeList[i] + if size <= 0 { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback invalid size: seq=%d size=%d", seq, size) + } + if offset+int(size) > len(data) { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback payload out of range: seq=%d offset=%d size=%d total=%d", seq, offset, size, len(data)) + } + payload := data[offset : offset+int(size)] + block := UploadBlockNeed{ + Seq: seq, + Size: size, + Checksum: checksumList[i], + } + if err := d.uploadBlockV3(ctx, uploadID, block, payload); err != nil { + return UploadMergeData{SuccessSeqList: success}, err + } + success = append(success, seq) + offset += int(size) + } + return UploadMergeData{SuccessSeqList: success}, nil + } + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UploadMergeData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlockV3(ctx context.Context, uploadID string, block UploadBlockNeed, data []byte) error { + if uploadID == "" { + return fmt.Errorf("[doubao_new] upload v3 block missing upload_id") + } + if block.Seq < 0 { + return fmt.Errorf("[doubao_new] upload v3 block invalid seq") + } + if len(data) == 0 { + return fmt.Errorf("[doubao_new] upload v3 block empty data") + } + + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("x-block-seq", strconv.Itoa(block.Seq)) + req.SetHeader("x-block-checksum", block.Checksum) + req.SetMultipartFormData(map[string]string{ + "upload_id": uploadID, + "size": strconv.FormatInt(int64(len(data)), 10), + }) + req.SetMultipartField("file", "blob", "application/octet-stream", bytes.NewReader(data)) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("seq", strconv.Itoa(block.Seq)) + values.Set("size", strconv.FormatInt(int64(len(data)), 10)) + values.Set("checksum", block.Checksum) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + urlStr := DownloadBaseURL + "/space/api/box/stream/upload/v3/block/?" + values.Encode() + if err := d.applyAuthHeaders(req, http.MethodPost, urlStr); err != nil { + return err + } + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return err + } + body := res.Body() + if err := decodeBaseResp(body, res); err != nil { + return err + } + return nil +} + +func (d *DoubaoNew) finishUpload(ctx context.Context, uploadID string, numBlocks int, mountPoint string) (UploadFinishData, error) { + if uploadID == "" { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload missing upload_id") + } + if numBlocks <= 0 { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload invalid num_blocks") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadFinishResp + _, err := d.request(ctx, "/space/api/box/upload/finish/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", d.AppID) + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.finish") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetHeader("biz-scene", "file_upload") + req.SetHeader("biz-ua-type", "Web") + req.SetBody(base.Json{ + "upload_id": uploadID, + "num_blocks": numBlocks, + "mount_point": mountPoint, + "push_open_history_record": 1, + }) + }, &resp) + if err != nil { + return UploadFinishData{}, err + } + return resp.Data, nil +} diff --git a/drivers/doubao_new/util.go b/drivers/doubao_new/util.go new file mode 100644 index 000000000..8ee38625d --- /dev/null +++ b/drivers/doubao_new/util.go @@ -0,0 +1,713 @@ +package doubao_new + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "hash/adler32" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/OpenListTeam/OpenList/v4/drivers/base" + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/cookie" + "github.com/go-resty/resty/v2" +) + +const ( + BaseURL = "https://my.feishu.cn" + DownloadBaseURL = "https://internal-api-drive-stream.feishu.cn" + DoubaoURL = "https://www.doubao.com" +) + +var defaultObjTypes = []string{"124", "0", "12", "30", "123", "22"} + +func (d *DoubaoNew) request(ctx context.Context, path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, method, BaseURL+path); err != nil { + return nil, err + } + + if callback != nil { + callback(req) + } + + res, err := req.Execute(method, BaseURL+path) + if err != nil { + return nil, err + } + if res != nil { + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + } + + body := res.Body() + var common BaseResp + if err = json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return body, fmt.Errorf("%s", msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return body, fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + if resp != nil { + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func adler32String(data []byte) string { + sum := adler32.Checksum(data) + return strconv.FormatUint(uint64(sum), 10) +} + +func buildCommaHeader(items []string) string { + return strings.Join(items, ",") +} + +func joinIntComma(items []int) string { + if len(items) == 0 { + return "" + } + var sb strings.Builder + for i, v := range items { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(strconv.Itoa(v)) + } + return sb.String() +} + +func previewList(items []string, n int) string { + if n <= 0 || len(items) == 0 { + return "" + } + if len(items) < n { + n = len(items) + } + return strings.Join(items[:n], ",") +} + +func parseSize(size string) int64 { + if size == "" { + return 0 + } + val, err := strconv.ParseInt(size, 10, 64) + if err != nil { + return 0 + } + return val +} + +func (d *DoubaoNew) listChildren(ctx context.Context, parentToken string, lastLabel string, length int) (ListData, error) { + var resp ListResp + _, err := d.request(ctx, "/space/api/explorer/doubao/children/list/", http.MethodGet, func(req *resty.Request) { + values := url.Values{} + for _, t := range defaultObjTypes { + values.Add("obj_type", t) + } + values.Set("length", strconv.Itoa(length)) + values.Set("rank", "0") + values.Set("asc", "0") + values.Set("min_length", "40") + values.Set("thumbnail_width", "1028") + values.Set("thumbnail_height", "1028") + values.Set("thumbnail_policy", "4") + if parentToken != "" { + values.Set("token", parentToken) + } + if lastLabel != "" { + values.Set("last_label", lastLabel) + } + req.SetQueryParamsFromValues(values) + }, &resp) + if err != nil { + return ListData{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) listAllChildren(ctx context.Context, parentToken string) ([]Node, error) { + length := 50 + nodes := make([]Node, 0, length) + lastLabel := "" + for range 100 { + data, err := d.listChildren(ctx, parentToken, lastLabel, length) + if err != nil { + return nil, err + } + + if len(data.NodeList) > 0 { + for _, token := range data.NodeList { + node, ok := data.Entities.Nodes[token] + if !ok { + continue + } + nodes = append(nodes, node) + } + } else { + for _, node := range data.Entities.Nodes { + nodes = append(nodes, node) + } + } + + if !data.HasMore || data.LastLabel == "" || data.LastLabel == lastLabel { + break + } + lastLabel = data.LastLabel + } + + if len(nodes) == 0 { + return nil, nil + } + return nodes, nil +} + +func (d *DoubaoNew) getFileInfo(ctx context.Context, fileToken string) (FileInfo, error) { + var resp FileInfoResp + _, err := d.request(ctx, "/space/api/box/file/info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "caller": "explorer", + "file_token": fileToken, + "mount_point": "explorer", + "option_params": []string{"preview_meta", "check_cipher"}, + }) + }, &resp) + if err != nil { + return FileInfo{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) previewLink(ctx context.Context, obj *Object, args model.LinkArgs) (*model.Link, error) { + auth := d.resolveAuthorization() + dpop, err := d.resolveDpopForRequest(http.MethodGet, fmt.Sprintf("%s/space/api/box/stream/download/preview_sub/%s", BaseURL, obj.ObjToken)) + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errors.New("preview not available") + } + + subID := "" + pageIndex := 0 + + if subID == "" { + imgExt := ".webp" + pageNums := 0 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + pageNums = extra.PageNums + } + } + if pageNums > 0 && pageIndex >= pageNums { + pageIndex = pageNums - 1 + } + subID = fmt.Sprintf("img_%d%s", pageIndex, imgExt) + } + + query := url.Values{} + query.Set("preview_type", "22") + query.Set("sub_id", subID) + if info.Version != "" { + query.Set("version", info.Version) + } + previewURL := fmt.Sprintf("%s/space/api/box/stream/download/preview_sub/%s?%s", BaseURL, obj.ObjToken, query.Encode()) + + headers := http.Header{ + "Referer": []string{DoubaoURL + "/"}, + "User-Agent": []string{base.UserAgent}, + "Authorization": []string{auth}, + "Dpop": []string{dpop}, + } + + return &model.Link{ + URL: previewURL, + Header: headers, + }, nil +} + +func (d *DoubaoNew) createShare(ctx context.Context, obj *Object) error { + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/suite/permission/public/update.v5/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "external_access_entity": 1, + "link_share_entity": 4, + "token": obj.ObjToken, + "type": obj.ObjType, + }) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/suite/permission/public/update.v5/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + if err := decodeBaseResp(body, res); err != nil { + return err + } + return nil +} + +func (d *DoubaoNew) createFolder(ctx context.Context, parentToken, name string) (Node, error) { + data := url.Values{} + data.Set("name", name) + data.Set("source", "0") + if parentToken != "" { + data.Set("parent_token", parentToken) + } + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/explorer/v2/create/folder/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/create/folder/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return Node{}, err + } + if err := decodeBaseResp(body, res); err != nil { + return Node{}, err + } + + var resp CreateFolderResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return Node{}, fmt.Errorf("%s", msg) + } + + var node Node + if len(resp.Data.NodeList) > 0 { + if n, ok := resp.Data.Entities.Nodes[resp.Data.NodeList[0]]; ok { + node = n + } + } + if node.Token == "" { + for _, n := range resp.Data.Entities.Nodes { + node = n + break + } + } + if node.Token == "" && node.ObjToken == "" && node.NodeToken == "" { + return Node{}, fmt.Errorf("[doubao_new] create folder failed: empty response") + } + if node.NodeToken == "" { + if node.Token != "" { + node.NodeToken = node.Token + } else if node.ObjToken != "" { + node.NodeToken = node.ObjToken + } + } + if node.ObjToken == "" && node.Token != "" { + node.ObjToken = node.Token + } + return node, nil +} + +func (d *DoubaoNew) renameFolder(ctx context.Context, token, name string) error { + if token == "" { + return fmt.Errorf("[doubao_new] rename folder missing token") + } + data := url.Values{} + data.Set("token", token) + data.Set("name", name) + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/explorer/v2/rename/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/rename/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func isCsrfTokenError(body []byte, res *resty.Response) bool { + if len(body) == 0 { + return false + } + if strings.Contains(strings.ToLower(string(body)), "csrf token error") { + return true + } + if res != nil && res.StatusCode() == http.StatusForbidden { + return true + } + return false +} + +func doRequestWithCsrf(doRequest func(csrfToken string) (*resty.Response, []byte, error)) (*resty.Response, []byte, error) { + res, body, err := doRequest("") + if err != nil { + return res, body, err + } + if isCsrfTokenError(body, res) { + csrfToken := extractCsrfTokenFromResponse(res) + if csrfToken != "" { + return doRequest(csrfToken) + } + } + return res, body, err +} + +func extractCsrfTokenFromResponse(res *resty.Response) string { + if res == nil || res.Request == nil { + return "" + } + if res.Request.RawRequest != nil { + if csrf := cookie.GetStr(res.Request.RawRequest.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + } + if csrf := cookie.GetStr(res.Request.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + for _, c := range res.Cookies() { + if c.Name == "_csrf_token" { + return c.Value + } + } + return "" +} + +func decodeBaseResp(body []byte, res *resty.Response) error { + var common BaseResp + if err := json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf("%s", msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + return nil +} + +func (d *DoubaoNew) renameFile(ctx context.Context, fileToken, name string) error { + if fileToken == "" { + return fmt.Errorf("[doubao_new] rename file missing file token") + } + _, err := d.request(ctx, "/space/api/box/file/update_info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "file_token": fileToken, + "name": name, + }) + }, nil) + return err +} + +func (d *DoubaoNew) moveObj(ctx context.Context, srcToken, destToken string) error { + if srcToken == "" { + return fmt.Errorf("[doubao_new] move missing src token") + } + data := url.Values{} + data.Set("src_token", srcToken) + if destToken != "" { + data.Set("dest_token", destToken) + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/explorer/v2/move/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/move/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func (d *DoubaoNew) removeObj(ctx context.Context, tokens []string) error { + if len(tokens) == 0 { + return fmt.Errorf("[doubao_new] remove missing tokens") + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodPost, BaseURL+"/space/api/explorer/v3/remove/"); err != nil { + return nil, nil, err + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "tokens": tokens, + "apply": 1, + }) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v3/remove/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + var resp RemoveResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf("%s", msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + if resp.Data.TaskID == "" { + return nil + } + return d.waitTask(ctx, resp.Data.TaskID) +} + +func (d *DoubaoNew) getUserStorage(ctx context.Context) (UserStorageData, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + req.SetHeader("agw-js-conv", "str") + req.SetHeader("content-type", "application/json") + if err := d.applyAuthHeaders(req, http.MethodPost, DoubaoURL+"/alice/aispace/facade/get_user_storage"); err != nil { + return UserStorageData{}, err + } + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + } + req.SetBody(base.Json{}) + + res, err := req.Execute(http.MethodPost, DoubaoURL+"/alice/aispace/facade/get_user_storage") + if err != nil { + return UserStorageData{}, err + } + + body := res.Body() + var resp UserStorageResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UserStorageData{}, fmt.Errorf("%s", msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UserStorageData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) waitTask(ctx context.Context, taskID string) error { + const ( + taskPollInterval = time.Second + taskPollMaxAttempts = 120 + ) + var lastErr error + for attempt := 0; attempt < taskPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, taskPollInterval); err != nil { + return err + } + } + status, err := d.getTaskStatus(ctx, taskID) + if err != nil { + lastErr = err + continue + } + if status.IsFail { + return fmt.Errorf("[doubao_new] remove task failed: %s", taskID) + } + if status.IsFinish { + return nil + } + } + if lastErr != nil { + return lastErr + } + return fmt.Errorf("[doubao_new] remove task timed out: %s", taskID) +} + +func (d *DoubaoNew) getTaskStatus(ctx context.Context, taskID string) (TaskStatusData, error) { + if taskID == "" { + return TaskStatusData{}, fmt.Errorf("[doubao_new] task status missing task_id") + } + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", DoubaoURL) + req.SetHeader("referer", DoubaoURL+"/") + if err := d.applyAuthHeaders(req, http.MethodGet, BaseURL+"/space/api/explorer/v2/task/"); err != nil { + return TaskStatusData{}, err + } + req.SetQueryParam("task_id", taskID) + res, err := req.Execute(http.MethodGet, BaseURL+"/space/api/explorer/v2/task/") + if err != nil { + return TaskStatusData{}, err + } + body := res.Body() + var resp TaskStatusResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return TaskStatusData{}, fmt.Errorf("%s", msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return TaskStatusData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + return resp.Data, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index 94ef854f2..1e2e476ce 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -3,17 +3,29 @@ package google_drive import ( "context" "fmt" + "io" "net/http" "strconv" + "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) +// mkdirLocks prevents race conditions when creating folders with the same name +// Google Drive allows duplicate folder names, so we need application-level locking +var mkdirLocks sync.Map // map[string]*sync.Mutex - key is parentID + "/" + dirName + type GoogleDrive struct { model.Storage Addition @@ -67,15 +79,76 @@ func (d *GoogleDrive) Link(ctx context.Context, file model.Obj, args model.LinkA } func (d *GoogleDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + // Use per-folder lock to prevent concurrent creation of same folder + // This is critical because Google Drive allows duplicate folder names + lockKey := parentDir.GetID() + "/" + dirName + lockVal, _ := mkdirLocks.LoadOrStore(lockKey, &sync.Mutex{}) + lock := lockVal.(*sync.Mutex) + lock.Lock() + defer lock.Unlock() + + // Check if folder already exists with retry to handle API eventual consistency + escapedDirName := strings.ReplaceAll(dirName, "'", "\\'") + query := map[string]string{ + "q": fmt.Sprintf("name='%s' and '%s' in parents and mimeType='application/vnd.google-apps.folder' and trashed=false", escapedDirName, parentDir.GetID()), + "fields": "files(id)", + } + + var existingFiles Files + err := retry.Do(func() error { + var checkErr error + _, checkErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &existingFiles) + return checkErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(200*time.Millisecond), + ) + + // If query succeeded and folder exists, return success (idempotent) + if err == nil && len(existingFiles.Files) > 0 { + log.Debugf("[google_drive] Folder '%s' already exists in parent %s, skipping creation", dirName, parentDir.GetID()) + return nil + } + // If query failed, return error to prevent duplicate creation + if err != nil { + return fmt.Errorf("failed to check existing folder '%s': %w", dirName, err) + } + + // Create new folder (only when confirmed folder doesn't exist) data := base.Json{ "name": dirName, "parents": []string{parentDir.GetID()}, "mimeType": "application/vnd.google-apps.folder", } - _, err := d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { - req.SetBody(data) - }, nil) - return err + + var createErr error + err = retry.Do(func() error { + _, createErr = d.request("https://www.googleapis.com/drive/v3/files", http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + return createErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) + + if err != nil { + return err + } + + // Wait for API eventual consistency before releasing lock + // This helps prevent race conditions where a concurrent request + // checks for folder existence before the newly created folder is visible + // 500ms is needed because Google Drive API has significant sync delay + time.Sleep(500 * time.Millisecond) + + return nil } func (d *GoogleDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -111,8 +184,44 @@ func (d *GoogleDrive) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - obj := stream.GetExist() +func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + // 1. 准备MD5（用于完整性校验） + md5Hash := file.GetHash().GetHash(utils.MD5) + + // 检查是否是可重复读取的流 + _, isSeekable := file.(*stream.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 计算 hash，不缓存 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(file, utils.MD5, 100, &up) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验（Google Drive会自动校验） + } + } else { + // 不可重复读取的流（如 HTTP body） + if len(md5Hash) != utils.MD5.Width { + // 缓存整个文件并计算 MD5 + var err error + _, md5Hash, err = stream.CacheFullAndHash(file, &up, utils.MD5) + if err != nil { + return err + } + _ = md5Hash // MD5用于后续完整性校验 + } else if file.GetFile() == nil { + // 有 MD5 但没有缓存，需要缓存以支持后续 RangeRead + _, err := file.CacheFullAndWriter(&up, nil) + if err != nil { + return err + } + } + } + + // 2. 初始化可恢复上传会话 + obj := file.GetExist() var ( e Error url string @@ -125,7 +234,7 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi data = base.Json{} } else { data = base.Json{ - "name": stream.GetName(), + "name": file.GetName(), "parents": []string{dstDir.GetID()}, } url = "https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true" @@ -133,8 +242,8 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi req := base.NoRedirectClient.R(). SetHeaders(map[string]string{ "Authorization": "Bearer " + d.AccessToken, - "X-Upload-Content-Type": stream.GetMimetype(), - "X-Upload-Content-Length": strconv.FormatInt(stream.GetSize(), 10), + "X-Upload-Content-Type": file.GetMimetype(), + "X-Upload-Content-Length": strconv.FormatInt(file.GetSize(), 10), }). SetError(&e).SetBody(data).SetContext(ctx) if obj != nil { @@ -151,20 +260,40 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi if err != nil { return err } - return d.Put(ctx, dstDir, stream, up) + return d.Put(ctx, dstDir, file, up) } return fmt.Errorf("%s: %v", e.Error.Message, e.Error.Errors) } + + // 3. 上传文件内容 putUrl := res.Header().Get("location") - if stream.GetSize() < d.ChunkSize*1024*1024 { - _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { - req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). - SetBody(driver.NewLimitedUploadStream(ctx, stream)) - }, nil) + if file.GetSize() < d.ChunkSize*1024*1024 { + // 小文件上传：使用 RangeRead 读取整个文件（避免消费已计算hash的stream） + err = retry.Do(func() error { + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } + + _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { + req.SetHeader("Content-Length", strconv.FormatInt(file.GetSize(), 10)). + SetBody(driver.NewLimitedUploadStream(ctx, reader)) + }, nil) + return err + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second), + ) + return err } else { - err = d.chunkUpload(ctx, stream, putUrl, up) + // 大文件分片上传 + return d.chunkUpload(ctx, file, putUrl, up) } - return err } func (d *GoogleDrive) GetDetails(ctx context.Context) (*model.StorageDetails, error) { diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 042abafa4..1fc68c335 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -296,9 +296,60 @@ func (d *GoogleDrive) getFiles(id string) ([]File, error) { res = append(res, resp.Files...) } + + // Handle duplicate filenames by adding suffixes like (1), (2), etc. + // Google Drive allows multiple files with the same name in one folder, + // but OpenList uses path-based file system which requires unique names + res = handleDuplicateNames(res) + return res, nil } +// handleDuplicateNames adds suffixes to duplicate filenames to make them unique +// For example: file.txt, file (1).txt, file (2).txt +func handleDuplicateNames(files []File) []File { + if len(files) <= 1 { + return files + } + + // Track how many files with each name we've seen + nameCount := make(map[string]int) + + // First pass: count occurrences of each name + for _, file := range files { + nameCount[file.Name]++ + } + + // Second pass: add suffixes to duplicates + nameIndex := make(map[string]int) + for i := range files { + name := files[i].Name + if nameCount[name] > 1 { + index := nameIndex[name] + nameIndex[name]++ + + if index > 0 { + // Add suffix for all except the first occurrence + // Split name into base and extension + ext := "" + base := name + for j := len(name) - 1; j >= 0; j-- { + if name[j] == '.' { + ext = name[j:] + base = name[:j] + break + } + } + + // Add (1), (2), etc. suffix + files[i].Name = fmt.Sprintf("%s (%d)%s", base, index, ext) + } + } + } + + return files +} + // getTargetFileInfo gets target file details for shortcuts func (d *GoogleDrive) getTargetFileInfo(targetId string) (File, error) { var targetFile File diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index 01d7c1ece..e143cde20 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -117,7 +117,7 @@ func (d *LanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return &model.Link{ URL: dfile.Url, Header: http.Header{ - "User-Agent": []string{base.UserAgent}, + "User-Agent": {d.UserAgent}, }, Expiration: &exp, }, nil diff --git a/drivers/openlist/driver.go b/drivers/openlist/driver.go index 2ca60ff61..b37d72a06 100644 --- a/drivers/openlist/driver.go +++ b/drivers/openlist/driver.go @@ -14,6 +14,8 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/go-resty/resty/v2" @@ -84,7 +86,7 @@ func (d *OpenList) List(ctx context.Context, dir model.Obj, args model.ListArgs) }, Path: dir.GetPath(), Password: d.MetaPassword, - Refresh: false, + Refresh: d.PassRefreshFlagToUpsteam && args.Refresh, }) }) if err != nil { @@ -195,6 +197,92 @@ func (d *OpenList) Remove(ctx context.Context, obj model.Obj) error { } func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + // 预计算 hash（如果不存在），使用 RangeRead 不消耗 Reader + // 这样远端驱动不需要再计算，避免 HTTP body 被重复读取 + md5Hash := s.GetHash().GetHash(utils.MD5) + sha1Hash := s.GetHash().GetHash(utils.SHA1) + sha256Hash := s.GetHash().GetHash(utils.SHA256) + sha1_128kHash := s.GetHash().GetHash(utils.SHA1_128K) + preHash := s.GetHash().GetHash(utils.PRE_HASH) + + // 计算所有缺失的 hash，确保最大兼容性 + if len(md5Hash) != utils.MD5.Width { + var err error + md5Hash, err = stream.StreamHashFile(s, utils.MD5, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate MD5: %v", err) + md5Hash = "" + } + } + if len(sha1Hash) != utils.SHA1.Width { + var err error + sha1Hash, err = stream.StreamHashFile(s, utils.SHA1, 33, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1: %v", err) + sha1Hash = "" + } + } + if len(sha256Hash) != utils.SHA256.Width { + var err error + sha256Hash, err = stream.StreamHashFile(s, utils.SHA256, 34, &up) + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA256: %v", err) + sha256Hash = "" + } + } + + // 计算特殊 hash（用于秒传验证） + // SHA1_128K: 前128KB的SHA1，115网盘使用 + if len(sha1_128kHash) != utils.SHA1_128K.Width { + const PreHashSize int64 = 128 * 1024 // 128KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + sha1_128kHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate SHA1_128K: %v", err) + sha1_128kHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for SHA1_128K: %v", err) + } + } + + // PRE_HASH: 前1024字节的SHA1，阿里云盘使用 + if len(preHash) != utils.PRE_HASH.Width { + const PreHashSize int64 = 1024 // 1KB + hashSize := PreHashSize + if s.GetSize() < PreHashSize { + hashSize = s.GetSize() + } + reader, err := s.RangeRead(http_range.Range{Start: 0, Length: hashSize}) + if err == nil { + preHash, err = utils.HashReader(utils.SHA1, reader) + if closer, ok := reader.(io.Closer); ok { + _ = closer.Close() + } + if err != nil { + log.Warnf("[openlist] failed to pre-calculate PRE_HASH: %v", err) + preHash = "" + } + } else { + log.Warnf("[openlist] failed to RangeRead for PRE_HASH: %v", err) + } + } + + // 诊断日志：检查流的状态 + if ss, ok := s.(*stream.SeekableStream); ok { + if ss.Reader != nil { + log.Warnf("[openlist] WARNING: SeekableStream.Reader is not nil for file %s, stream may have been consumed!", s.GetName()) + } + } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, @@ -206,14 +294,20 @@ func (d *OpenList) Put(ctx context.Context, dstDir model.Obj, s model.FileStream req.Header.Set("Authorization", d.Token) req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) req.Header.Set("Password", d.MetaPassword) - if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { - req.Header.Set("X-File-Md5", md5) + if len(md5Hash) > 0 { + req.Header.Set("X-File-Md5", md5Hash) + } + if len(sha1Hash) > 0 { + req.Header.Set("X-File-Sha1", sha1Hash) + } + if len(sha256Hash) > 0 { + req.Header.Set("X-File-Sha256", sha256Hash) } - if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { - req.Header.Set("X-File-Sha1", sha1) + if len(sha1_128kHash) > 0 { + req.Header.Set("X-File-Sha1-128k", sha1_128kHash) } - if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { - req.Header.Set("X-File-Sha256", sha256) + if len(preHash) > 0 { + req.Header.Set("X-File-Pre-Hash", preHash) } req.ContentLength = s.GetSize() diff --git a/drivers/openlist/meta.go b/drivers/openlist/meta.go index 16c6a155c..3c4d0801a 100644 --- a/drivers/openlist/meta.go +++ b/drivers/openlist/meta.go @@ -7,14 +7,15 @@ import ( type Addition struct { driver.RootPath - Address string `json:"url" required:"true"` - MetaPassword string `json:"meta_password"` - Username string `json:"username"` - Password string `json:"password"` - Token string `json:"token"` - PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` - PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` - ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` + Address string `json:"url" required:"true"` + MetaPassword string `json:"meta_password"` + Username string `json:"username"` + Password string `json:"password"` + Token string `json:"token"` + PassIPToUpsteam bool `json:"pass_ip_to_upsteam" default:"true"` + PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` + ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` + PassRefreshFlagToUpsteam bool `json:"pass_refresh_flag_to_upsteam" default:"false"` } var config = driver.Config{ diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 9b7207fa2..1d091217a 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -19,6 +19,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/model" + netutil "github.com/OpenListTeam/OpenList/v4/internal/net" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" @@ -418,7 +419,7 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err } func (d *PikPak) UploadByOSS(ctx context.Context, params *S3Params, s model.FileStreamer, up driver.UpdateProgress) error { - ossClient, err := oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret) + ossClient, err := netutil.NewOSSClient(params.Endpoint, params.AccessKeyID, params.AccessKeySecret) if err != nil { return err } @@ -451,7 +452,7 @@ func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSi bucket *oss.Bucket ) - if ossClient, err = oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret); err != nil { + if ossClient, err = netutil.NewOSSClient(params.Endpoint, params.AccessKeyID, params.AccessKeySecret); err != nil { return err } diff --git a/drivers/quark_open/driver.go b/drivers/quark_open/driver.go index f0b8baf09..26a4288cc 100644 --- a/drivers/quark_open/driver.go +++ b/drivers/quark_open/driver.go @@ -8,6 +8,7 @@ import ( "hash" "io" "net/http" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -18,15 +19,23 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" ) type QuarkOpen struct { model.Storage Addition - config driver.Config - conf Conf + config driver.Config + conf Conf + limiter *rate.Limiter } +// 速率限制常量：夸克开放平台限流，保守设置 +const ( + quarkRateLimit = 2.0 // 每秒2个请求，避免限流 +) + func (d *QuarkOpen) Config() driver.Config { return d.config } @@ -36,6 +45,9 @@ func (d *QuarkOpen) GetAddition() driver.Additional { } func (d *QuarkOpen) Init(ctx context.Context) error { + // 初始化速率限制器 + d.limiter = rate.NewLimiter(rate.Limit(quarkRateLimit), 1) + var resp UserInfoResp _, err := d.request(ctx, "/open/v1/user/info", http.MethodGet, nil, &resp) @@ -52,11 +64,22 @@ func (d *QuarkOpen) Init(ctx context.Context) error { return err } +// waitLimit 等待速率限制 +func (d *QuarkOpen) waitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + func (d *QuarkOpen) Drop(ctx context.Context) error { return nil } func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } files, err := d.GetFiles(ctx, dir.GetID()) if err != nil { return nil, err @@ -67,6 +90,9 @@ func (d *QuarkOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs } func (d *QuarkOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.waitLimit(ctx); err != nil { + return nil, err + } data := base.Json{ "fid": file.GetID(), } @@ -143,35 +169,116 @@ func (d *QuarkOpen) Remove(ctx context.Context, obj model.Obj) error { } func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) - var ( - md5 hash.Hash - sha1 hash.Hash - ) - writers := []io.Writer{} - if len(md5Str) != utils.MD5.Width { - md5 = utils.MD5.NewFunc() - writers = append(writers, md5) - } - if len(sha1Str) != utils.SHA1.Width { - sha1 = utils.SHA1.NewFunc() - writers = append(writers, sha1) + if err := d.waitLimit(ctx); err != nil { + return err } + md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) - if len(writers) > 0 { - _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) - if err != nil { - return err - } - if md5 != nil { - md5Str = hex.EncodeToString(md5.Sum(nil)) - } - if sha1 != nil { - sha1Str = hex.EncodeToString(sha1.Sum(nil)) + // 检查是否需要计算hash + needMD5 := len(md5Str) != utils.MD5.Width + needSHA1 := len(sha1Str) != utils.SHA1.Width + + if needMD5 || needSHA1 { + // 检查是否为可重复读取的流 + _, isSeekable := stream.(*streamPkg.SeekableStream) + + if isSeekable { + // 可重复读取的流，使用 RangeRead 一次性计算所有hash，避免重复读取 + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + // 使用 RangeRead 分块读取文件，同时计算多个hash + multiWriter := io.MultiWriter(writers...) + size := stream.GetSize() + chunkSize := int64(10 * utils.MB) // 10MB per chunk + buf := make([]byte, chunkSize) + var offset int64 = 0 + + for offset < size { + readSize := min(chunkSize, size-offset) + + n, err := streamPkg.ReadFullWithRangeRead(stream, buf[:readSize], offset) + if err != nil { + return fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + + multiWriter.Write(buf[:n]) + offset += int64(n) + + // 更新进度（hash计算占用40%的进度） + up(40 * float64(offset) / float64(size)) + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } + } else { + // 不可重复读取的流（如网络流），需要缓存并计算hash + var md5 hash.Hash + var sha1 hash.Hash + writers := []io.Writer{} + + if needMD5 { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if needSHA1 { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + _, err := stream.CacheFullAndWriter(&up, io.MultiWriter(writers...)) + if err != nil { + return err + } + + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } } } - // pre - pre, err := d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + // pre - 带有 proof fail 重试逻辑 + var pre UpPreResp + var err error + err = retry.Do(func() error { + var preErr error + pre, preErr = d.upPre(ctx, stream, dstDir.GetID(), md5Str, sha1Str) + if preErr != nil { + // 检查是否为 proof fail 错误 + if strings.Contains(preErr.Error(), "proof") || strings.Contains(preErr.Error(), "43010") { + log.Warnf("[quark_open] Proof verification failed, retrying: %v", preErr) + return preErr // 返回错误触发重试 + } + // 检查是否为限流错误 + if strings.Contains(preErr.Error(), "限流") || strings.Contains(preErr.Error(), "rate") { + log.Warnf("[quark_open] Rate limited, waiting before retry: %v", preErr) + time.Sleep(2 * time.Second) // 额外等待 + return preErr + } + } + return preErr + }, + retry.Context(ctx), + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(500*time.Millisecond), + ) if err != nil { return err } @@ -181,16 +288,70 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return nil } - // get part info - partInfo := d._getPartInfo(stream, pre.Data.PartSize) - // get upload url info - upUrlInfo, err := d.upUrl(ctx, pre, partInfo) - if err != nil { + // 空文件特殊处理：跳过分片上传，直接调用 upFinish + // 由于夸克 API 对空文件处理不稳定，尝试完成上传，失败则直接成功返回 + if stream.GetSize() == 0 { + log.Infof("[quark_open] Empty file detected, attempting direct finish (task_id: %s)", pre.Data.TaskID) + err = d.upFinish(ctx, pre, []base.Json{}, []string{}) + if err != nil { + // 空文件 upFinish 失败，可能是 API 不支持，直接视为成功 + log.Warnf("[quark_open] Empty file upFinish failed: %v, treating as success", err) + } + up(100) + return nil + } + + // 带重试的分片大小调整逻辑：如果检测到 "part list exceed" 错误，自动翻倍分片大小 + var upUrlInfo UpUrlInfo + var partInfo []base.Json + currentPartSize := pre.Data.PartSize + const maxRetries = 5 + const maxPartSize = 1024 * utils.MB // 1GB 上限 + + for attempt := 0; attempt < maxRetries; attempt++ { + // 计算分片信息 + partInfo = d._getPartInfo(stream, currentPartSize) + + // 尝试获取上传 URL + upUrlInfo, err = d.upUrl(ctx, pre, partInfo) + if err == nil { + // 成功获取上传 URL + log.Infof("[quark_open] Successfully obtained upload URLs with part size: %d MB (%d parts)", + currentPartSize/(1024*1024), len(partInfo)) + break + } + + // 检查是否为分片超限错误 + if strings.Contains(err.Error(), "exceed") { + if attempt < maxRetries-1 { + // 还有重试机会，翻倍分片大小 + newPartSize := currentPartSize * 2 + + // 检查是否超过上限 + if newPartSize > maxPartSize { + return fmt.Errorf("part list exceeded and cannot increase part size (current: %d MB, max: %d MB). File may be too large for Quark API", + currentPartSize/(1024*1024), maxPartSize/(1024*1024)) + } + + log.Warnf("[quark_open] Part list exceeded (attempt %d/%d, %d parts). Retrying with doubled part size: %d MB -> %d MB", + attempt+1, maxRetries, len(partInfo), + currentPartSize/(1024*1024), newPartSize/(1024*1024)) + + currentPartSize = newPartSize + continue // 重试 + } else { + // 已达到最大重试次数 + return fmt.Errorf("part list exceeded after %d retries. Last attempt: part size %d MB, %d parts", + maxRetries, currentPartSize/(1024*1024), len(partInfo)) + } + } + + // 其他错误，直接返回 return err } - // part up - ss, err := streamPkg.NewStreamSectionReader(stream, int(pre.Data.PartSize), &up) + // part up - 使用调整后的 currentPartSize + ss, err := streamPkg.NewStreamSectionReader(stream, int(currentPartSize), &up) if err != nil { return err } @@ -204,30 +365,49 @@ func (d *QuarkOpen) Put(ctx context.Context, dstDir model.Obj, stream model.File return ctx.Err() } - offset := int64(i) * pre.Data.PartSize - size := min(pre.Data.PartSize, total-offset) + offset := int64(i) * currentPartSize + size := min(currentPartSize, total-offset) rd, err := ss.GetSectionReader(offset, size) if err != nil { return err } + + // 上传重试逻辑，包含URL刷新 + var etag string err = retry.Do(func() error { rd.Seek(0, io.SeekStart) - etag, err := d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) - if err != nil { - return err + var uploadErr error + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) + + // 检查是否为URL过期错误 + if uploadErr != nil && strings.Contains(uploadErr.Error(), "expire") { + log.Warnf("[quark_open] Upload URL expired for part %d, refreshing...", i) + // 刷新上传URL + newUpUrlInfo, refreshErr := d.upUrl(ctx, pre, partInfo) + if refreshErr != nil { + return fmt.Errorf("failed to refresh upload url: %w", refreshErr) + } + upUrlInfo = newUpUrlInfo + log.Infof("[quark_open] Upload URL refreshed successfully") + + // 使用新URL重试上传 + rd.Seek(0, io.SeekStart) + etag, uploadErr = d.upPart(ctx, upUrlInfo, i, driver.NewLimitedUploadStream(ctx, rd)) } - etags = append(etags, etag) - return nil + + return uploadErr }, retry.Context(ctx), retry.Attempts(3), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)) + ss.FreeSectionReader(rd) if err != nil { return fmt.Errorf("failed to upload part %d: %w", i, err) } + etags = append(etags, etag) up(95 * float64(offset+size) / float64(total)) } diff --git a/drivers/quark_open/meta.go b/drivers/quark_open/meta.go index 3527b52e9..ee1903939 100644 --- a/drivers/quark_open/meta.go +++ b/drivers/quark_open/meta.go @@ -13,8 +13,8 @@ type Addition struct { APIAddress string `json:"api_url_address" default:"https://api.oplist.org/quarkyun/renewapi"` AccessToken string `json:"access_token" required:"false" default:""` RefreshToken string `json:"refresh_token" required:"true"` - AppID string `json:"app_id" required:"true" help:"Keep it empty if you don't have one"` - SignKey string `json:"sign_key" required:"true" help:"Keep it empty if you don't have one"` + AppID string `json:"app_id" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` + SignKey string `json:"sign_key" required:"false" default:"" help:"Optional - Auto-filled from online API, or use your own"` } type Conf struct { diff --git a/drivers/quark_open/util.go b/drivers/quark_open/util.go index 788ca0e99..1a3058375 100644 --- a/drivers/quark_open/util.go +++ b/drivers/quark_open/util.go @@ -20,6 +20,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -283,8 +284,15 @@ func (d *QuarkOpen) getProofRange(proofSeed string, fileSize int64) (*ProofRange func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []base.Json { // 计算分片信息 - partInfo := make([]base.Json, 0) total := stream.GetSize() + + // 确保partSize合理：最小4MB，避免分片过多 + const minPartSize int64 = 4 * utils.MB + if partSize < minPartSize { + partSize = minPartSize + } + + partInfo := make([]base.Json, 0) left := total partNumber := 1 @@ -304,6 +312,7 @@ func (d *QuarkOpen) _getPartInfo(stream model.FileStreamer, partSize int64) []ba partNumber++ } + log.Infof("[quark_open] Upload plan: file_size=%d, part_size=%d, part_count=%d", total, partSize, len(partInfo)) return partInfo } @@ -315,11 +324,17 @@ func (d *QuarkOpen) upUrl(ctx context.Context, pre UpPreResp, partInfo []base.Js } var resp UpUrlResp + log.Infof("[quark_open] Requesting upload URLs for %d parts (task_id: %s)", len(partInfo), pre.Data.TaskID) + _, err = d.request(ctx, "/open/v1/file/get_upload_urls", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) if err != nil { + // 如果是分片超限错误，记录详细信息 + if strings.Contains(err.Error(), "part list exceed") { + log.Errorf("[quark_open] Part list exceeded limit! Requested %d parts. Please check Quark API documentation for actual limit.", len(partInfo)) + } return upUrlInfo, err } @@ -340,13 +355,43 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber req.Header.Set("Accept-Encoding", "gzip") req.Header.Set("User-Agent", "Go-http-client/1.1") + // ✅ 关键修复：使用更长的超时时间（10分钟） + // 慢速网络下大文件分片上传可能需要很长时间 + client := &http.Client{ + Timeout: 10 * time.Minute, + Transport: base.HttpClient.Transport, + } + // 发送请求 - resp, err := base.HttpClient.Do(req) + resp, err := client.Do(req) if err != nil { return "", err } defer resp.Body.Close() + // 检查是否为URL过期错误（403, 410等状态码） + if resp.StatusCode == 403 || resp.StatusCode == 410 { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("upload url expired (status: %d): %s", resp.StatusCode, string(body)) + } + + // ✅ 关键修复：409 PartAlreadyExist 不是错误！ + // 夸克使用Sequential模式，超时重试时如果分片已存在，说明第一次其实成功了 + if resp.StatusCode == 409 { + body, _ := io.ReadAll(resp.Body) + // 从响应体中提取已存在分片的ETag + if strings.Contains(string(body), "PartAlreadyExist") { + // 尝试从XML响应中提取ETag + if etag := extractEtagFromXML(string(body)); etag != "" { + log.Infof("[quark_open] Part %d already exists (409), using existing ETag: %s", partNumber+1, etag) + return etag, nil + } + // 如果无法提取ETag，返回错误 + log.Warnf("[quark_open] Part %d already exists but cannot extract ETag from response: %s", partNumber+1, string(body)) + return "", fmt.Errorf("part already exists but ETag not found in response") + } + } + if resp.StatusCode != 200 { body, _ := io.ReadAll(resp.Body) return "", fmt.Errorf("up status: %d, error: %s", resp.StatusCode, string(body)) @@ -355,6 +400,23 @@ func (d *QuarkOpen) upPart(ctx context.Context, upUrlInfo UpUrlInfo, partNumber return resp.Header.Get("Etag"), nil } +// extractEtagFromXML 从OSS的XML错误响应中提取ETag +// 示例: "2F796AC486BB2891E3237D8BFDE020B5" +func extractEtagFromXML(xmlBody string) string { + start := strings.Index(xmlBody, "") + if start == -1 { + return "" + } + start += len("") + end := strings.Index(xmlBody[start:], "") + if end == -1 { + return "" + } + etag := xmlBody[start : start+end] + // 移除引号 + return strings.Trim(etag, "\"") +} + func (d *QuarkOpen) upFinish(ctx context.Context, pre UpPreResp, partInfo []base.Json, etags []string) error { // 创建 part_info_list partInfoList := make([]base.Json, len(partInfo)) @@ -417,25 +479,36 @@ func (d *QuarkOpen) generateReqSign(method string, pathname string, signKey stri } func (d *QuarkOpen) refreshToken() error { - refresh, access, err := d._refreshToken() + refresh, access, appID, signKey, err := d._refreshToken() for i := 0; i < 3; i++ { if err == nil { break } else { log.Errorf("[quark_open] failed to refresh token: %s", err) } - refresh, access, err = d._refreshToken() + refresh, access, appID, signKey, err = d._refreshToken() } if err != nil { return err } log.Infof("[quark_open] token exchange: %s -> %s", d.RefreshToken, refresh) d.RefreshToken, d.AccessToken = refresh, access + + // 如果在线API返回了AppID和SignKey，保存它们（不为空时才更新） + if appID != "" && appID != d.AppID { + d.AppID = appID + log.Infof("[quark_open] AppID updated from online API: %s", appID) + } + if signKey != "" && signKey != d.SignKey { + d.SignKey = signKey + log.Infof("[quark_open] SignKey updated from online API") + } + op.MustSaveDriverStorage(d) return nil } -func (d *QuarkOpen) _refreshToken() (string, string, error) { +func (d *QuarkOpen) _refreshToken() (string, string, string, string, error) { if d.UseOnlineAPI && d.APIAddress != "" { u := d.APIAddress var resp RefreshTokenOnlineAPIResp @@ -448,19 +521,20 @@ func (d *QuarkOpen) _refreshToken() (string, string, error) { }). Get(u) if err != nil { - return "", "", err + return "", "", "", "", err } if resp.RefreshToken == "" || resp.AccessToken == "" { if resp.ErrorMessage != "" { - return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) + return "", "", "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) } - return "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") + return "", "", "", "", fmt.Errorf("empty token returned from official API, a wrong refresh token may have been used") } - return resp.RefreshToken, resp.AccessToken, nil + // 返回所有字段，包括AppID和SignKey + return resp.RefreshToken, resp.AccessToken, resp.AppID, resp.SignKey, nil } // TODO 本地刷新逻辑 - return "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") + return "", "", "", "", fmt.Errorf("local refresh token logic is not implemented yet, please use online API or contact the developer") } // 生成认证 Cookie diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index 48aabc48f..87798c6ef 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "fmt" + "html" "io" "net/http" "strconv" @@ -70,10 +71,10 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { page := 1 size := 100 query := map[string]string{ - "pdir_fid": parent, - "_size": strconv.Itoa(size), - "_fetch_total": "1", - "fetch_all_file": "1", + "pdir_fid": parent, + "_size": strconv.Itoa(size), + "_fetch_total": "1", + "fetch_all_file": "1", "fetch_risk_file_name": "1", } if d.OrderBy != "none" { @@ -89,6 +90,7 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { return nil, err } for _, file := range resp.Data.List { + file.FileName = html.UnescapeString(file.FileName) if d.OnlyListVideoFile { // 开启后 只列出视频文件和文件夹 if file.IsDir() || file.Category == 1 { diff --git a/drivers/quark_uc_tv/util.go b/drivers/quark_uc_tv/util.go index d68a2f3c3..c0da6eb6b 100644 --- a/drivers/quark_uc_tv/util.go +++ b/drivers/quark_uc_tv/util.go @@ -8,6 +8,7 @@ import ( "errors" "net/http" "strconv" + "strings" "time" "github.com/OpenListTeam/OpenList/v4/internal/model" @@ -70,8 +71,16 @@ func (d *QuarkUCTV) request(ctx context.Context, pathname string, method string, return nil, err } // 判断 是否需要 刷新 access_token - if e.Status == -1 && e.Errno == 10001 { - // token 过期 + errInfoLower := strings.ToLower(strings.TrimSpace(e.ErrorInfo)) + maybeTokenInvalid := + (e.Status == -1 && (e.Errno == 10001 || e.Errno == 11001)) || + (errInfoLower != "" && + (strings.Contains(errInfoLower, "access token") || + strings.Contains(errInfoLower, "access_token") || + strings.Contains(errInfoLower, "token无效") || + strings.Contains(errInfoLower, "token 无效"))) + if maybeTokenInvalid { + // token 过期 / 无效 err = d.getRefreshTokenByTV(ctx, d.Addition.RefreshToken, true) if err != nil { return nil, err diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go index 00a32f001..a57076e08 100644 --- a/drivers/sftp/types.go +++ b/drivers/sftp/types.go @@ -48,8 +48,8 @@ func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { Size: _f.Size(), Modified: _f.ModTime(), IsFolder: _f.IsDir(), - Path: target, + Path: path, // Use symlink's own path, not target path } - log.Debugf("[sftp] obj: %+v, is symlink: %v", obj, symlink) + log.Debugf("[sftp] obj: %+v, is symlink: %v, target: %s", obj, symlink, target) return obj, nil } diff --git a/drivers/teldrive/driver.go b/drivers/teldrive/driver.go index 11ba0971e..d420eb4d0 100644 --- a/drivers/teldrive/driver.go +++ b/drivers/teldrive/driver.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "path" + "strconv" "strings" "github.com/OpenListTeam/OpenList/v4/drivers/base" @@ -17,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" + "golang.org/x/sync/errgroup" ) type Teldrive struct { @@ -53,18 +55,58 @@ func (d *Teldrive) Drop(ctx context.Context) error { } func (d *Teldrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - var listResp ListResp + var firstResp ListResp err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "path": dir.GetPath(), - "limit": "1000", // overide default 500, TODO pagination + "limit": "500", + "page": "1", }) - }, &listResp) + }, &firstResp) + if err != nil { return nil, err } - return utils.SliceConvert(listResp.Items, func(src Object) (model.Obj, error) { + pagesData := make([][]Object, firstResp.Meta.TotalPages) + pagesData[0] = firstResp.Items + + if firstResp.Meta.TotalPages > 1 { + g, _ := errgroup.WithContext(ctx) + g.SetLimit(8) + + for i := 2; i <= firstResp.Meta.TotalPages; i++ { + page := i + g.Go(func() error { + var resp ListResp + err := d.request(http.MethodGet, "/api/files", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "path": dir.GetPath(), + "limit": "500", + "page": strconv.Itoa(page), + }) + }, &resp) + + if err != nil { + return err + } + + pagesData[page-1] = resp.Items + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + } + + var allItems []Object + for _, items := range pagesData { + allItems = append(allItems, items...) + } + + return utils.SliceConvert(allItems, func(src Object) (model.Obj, error) { return &model.Object{ Path: path.Join(dir.GetPath(), src.Name), ID: src.ID, @@ -184,7 +226,7 @@ func (d *Teldrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStr } if totalParts <= 1 { - return d.doSingleUpload(ctx, dstDir, file, up, totalParts, chunkSize, fileId) + return d.doSingleUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) } return d.doMultiUpload(ctx, dstDir, file, up, maxRetried, totalParts, chunkSize, fileId) diff --git a/drivers/teldrive/meta.go b/drivers/teldrive/meta.go index 23bae5f94..cc7a5dbf7 100644 --- a/drivers/teldrive/meta.go +++ b/drivers/teldrive/meta.go @@ -11,6 +11,7 @@ type Addition struct { Cookie string `json:"cookie" type:"string" required:"true" help:"access_token=xxx"` UseShareLink bool `json:"use_share_link" type:"bool" default:"false" help:"Create share link when getting link to support 302. If disabled, you need to enable web proxy."` ChunkSize int64 `json:"chunk_size" type:"number" default:"10" help:"Chunk size in MiB"` + RandomChunkName bool `json:"random_chunk_name" type:"bool" default:"true" help:"Random chunk name"` UploadConcurrency int64 `json:"upload_concurrency" type:"number" default:"4" help:"Concurrency upload requests"` } diff --git a/drivers/teldrive/upload.go b/drivers/teldrive/upload.go index 87cffa1ae..b94f5fc93 100644 --- a/drivers/teldrive/upload.go +++ b/drivers/teldrive/upload.go @@ -1,6 +1,8 @@ package teldrive import ( + "crypto/md5" + "encoding/hex" "fmt" "io" "net/http" @@ -16,6 +18,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" + "github.com/google/uuid" "github.com/pkg/errors" "golang.org/x/net/context" "golang.org/x/sync/errgroup" @@ -38,6 +41,11 @@ func (d *Teldrive) touch(name, path string) error { return nil } +func getMD5Hash(text string) string { + hash := md5.Sum([]byte(text)) + return hex.EncodeToString(hash[:]) +} + func (d *Teldrive) createFileOnUploadSuccess(name, id, path string, uploadedFileParts []FilePart, totalSize int64) error { remoteFileParts, err := d.getFilePart(id) if err != nil { @@ -101,12 +109,10 @@ func (d *Teldrive) getFilePart(fileId string) ([]FilePart, error) { return uploadedParts, nil } -func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback, resp interface{}) error { +func (d *Teldrive) singleUploadRequest(ctx context.Context, fileId string, callback base.ReqCallback, resp any) error { url := d.Address + "/api/uploads/" + fileId client := resty.New().SetTimeout(0) - ctx := context.Background() - req := client.R(). SetContext(ctx) req.SetHeader("Cookie", d.Cookie) @@ -135,16 +141,18 @@ func (d *Teldrive) singleUploadRequest(fileId string, callback base.ReqCallback, } func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up model.UpdateProgress, - totalParts int, chunkSize int64, fileId string) error { + maxRetried, totalParts int, chunkSize int64, fileId string) error { totalSize := file.GetSize() var fileParts []FilePart var uploaded int64 = 0 - ss, err := stream.NewStreamSectionReader(file, int(totalSize), &up) + var partName string + chunkSize = min(totalSize, chunkSize) + ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up) if err != nil { return err } - + chunkCnt := 0 for uploaded < totalSize { if utils.IsCanceled(ctx) { return ctx.Err() @@ -154,6 +162,7 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo if err != nil { return err } + chunkCnt += 1 filePart := &FilePart{} if err := retry.Do(func() error { @@ -161,13 +170,19 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo return err } - if err := d.singleUploadRequest(fileId, func(req *resty.Request) { + if d.RandomChunkName { + partName = getMD5Hash(uuid.New().String()) + } else { + partName = file.GetName() + if totalParts > 1 { + partName = fmt.Sprintf("%s.part.%03d", file.GetName(), chunkCnt) + } + } + + if err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ - "partName": func() string { - digits := len(strconv.Itoa(totalParts)) - return file.GetName() + fmt.Sprintf(".%0*d", digits, 1) - }(), - "partNo": strconv.Itoa(1), + "partName": partName, + "partNo": strconv.Itoa(chunkCnt), "fileName": file.GetName(), } req.SetQueryParams(uploadParams) @@ -180,7 +195,7 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo return nil }, retry.Context(ctx), - retry.Attempts(3), + retry.Attempts(uint(maxRetried)), retry.DelayType(retry.BackOffDelay), retry.Delay(time.Second)); err != nil { return err @@ -189,8 +204,11 @@ func (d *Teldrive) doSingleUpload(ctx context.Context, dstDir model.Obj, file mo if filePart.Name != "" { fileParts = append(fileParts, *filePart) uploaded += curChunkSize - up(float64(uploaded) / float64(totalSize)) + up(float64(uploaded) / float64(totalSize) * 100) ss.FreeSectionReader(rd) + } else { + // For common situation this code won't reach + return fmt.Errorf("[Teldrive] upload chunk %d failed: filePart Somehow missing", chunkCnt) } } @@ -318,6 +336,7 @@ func (d *Teldrive) doMultiUpload(ctx context.Context, dstDir model.Obj, file mod func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task chunkTask, totalParts, maxRetried int) (*FilePart, error) { filePart := &FilePart{} retryCount := 0 + var partName string defer task.ss.FreeSectionReader(task.reader) for { @@ -331,12 +350,22 @@ func (d *Teldrive) uploadSingleChunk(ctx context.Context, fileId string, task ch return &existingPart, nil } - err := d.singleUploadRequest(fileId, func(req *resty.Request) { + if _, err := task.reader.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + if d.RandomChunkName { + partName = getMD5Hash(uuid.New().String()) + } else { + partName = task.fileName + if totalParts > 1 { + partName = fmt.Sprintf("%s.part.%03d", task.fileName, task.chunkIdx) + } + } + + err := d.singleUploadRequest(ctx, fileId, func(req *resty.Request) { uploadParams := map[string]string{ - "partName": func() string { - digits := len(strconv.Itoa(totalParts)) - return task.fileName + fmt.Sprintf(".%0*d", digits, task.chunkIdx) - }(), + "partName": partName, "partNo": strconv.Itoa(task.chunkIdx), "fileName": task.fileName, } diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index cb352afed..492b9814f 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -433,6 +433,32 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi return nil } +func (xc *XunLeiCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string diff --git a/drivers/thunder/types.go b/drivers/thunder/types.go index 7b3ad5692..fcfa1fb1c 100644 --- a/drivers/thunder/types.go +++ b/drivers/thunder/types.go @@ -347,3 +347,21 @@ type ReviewData struct { Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } + +type AboutResponse struct { + // Kind string `json:"kind"` + Quota struct { + // Kind string `json:"kind"` + Limit string `json:"limit"` + Usage string `json:"usage"` + // UsageInTrash string `json:"usage_in_trash"` + // PlayTimesLimit string `json:"play_times_limit"` + // PlayTimesUsage string `json:"play_times_usage"` + // IsUnlimited bool `json:"is_unlimited"` + // UpgradeType string `json:"upgrade_type"` + } `json:"quota"` + // ExpiresAt string `json:"expires_at"` + // Quotas struct { + // } `json:"quotas"` + // IsSearchFlushed bool `json:"is_search_flushed"` +} diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index 8e0c6e1bc..e60522e52 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -543,6 +543,32 @@ func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream return nil } +func (xc *XunLeiBrowserCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, dir model.Obj, path string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string diff --git a/drivers/thunder_browser/types.go b/drivers/thunder_browser/types.go index 6b2a41023..a5fda2a5d 100644 --- a/drivers/thunder_browser/types.go +++ b/drivers/thunder_browser/types.go @@ -376,3 +376,10 @@ type ReviewData struct { Deviceid string `json:"deviceid"` Devicesign string `json:"devicesign"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + } `json:"quota"` +} diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index 86ff22bdc..acbb82513 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -423,6 +423,32 @@ func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.F return nil } +func (xc *XunLeiXCommon) GetDetails(ctx context.Context) (*model.StorageDetails, error) { + var about AboutResponse + _, err := xc.Request(API_URL+"/about", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + }, &about) + if err != nil { + return nil, err + } + + total, err := strconv.ParseInt(about.Quota.Limit, 10, 64) + if err != nil { + return nil, err + } + used, err := strconv.ParseInt(about.Quota.Usage, 10, 64) + if err != nil { + return nil, err + } + + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: total, + UsedSpace: used, + }, + }, nil +} + func (xc *XunLeiXCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string diff --git a/drivers/thunderx/types.go b/drivers/thunderx/types.go index e5fbaa241..728fdea31 100644 --- a/drivers/thunderx/types.go +++ b/drivers/thunderx/types.go @@ -303,3 +303,10 @@ type Media struct { IsVisible bool `json:"is_visible"` Category string `json:"category"` } + +type AboutResponse struct { + Quota struct { + Limit string `json:"limit"` + Usage string `json:"usage"` + } `json:"quota"` +} diff --git a/drivers/wps/driver.go b/drivers/wps/driver.go index 8a3ccb6ce..847425b11 100644 --- a/drivers/wps/driver.go +++ b/drivers/wps/driver.go @@ -3,16 +3,24 @@ package wps import ( "context" "fmt" + "net/http" + "strings" + "time" + "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/go-resty/resty/v2" ) type Wps struct { model.Storage Addition - companyID string + + login *loginState + client *resty.Client } func (d *Wps) Config() driver.Config { @@ -27,13 +35,84 @@ func (d *Wps) Init(ctx context.Context) error { if d.Cookie == "" { return fmt.Errorf("cookie is empty") } - return d.ensureCompanyID(ctx) + + d.client = base.NewRestyClient() + + resp, err := d.request(ctx).SetResult(&d.login).Get("https://account.kdocs.cn/api/v3/islogin") + if err != nil { + return err + } + if !resp.IsSuccess() { + return fmt.Errorf("failed to check login status, status code: %d, body: %s", resp.StatusCode(), resp.String()) + } + + return nil } func (d *Wps) Drop(ctx context.Context) error { + + if d.client != nil { + d.client = nil + } + if d.login != nil { + d.login = nil + } return nil } +func (d *Wps) GetRoot(ctx context.Context) (model.Obj, error) { + root := &Obj{ + Obj: &model.Object{ + Path: "/", + Name: "root", + Modified: d.Modified, + Ctime: d.Modified, + IsFolder: true, + }, + Kind: "root", + } + rootPath := d.RootFolderPath + if rootPath != "" && rootPath != "/" { + parts := strings.Split(strings.Trim(rootPath, "/"), "/") + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + var current *Obj + for _, g := range groups { + if g.Name == parts[0] { + current = g.groupToObj("/") + break + } + } + if current == nil { + return nil, fmt.Errorf("root path %q not found", rootPath) + } + parentID := int64(0) + for _, name := range parts[1:] { + files, err := d.getFiles(ctx, current.GroupID, parentID) + if err != nil { + return nil, err + } + var next *Obj + for _, f := range files { + if f.Type == "folder" && f.Name == name { + next = f.fileToObj(current.GetPath(), d.isPersonal()) + break + } + } + if next == nil { + return nil, fmt.Errorf("root path %q not found", rootPath) + } + current = next + parentID = current.FileID + } + current.Obj = &model.Object{ID: current.GetID(), Path: "/", Name: current.GetName(), IsFolder: true} + root = current + } + return root, nil +} + func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]model.Obj, error) { basePath := "/" if dir != nil { @@ -41,34 +120,261 @@ func (d *Wps) List(ctx context.Context, dir model.Obj, _ model.ListArgs) ([]mode basePath = p } } - return d.list(ctx, basePath) + node, err := unwrapWpsObj(dir) + if err != nil { + return nil, err + } + if node.Kind == "root" { + groups, err := d.getGroups(ctx) + if err != nil { + return nil, err + } + return utils.SliceConvert(groups, func(g Group) (model.Obj, error) { + return g.groupToObj(basePath), nil + }) + } + if node.Kind != "group" && node.Kind != "folder" { + return nil, nil + } + parentID := int64(0) + if node.HasFile && node.Kind == "folder" { + parentID = node.FileID + } + files, err := d.getFiles(ctx, node.GroupID, parentID) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(f FileInfo) (model.Obj, error) { + return f.fileToObj(basePath, d.isPersonal()), nil + }) } func (d *Wps) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*model.Link, error) { if file == nil { return nil, errs.NotSupport } - return d.link(ctx, file.GetPath()) + node, err := unwrapWpsObj(file) + if err != nil { + return nil, err + } + if node.Kind != "file" || !node.HasFile { + return nil, errs.NotSupport + } + if !node.CanDownload { + return nil, fmt.Errorf("can not download") + } + url := fmt.Sprintf("%s/api/v5/groups/%d/files/%d/download?support_checksums=sha1", d.driveHost()+d.drivePrefix(), node.GroupID, node.FileID) + var resp downloadResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + if resp.URL == "" { + return nil, fmt.Errorf("empty download url") + } + return &model.Link{ + URL: resp.URL, + Header: http.Header{ + "User-Agent": []string{d.getUA()}, + "Referer": []string{d.driveHost()}, + }, + }, nil } func (d *Wps) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - return d.makeDir(ctx, parentDir, dirName) + if parentDir == nil { + return errs.NotSupport + } + node, err := unwrapWpsObj(parentDir) + if err != nil { + return err + } + if node.Kind != "group" && node.Kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.HasFile && node.Kind == "folder" { + parentID = node.FileID + } + body := map[string]interface{}{ + "groupid": node.GroupID, + "name": dirName, + "parentid": parentID, + } + if err := d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/folder"), body); err != nil { + return err + } + return nil } func (d *Wps) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.move(ctx, srcObj, dstDir) + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := unwrapWpsObj(srcObj) + if err != nil { + return fmt.Errorf("invalid source object type: %w", err) + } + nodeDst, err := unwrapWpsObj(dstDir) + if err != nil { + return fmt.Errorf("invalid destination object type: %w", err) + } + if nodeSrc.Kind != "file" && nodeSrc.Kind != "folder" { + return errs.NotSupport + } + if nodeDst.Kind != "group" && nodeDst.Kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.HasFile && nodeDst.Kind == "folder" { + targetParentID = nodeDst.FileID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.FileID}, + "target_groupid": nodeDst.GroupID, + "target_parentid": targetParentID, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/move", nodeSrc.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + return nil } func (d *Wps) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - return d.rename(ctx, srcObj, newName) + if srcObj == nil { + return errs.NotSupport + } + node, err := unwrapWpsObj(srcObj) + if err != nil { + return err + } + if node.Kind != "file" && node.Kind != "folder" { + return errs.NotSupport + } + url := fmt.Sprintf("/api/v3/groups/%d/files/%d", node.GroupID, node.FileID) + body := map[string]string{"fname": newName} + if err := d.doJSON(ctx, http.MethodPut, d.driveURL(url), body); err != nil { + return err + } + return nil } func (d *Wps) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.copy(ctx, srcObj, dstDir) + if srcObj == nil || dstDir == nil { + return errs.NotSupport + } + nodeSrc, err := unwrapWpsObj(srcObj) + if err != nil { + return fmt.Errorf("invalid source object type: %w", err) + } + nodeDst, err := unwrapWpsObj(dstDir) + if err != nil { + return fmt.Errorf("invalid destination object type: %w", err) + } + if nodeSrc.Kind != "file" && nodeSrc.Kind != "folder" { + return errs.NotSupport + } + if nodeDst.Kind != "group" && nodeDst.Kind != "folder" { + return errs.NotSupport + } + targetParentID := int64(0) + if nodeDst.HasFile && nodeDst.Kind == "folder" { + targetParentID = nodeDst.FileID + } + body := map[string]interface{}{ + "fileids": []int64{nodeSrc.FileID}, + "groupid": nodeSrc.GroupID, + "target_groupid": nodeDst.GroupID, + "target_parentid": targetParentID, + "duplicated_name_model": 1, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/copy", nodeSrc.GroupID) + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + return nil } func (d *Wps) Remove(ctx context.Context, obj model.Obj) error { - return d.remove(ctx, obj) + if obj == nil { + return errs.NotSupport + } + node, err := unwrapWpsObj(obj) + if err != nil { + return err + } + if node.Kind != "file" && node.Kind != "folder" { + return errs.NotSupport + } + + body := map[string]interface{}{ + "fileids": []int64{node.FileID}, + } + url := fmt.Sprintf("/api/v3/groups/%d/files/batch/delete", node.GroupID) + + for { + var res apiResult + resp, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&res). + SetError(&res). + Post(d.driveURL(url)) + if err != nil { + return err + } + + // 无法连续创建文件夹删除。如果一定要删除，每0.5s 尝试一次创建下一个删除请求，应当避免递归删除文件夹 + if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { + time.Sleep(500 * time.Millisecond) + continue + } + + if err := checkAPI(resp, res); err != nil { + return err + } + break + } + return nil } func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { @@ -76,16 +382,48 @@ func (d *Wps) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer } func (d *Wps) GetDetails(ctx context.Context) (*model.StorageDetails, error) { - quota, err := d.spaces(ctx) + if d.isPersonal() { + url := ENDPOINT_PERSONAL + "/api/v3/spaces" + var resp spacesResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: resp.Total, + UsedSpace: resp.Used, + }, + }, nil + } + url := ENDPOINT_BUSINESS + "/3rd/plussvr/compose/v1/u/companies/batch/service-space?comp_ids=" + fmt.Sprint(d.login.CompanyID) + var resp serviceSpaceResp + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) if err != nil { return nil, err } - return &model.StorageDetails{ - DiskUsage: model.DiskUsage{ - TotalSpace: quota.Total, - UsedSpace: quota.Used, - }, - }, nil + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + if len(resp.Info) == 0 { + return nil, fmt.Errorf("empty service space info") + } + // info := resp.Info[0] + for _, info := range resp.Info { + if info.ID == d.login.CompanyID { + return &model.StorageDetails{ + DiskUsage: model.DiskUsage{ + TotalSpace: info.SpaceTotal, + UsedSpace: info.SpaceUsed, + }, + }, nil + } + } + return nil, fmt.Errorf("service space info not found for company ID: %d", d.login.CompanyID) } var _ driver.Driver = (*Wps)(nil) +var _ driver.GetRooter = (*Wps)(nil) diff --git a/drivers/wps/meta.go b/drivers/wps/meta.go index 7a3362f3a..a520bb433 100644 --- a/drivers/wps/meta.go +++ b/drivers/wps/meta.go @@ -7,16 +7,16 @@ import ( type Addition struct { driver.RootPath - Cookie string `json:"cookie" required:"true" type:"text"` - Mode string `json:"mode" type:"select" options:"Personal,Business" default:"Business"` + Cookie string `json:"cookie" required:"true"` + Mode string `json:"mode" type:"select" options:"Personal,Business" default:"Personal"` + CustomUA string `json:"custom_ua"` } var config = driver.Config{ - Name: "WPS", - LocalSort: true, - DefaultRoot: "/", - Alert: "", - NoOverwriteUpload: true, + Name: "WPS", + LocalSort: true, + DefaultRoot: "/", + Alert: "", } func init() { diff --git a/drivers/wps/put.go b/drivers/wps/put.go new file mode 100644 index 000000000..94ba72e72 --- /dev/null +++ b/drivers/wps/put.go @@ -0,0 +1,315 @@ +package wps + +import ( + "bytes" + "context" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/driver" + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type countingWriter struct { + n *int64 +} + +func (w countingWriter) Write(p []byte) (int, error) { + *w.n += int64(len(p)) + return len(p), nil +} + +func cacheAndHash(file model.FileStreamer, up driver.UpdateProgress) (model.File, int64, string, string, error) { + h1 := sha1.New() + h256 := sha256.New() + size := file.GetSize() + var counted int64 + ws := []io.Writer{h1, h256} + if size <= 0 { + ws = append(ws, countingWriter{n: &counted}) + } + p := up + f, err := file.CacheFullAndWriter(&p, io.MultiWriter(ws...)) + if err != nil { + return nil, 0, "", "", err + } + if size <= 0 { + size = counted + } + return f, size, hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h256.Sum(nil)), nil +} + +func (d *Wps) createUpload(ctx context.Context, groupID, parentID int64, name string, size int64, sha1Hex, sha256Hex string) (*uploadCreateUpdateResp, error) { + body := map[string]string{ + "group_id": strconv.FormatInt(groupID, 10), + "name": name, + "parent_id": strconv.FormatInt(parentID, 10), + "sha1": sha1Hex, + "sha256": sha256Hex, + "size": strconv.FormatInt(size, 10), + } + var resp uploadCreateUpdateResp + r, err := d.jsonRequest(ctx). + SetBody(body). + SetResult(&resp). + SetError(&resp). + Put(d.driveURL("/api/v5/files/upload/create_update")) + if err != nil { + return nil, err + } + if err := checkAPI(r, resp.apiResult); err != nil { + return nil, err + } + if resp.URL == "" { + return nil, fmt.Errorf("empty upload url") + } + return &resp, nil +} + +func normalizeETag(v string) string { + v = strings.TrimSpace(v) + if strings.HasPrefix(v, "W/") { + v = strings.TrimSpace(strings.TrimPrefix(v, "W/")) + } + return strings.Trim(v, `"`) +} + +func (d *Wps) commitUpload(ctx context.Context, etag, key string, groupID, parentID int64, name, sha1Hex string, size int64, store string) error { + store = strings.TrimSpace(store) + if store == "" { + store = "ks3" + } + storeKey := "" + if key != "" { + storeKey = key + } + body := map[string]interface{}{ + "etag": etag, + "groupid": groupID, + "key": key, + "name": name, + "parentid": parentID, + "sha1": sha1Hex, + "size": size, + "store": store, + "storekey": storeKey, + } + return d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/file"), body) +} + +func (d *Wps) put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if dstDir == nil || file == nil { + return errs.NotSupport + } + if up == nil { + up = func(float64) {} + } + node, err := unwrapWpsObj(dstDir) + if err != nil { + return err + } + if node.Kind != "group" && node.Kind != "folder" { + return errs.NotSupport + } + parentID := int64(0) + if node.HasFile && node.Kind == "folder" { + parentID = node.FileID + } + f, size, sha1Hex, sha256Hex, err := cacheAndHash(file, func(float64) {}) + if err != nil { + return err + } + if c, ok := f.(io.Closer); ok { + defer c.Close() + } + + // 在隐藏文件名前加_上传，这是WPS的限制，无法上传隐藏文件，也无法将任何文件重命名为隐藏文件，所有隐藏文件会被自动加上_ 上传 + // 甚至可以上传前缀是..的文件，但是单个点就是不行 + realName := file.GetName() + uploadName := realName + if strings.HasPrefix(realName, ".") { + uploadName = "_" + realName + } + + info, err := d.createUpload(ctx, node.GroupID, parentID, uploadName, size, sha1Hex, sha256Hex) + if err != nil { + return err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err + } + rf := driver.NewLimitedUploadFile(ctx, f) + prog := driver.NewProgress(size, model.UpdateProgressWithRange(up, 0, 1)) + + method := strings.ToUpper(strings.TrimSpace(info.Method)) + if method == "" { + method = http.MethodPut + } + + var req *http.Request + if method == http.MethodPost && len(info.Request.FormData) > 0 { + if size == 0 { + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + return err + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + return err + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + return err + } + if err := mw.Close(); err != nil { + return err + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, bytes.NewReader(buf.Bytes())) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.ContentLength = int64(buf.Len()) + req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10)) + } else { + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + req, err = http.NewRequestWithContext(ctx, method, info.URL, pr) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + go func() { + for k, v := range info.Request.FormData { + if err := mw.WriteField(k, v); err != nil { + pw.CloseWithError(err) + return + } + } + part, err := mw.CreateFormFile("file", uploadName) + if err != nil { + pw.CloseWithError(err) + return + } + if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { + pw.CloseWithError(err) + return + } + if err := mw.Close(); err != nil { + pw.CloseWithError(err) + return + } + pw.Close() + }() + } + } else { + var body = io.TeeReader(rf, prog) + if size == 0 { + body = bytes.NewReader(nil) + } + req, err = http.NewRequestWithContext(ctx, method, info.URL, body) + if err != nil { + return err + } + for k, v := range info.Request.Headers { + req.Header.Set(k, v) + } + req.ContentLength = size + req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) + } + + c := *d.client.GetClient() + c.Timeout = 0 + resp, err := (&c).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if !statusOK(resp.StatusCode, info.Response.ExpectCode) { + io.Copy(io.Discard, resp.Body) + return fmt.Errorf("http error: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + etag := normalizeETag(respArg(info.Response.ArgsETag, resp, body)) + if etag == "" { + etag = normalizeETag(resp.Header.Get("ETag")) + } + + key := strings.TrimSpace(respArg(info.Response.ArgsKey, resp, body)) + if key == "" { + key = strings.TrimSpace(resp.Header.Get("x-obs-save-key")) + } + + var pr uploadPutResp + sha1FromServer := "" + if err := json.Unmarshal(body, &pr); err == nil { + sha1FromServer = strings.TrimSpace(pr.NewFilename) + if sha1FromServer == "" { + sha1FromServer = strings.TrimSpace(pr.Sha1) + } + if etag == "" && pr.MD5 != "" { + etag = strings.TrimSpace(pr.MD5) + } + } + + if sha1FromServer == "" { + if v := extractXMLTag(string(body), "ETag"); v != "" { + sha1FromServer = v + if etag == "" { + etag = v + } + } + } + if sha1FromServer == "" && key != "" && len(key) == 40 { + sha1FromServer = key + } + if sha1FromServer == "" { + sha1FromServer = sha1Hex + } + + if etag == "" { + return fmt.Errorf("empty etag") + } + if sha1FromServer == "" { + return fmt.Errorf("empty sha1") + } + + store := strings.TrimSpace(info.Store) + commitKey := "" + if strings.TrimSpace(info.Response.ArgsKey) != "" { + commitKey = key + if commitKey == "" { + commitKey = sha1FromServer + } + } + + if err := d.commitUpload(ctx, etag, commitKey, node.GroupID, parentID, uploadName, sha1FromServer, size, store); err != nil { + return err + } + + up(1) + return nil +} diff --git a/drivers/wps/types.go b/drivers/wps/types.go index 7ff6119dd..d5329808f 100644 --- a/drivers/wps/types.go +++ b/drivers/wps/types.go @@ -1,15 +1,24 @@ package wps import ( - "time" + "strconv" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/internal/model" ) -type workspaceResp struct { - Companies []struct { - ID int64 `json:"id"` - } `json:"companies"` +type apiResult struct { + Result string `json:"result"` + Msg string `json:"msg"` +} + +type loginState struct { + AccountNum int `json:"account_num"` + CompanyID int64 `json:"companyid"` + CurrentCompanyID int64 `json:"current_companyid"` + IsCompanyAccount bool `json:"is_company_account"` + IsPlus bool `json:"is_plus"` + LoginMode string `json:"loginmode"` + UserID int64 `json:"userid"` } type Group struct { @@ -23,6 +32,14 @@ type groupsResp struct { Groups []Group `json:"groups"` } +type personalGroupsResp struct { + apiResult + Groups []struct { + ID int64 `json:"id"` + Name string `json:"name"` + } `json:"groups"` +} + type filePerms struct { Download int `json:"download"` } @@ -40,8 +57,58 @@ type FileInfo struct { FilePerms filePerms `json:"file_perms_acl"` } +func (f *FileInfo) canDownload(isPersonal bool) bool { + if f == nil || f.Type == "folder" { + return false + } + if f.FilePerms.Download != 0 { + return true + } + return isPersonal +} + +func (f FileInfo) fileToObj(basePath string, isPersonal bool) *Obj { + name := f.Name + path := joinPath(basePath, name) + kind := "file" + if f.Type == "folder" { + kind = "folder" + } + obj := &Obj{ + Obj: &model.Object{ + ID: strconv.FormatInt(f.ID, 10), + Path: path, + Name: name, + Size: f.Size, + Modified: parseTime(f.Mtime), + Ctime: parseTime(f.Ctime), + IsFolder: f.Type == "folder", + }, + Kind: kind, + FileID: f.ID, + GroupID: f.GroupID, + HasFile: true, + CanDownload: f.canDownload(isPersonal), + } + return obj +} + +func (g Group) groupToObj(basePath string) *Obj { + return &Obj{ + Obj: &model.Object{ + ID: strconv.FormatInt(g.GroupID, 10), + Path: joinPath(basePath, g.Name), + Name: g.Name, + IsFolder: true, + }, + Kind: "group", + GroupID: g.GroupID, + } +} + type filesResp struct { - Files []FileInfo `json:"files"` + Files []FileInfo `json:"files"` + NextOffset int `json:"next_offset"` } type downloadResp struct { @@ -61,46 +128,41 @@ type spacesResp struct { } `json:"used_parts"` } -type Obj struct { - id string - name string - size int64 - ctime time.Time - mtime time.Time - isDir bool - hash utils.HashInfo - path string - canDownload bool -} - -func (o *Obj) GetSize() int64 { - return o.size +type serviceSpaceResp struct { + Info []struct { + ID int64 `json:"id"` + SpaceTotal int64 `json:"space_total"` + SpaceUsed int64 `json:"space_used"` + } `json:"info"` } -func (o *Obj) GetName() string { - return o.name +type uploadCreateUpdateResp struct { + apiResult + Method string `json:"method"` + URL string `json:"url"` + Store string `json:"store"` + Request struct { + Headers map[string]string `json:"headers"` + FormData map[string]string `json:"formData"` + } `json:"request"` + Response struct { + ExpectCode []int `json:"expect_code"` + ArgsETag string `json:"args_etag"` + ArgsKey string `json:"args_key"` + } `json:"response"` } -func (o *Obj) ModTime() time.Time { - return o.mtime +type uploadPutResp struct { + NewFilename string `json:"newfilename"` + Sha1 string `json:"sha1"` + MD5 string `json:"md5"` } -func (o *Obj) CreateTime() time.Time { - return o.ctime -} - -func (o *Obj) IsDir() bool { - return o.isDir -} - -func (o *Obj) GetHash() utils.HashInfo { - return o.hash -} - -func (o *Obj) GetID() string { - return o.id -} - -func (o *Obj) GetPath() string { - return o.path +type Obj struct { + model.Obj + Kind string // root / group / file / folder + FileID int64 + GroupID int64 + HasFile bool // only FileInfo has file, otherwise the FileID is 0 + CanDownload bool } diff --git a/drivers/wps/util.go b/drivers/wps/util.go index 79be1ac4b..cb6e84d66 100644 --- a/drivers/wps/util.go +++ b/drivers/wps/util.go @@ -1,102 +1,39 @@ package wps import ( - "bytes" "context" - "crypto/sha1" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" - "io" - "mime/multipart" "net/http" "strconv" "strings" - "sync" "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" - "github.com/OpenListTeam/OpenList/v4/internal/driver" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/go-resty/resty/v2" ) -const endpoint = "https://365.kdocs.cn" -const personalEndpoint = "https://drive.wps.cn" - -type resolvedNode struct { - kind string - group Group - file *FileInfo -} - -type resolveCacheEntry struct { - node *resolvedNode - expire time.Time -} - -type resolveCacheStore struct { - mu sync.RWMutex - m map[string]resolveCacheEntry -} - -var resolveCaches sync.Map - -type apiResult struct { - Result string `json:"result"` - Msg string `json:"msg"` -} - -type uploadCreateUpdateResp struct { - apiResult - Method string `json:"method"` - URL string `json:"url"` - Store string `json:"store"` - Request struct { - Headers map[string]string `json:"headers"` - FormData map[string]string `json:"formData"` - } `json:"request"` - Response struct { - ExpectCode []int `json:"expect_code"` - ArgsETag string `json:"args_etag"` - ArgsKey string `json:"args_key"` - } `json:"response"` -} - -type uploadPutResp struct { - NewFilename string `json:"newfilename"` - Sha1 string `json:"sha1"` - MD5 string `json:"md5"` -} - -type personalGroupsResp struct { - apiResult - Groups []struct { - ID int64 `json:"id"` - Name string `json:"name"` - } `json:"groups"` -} - -type countingWriter struct { - n *int64 -} - -func (w countingWriter) Write(p []byte) (int, error) { - *w.n += int64(len(p)) - return len(p), nil -} +const ENDPOINT_BUSINESS = "https://365.kdocs.cn" +const ENDPOINT_PERSONAL = "https://drive.wps.cn" func (d *Wps) isPersonal() bool { + // prefer d.login if available, as it may be set by islogin API + // which can determine account type more reliably + // one login session only support one type + // can not use personal and company account at the same time + if d.login != nil { + return !d.login.IsCompanyAccount + } return strings.TrimSpace(d.Mode) == "Personal" } func (d *Wps) driveHost() string { if d.isPersonal() { - return personalEndpoint + return ENDPOINT_PERSONAL } - return endpoint + return ENDPOINT_BUSINESS } func (d *Wps) drivePrefix() string { @@ -114,20 +51,18 @@ func (d *Wps) origin() string { return d.driveHost() } -func (d *Wps) canDownload(f *FileInfo) bool { - if f == nil || f.Type == "folder" { - return false +func (d *Wps) getUA() string { + if d.CustomUA != "" { + return d.CustomUA } - if f.FilePerms.Download != 0 { - return true - } - return d.isPersonal() + return base.UserAgent } func (d *Wps) request(ctx context.Context) *resty.Request { - return base.RestyClient.R(). + return d.client.R(). SetHeader("Cookie", d.Cookie). SetHeader("Accept", "application/json"). + SetHeader("User-Agent", d.getUA()). SetContext(ctx) } @@ -219,30 +154,10 @@ func checkAPI(resp *resty.Response, result apiResult) error { return nil } -func (d *Wps) ensureCompanyID(ctx context.Context) error { - if d.isPersonal() { - return nil - } - if d.companyID != "" { - return nil - } - var resp workspaceResp - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(endpoint + "/3rd/plussvr/compose/v1/users/self/workspaces?fields=name&comp_status=active") - if err != nil { - return err - } - if r != nil && r.IsError() { - return fmt.Errorf("http error: %d", r.StatusCode()) - } - if len(resp.Companies) == 0 { - return fmt.Errorf("no company id") - } - d.companyID = strconv.FormatInt(resp.Companies[0].ID, 10) - return nil -} - func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { - if d.isPersonal() { + // different APIs + switch d.Mode { + case "Personal": var resp personalGroupsResp r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(d.driveURL("/api/v3/groups")) if err != nil { @@ -256,37 +171,46 @@ func (d *Wps) getGroups(ctx context.Context) ([]Group, error) { res = append(res, Group{GroupID: g.ID, Name: g.Name}) } return res, nil + case "Business": + var resp groupsResp + url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%d/users/self/groups/private", ENDPOINT_BUSINESS, d.login.CompanyID) + r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + return resp.Groups, nil } - if err := d.ensureCompanyID(ctx); err != nil { - return nil, err - } - var resp groupsResp - url := fmt.Sprintf("%s/3rd/plus/groups/v1/companies/%s/users/self/groups/private", endpoint, d.companyID) - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) - } - return resp.Groups, nil + return nil, fmt.Errorf("unsupported mode: %s", d.Mode) } func (d *Wps) getFiles(ctx context.Context, groupID, parentID int64) ([]FileInfo, error) { var resp filesResp - url := fmt.Sprintf("%s/api/v5/groups/%d/files", d.driveHost()+d.drivePrefix(), groupID) - r, err := d.request(ctx). - SetQueryParam("parentid", strconv.FormatInt(parentID, 10)). - SetResult(&resp). - SetError(&resp). - Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) + var files []FileInfo + next_offset := 0 + for range 50 { + url := fmt.Sprintf("%s/api/v5/groups/%d/files", d.driveHost()+d.drivePrefix(), groupID) + r, err := d.request(ctx). + SetQueryParam("parentid", strconv.FormatInt(parentID, 10)). + SetQueryParam("offset", fmt.Sprint(next_offset)). + SetResult(&resp). + SetError(&resp). + Get(url) + if err != nil { + return nil, err + } + if r != nil && r.IsError() { + return nil, fmt.Errorf("http error: %d", r.StatusCode()) + } + files = append(files, resp.Files...) + if resp.NextOffset == -1 { + break + } + next_offset = resp.NextOffset } - return resp.Files, nil + return files, nil } func parseTime(v int64) time.Time { @@ -303,165 +227,6 @@ func joinPath(basePath, name string) string { return strings.TrimRight(basePath, "/") + "/" + name } -func normalizePath(path string) string { - clean := strings.TrimSpace(path) - if clean == "" || clean == "/" { - return "/" - } - return "/" + strings.Trim(clean, "/") -} - -func (d *Wps) resolveCacheStore() *resolveCacheStore { - if d == nil { - return nil - } - if v, ok := resolveCaches.Load(d); ok { - if s, ok := v.(*resolveCacheStore); ok { - return s - } - } - s := &resolveCacheStore{m: make(map[string]resolveCacheEntry)} - if v, loaded := resolveCaches.LoadOrStore(d, s); loaded { - if s2, ok := v.(*resolveCacheStore); ok { - return s2 - } - } - return s -} - -func (d *Wps) getResolveCache(path string) (*resolvedNode, bool) { - s := d.resolveCacheStore() - if s == nil { - return nil, false - } - s.mu.RLock() - e, ok := s.m[path] - s.mu.RUnlock() - if !ok || e.node == nil { - return nil, false - } - if !e.expire.IsZero() && time.Now().After(e.expire) { - s.mu.Lock() - delete(s.m, path) - s.mu.Unlock() - return nil, false - } - return e.node, true -} - -func (d *Wps) setResolveCache(path string, node *resolvedNode) { - s := d.resolveCacheStore() - if s == nil || node == nil { - return - } - s.mu.Lock() - s.m[path] = resolveCacheEntry{node: node, expire: time.Now().Add(10 * time.Minute)} - s.mu.Unlock() -} - -func (d *Wps) clearResolveCache() { - s := d.resolveCacheStore() - if s == nil { - return - } - s.mu.Lock() - if len(s.m) != 0 { - s.m = make(map[string]resolveCacheEntry) - } - s.mu.Unlock() -} - -func (d *Wps) resolvePath(ctx context.Context, path string) (*resolvedNode, error) { - cacheKey := normalizePath(path) - if n, ok := d.getResolveCache(cacheKey); ok { - return n, nil - } - clean := strings.TrimSpace(path) - if clean == "" { - clean = "/" - } - clean = strings.Trim(clean, "/") - if clean == "" { - n := &resolvedNode{kind: "root"} - d.setResolveCache("/", n) - return n, nil - } - seg := strings.Split(clean, "/") - groups, err := d.getGroups(ctx) - if err != nil { - return nil, err - } - var grp *Group - for i := range groups { - if groups[i].Name == seg[0] { - grp = &groups[i] - break - } - } - if grp == nil { - return nil, fmt.Errorf("group not found") - } - cur := "/" + seg[0] - gn := &resolvedNode{kind: "group", group: *grp} - d.setResolveCache(cur, gn) - if len(seg) == 1 { - return gn, nil - } - parentID := int64(0) - var lastNode *resolvedNode - for i := 1; i < len(seg); i++ { - files, err := d.getFiles(ctx, grp.GroupID, parentID) - if err != nil { - return nil, err - } - var found *FileInfo - for j := range files { - if files[j].Name == seg[i] { - found = &files[j] - break - } - } - if found == nil { - return nil, fmt.Errorf("path not found") - } - if i < len(seg)-1 && found.Type != "folder" { - return nil, fmt.Errorf("path not found") - } - fi := *found - parentID = fi.ID - cur = cur + "/" + seg[i] - kind := "file" - if fi.Type == "folder" { - kind = "folder" - } - n := &resolvedNode{kind: kind, group: *grp, file: &fi} - d.setResolveCache(cur, n) - lastNode = n - } - if lastNode == nil { - return nil, fmt.Errorf("path not found") - } - return lastNode, nil -} - -func (d *Wps) fileToObj(basePath string, f FileInfo) *Obj { - name := f.Name - path := joinPath(basePath, name) - obj := &Obj{ - id: path, - name: name, - size: f.Size, - ctime: parseTime(f.Ctime), - mtime: parseTime(f.Mtime), - isDir: f.Type == "folder", - path: path, - } - if !obj.isDir { - obj.canDownload = d.canDownload(&f) - } - return obj -} - func (d *Wps) doJSON(ctx context.Context, method, url string, body interface{}) error { var result apiResult req := d.jsonRequest(ctx).SetBody(body).SetResult(&result).SetError(&result) @@ -483,580 +248,16 @@ func (d *Wps) doJSON(ctx context.Context, method, url string, body interface{}) return checkAPI(resp, result) } -func (d *Wps) list(ctx context.Context, basePath string) ([]model.Obj, error) { - if strings.TrimSpace(basePath) == "" { - basePath = "/" - } - node, err := d.resolvePath(ctx, basePath) - if err != nil { - return nil, err - } - if node.kind == "root" { - groups, err := d.getGroups(ctx) - if err != nil { - return nil, err - } - res := make([]model.Obj, 0, len(groups)) - for _, g := range groups { - path := joinPath(basePath, g.Name) - obj := &Obj{ - id: path, - name: g.Name, - ctime: parseTime(0), - mtime: parseTime(0), - isDir: true, - path: path, - } - res = append(res, obj) - d.setResolveCache(normalizePath(path), &resolvedNode{kind: "group", group: g}) - } - d.setResolveCache("/", &resolvedNode{kind: "root"}) - return res, nil - } - if node.kind != "group" && node.kind != "folder" { - return nil, nil - } - parentID := int64(0) - if node.file != nil && node.kind == "folder" { - parentID = node.file.ID - } - files, err := d.getFiles(ctx, node.group.GroupID, parentID) - if err != nil { - return nil, err - } - res := make([]model.Obj, 0, len(files)) - for _, f := range files { - res = append(res, d.fileToObj(basePath, f)) - path := normalizePath(joinPath(basePath, f.Name)) - fi := f - kind := "file" - if fi.Type == "folder" { - kind = "folder" - } - d.setResolveCache(path, &resolvedNode{kind: kind, group: node.group, file: &fi}) - } - return res, nil -} - -func (d *Wps) link(ctx context.Context, path string) (*model.Link, error) { - node, err := d.resolvePath(ctx, path) - if err != nil { - return nil, err - } - if node.kind != "file" || node.file == nil { - return nil, errs.NotSupport - } - if !d.canDownload(node.file) { - return nil, fmt.Errorf("no download permission") - } - url := fmt.Sprintf("%s/api/v5/groups/%d/files/%d/download?support_checksums=sha1", d.driveHost()+d.drivePrefix(), node.group.GroupID, node.file.ID) - var resp downloadResp - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) - } - if resp.URL == "" { - return nil, fmt.Errorf("empty download url") - } - return &model.Link{URL: resp.URL, Header: http.Header{}}, nil -} - -func (d *Wps) makeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - if parentDir == nil { - return errs.NotSupport - } - node, err := d.resolvePath(ctx, parentDir.GetPath()) - if err != nil { - return err - } - if node.kind != "group" && node.kind != "folder" { - return errs.NotSupport - } - parentID := int64(0) - if node.file != nil && node.kind == "folder" { - parentID = node.file.ID - } - body := map[string]interface{}{ - "groupid": node.group.GroupID, - "name": dirName, - "parentid": parentID, - } - if err := d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/folder"), body); err != nil { - return err - } - d.clearResolveCache() - return nil -} - -func (d *Wps) move(ctx context.Context, srcObj, dstDir model.Obj) error { - if srcObj == nil || dstDir == nil { - return errs.NotSupport - } - nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) - if err != nil { - return err - } - nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) - if err != nil { - return err - } - if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { - return errs.NotSupport - } - if nodeDst.kind != "group" && nodeDst.kind != "folder" { - return errs.NotSupport - } - targetParentID := int64(0) - if nodeDst.file != nil && nodeDst.kind == "folder" { - targetParentID = nodeDst.file.ID - } - body := map[string]interface{}{ - "fileids": []int64{nodeSrc.file.ID}, - "target_groupid": nodeDst.group.GroupID, - "target_parentid": targetParentID, - } - url := fmt.Sprintf("/api/v3/groups/%d/files/batch/move", nodeSrc.group.GroupID) - for { - var res apiResult - resp, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&res). - SetError(&res). - Post(d.driveURL(url)) - if err != nil { - return err - } - - if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { - time.Sleep(500 * time.Millisecond) - continue - } - - if err := checkAPI(resp, res); err != nil { - return err - } - break - } - d.clearResolveCache() - return nil -} - -func (d *Wps) rename(ctx context.Context, srcObj model.Obj, newName string) error { - if srcObj == nil { - return errs.NotSupport - } - node, err := d.resolvePath(ctx, srcObj.GetPath()) - if err != nil { - return err - } - if node.kind != "file" && node.kind != "folder" { - return errs.NotSupport - } - url := fmt.Sprintf("/api/v3/groups/%d/files/%d", node.group.GroupID, node.file.ID) - body := map[string]string{"fname": newName} - if err := d.doJSON(ctx, http.MethodPut, d.driveURL(url), body); err != nil { - return err - } - d.clearResolveCache() - return nil -} - -func (d *Wps) copy(ctx context.Context, srcObj, dstDir model.Obj) error { - if srcObj == nil || dstDir == nil { - return errs.NotSupport - } - nodeSrc, err := d.resolvePath(ctx, srcObj.GetPath()) - if err != nil { - return err - } - nodeDst, err := d.resolvePath(ctx, dstDir.GetPath()) - if err != nil { - return err - } - if nodeSrc.kind != "file" && nodeSrc.kind != "folder" { - return errs.NotSupport - } - if nodeDst.kind != "group" && nodeDst.kind != "folder" { - return errs.NotSupport - } - targetParentID := int64(0) - if nodeDst.file != nil && nodeDst.kind == "folder" { - targetParentID = nodeDst.file.ID - } - body := map[string]interface{}{ - "fileids": []int64{nodeSrc.file.ID}, - "groupid": nodeSrc.group.GroupID, - "target_groupid": nodeDst.group.GroupID, - "target_parentid": targetParentID, - "duplicated_name_model": 1, - } - url := fmt.Sprintf("/api/v3/groups/%d/files/batch/copy", nodeSrc.group.GroupID) - for { - var res apiResult - resp, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&res). - SetError(&res). - Post(d.driveURL(url)) - if err != nil { - return err - } - - if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { - time.Sleep(500 * time.Millisecond) - continue - } - - if err := checkAPI(resp, res); err != nil { - return err - } - break - } - d.clearResolveCache() - return nil -} - -func (d *Wps) remove(ctx context.Context, obj model.Obj) error { - if obj == nil { - return errs.NotSupport - } - node, err := d.resolvePath(ctx, obj.GetPath()) - if err != nil { - return err - } - if node.kind != "file" && node.kind != "folder" { - return errs.NotSupport - } - - body := map[string]interface{}{ - "fileids": []int64{node.file.ID}, - } - url := fmt.Sprintf("/api/v3/groups/%d/files/batch/delete", node.group.GroupID) - - for { - var res apiResult - resp, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&res). - SetError(&res). - Post(d.driveURL(url)) - if err != nil { - return err - } - - // 无法连续创建文件夹删除。如果一定要删除，每0.5s 尝试一次创建下一个删除请求，应当避免递归删除文件夹 - if resp.StatusCode() == 403 && res.Result == "fileTaskDuplicated" { - time.Sleep(500 * time.Millisecond) - continue - } - - if err := checkAPI(resp, res); err != nil { - return err - } - break - } - d.clearResolveCache() - return nil -} - -func cacheAndHash(file model.FileStreamer, up driver.UpdateProgress) (model.File, int64, string, string, error) { - h1 := sha1.New() - h256 := sha256.New() - size := file.GetSize() - var counted int64 - ws := []io.Writer{h1, h256} - if size <= 0 { - ws = append(ws, countingWriter{n: &counted}) - } - p := up - f, err := file.CacheFullAndWriter(&p, io.MultiWriter(ws...)) - if err != nil { - return nil, 0, "", "", err - } - if size <= 0 { - size = counted - } - return f, size, hex.EncodeToString(h1.Sum(nil)), hex.EncodeToString(h256.Sum(nil)), nil -} - -func (d *Wps) createUpload(ctx context.Context, groupID, parentID int64, name string, size int64, sha1Hex, sha256Hex string) (*uploadCreateUpdateResp, error) { - body := map[string]string{ - "group_id": strconv.FormatInt(groupID, 10), - "name": name, - "parent_id": strconv.FormatInt(parentID, 10), - "sha1": sha1Hex, - "sha256": sha256Hex, - "size": strconv.FormatInt(size, 10), - } - var resp uploadCreateUpdateResp - r, err := d.jsonRequest(ctx). - SetBody(body). - SetResult(&resp). - SetError(&resp). - Put(d.driveURL("/api/v5/files/upload/create_update")) - if err != nil { - return nil, err - } - if err := checkAPI(r, resp.apiResult); err != nil { - return nil, err - } - if resp.URL == "" { - return nil, fmt.Errorf("empty upload url") - } - return &resp, nil -} - -func normalizeETag(v string) string { - v = strings.TrimSpace(v) - if strings.HasPrefix(v, "W/") { - v = strings.TrimSpace(strings.TrimPrefix(v, "W/")) - } - return strings.Trim(v, `"`) -} - -func (d *Wps) commitUpload(ctx context.Context, etag, key string, groupID, parentID int64, name, sha1Hex string, size int64, store string) error { - store = strings.TrimSpace(store) - if store == "" { - store = "ks3" - } - storeKey := "" - if key != "" { - storeKey = key - } - body := map[string]interface{}{ - "etag": etag, - "groupid": groupID, - "key": key, - "name": name, - "parentid": parentID, - "sha1": sha1Hex, - "size": size, - "store": store, - "storekey": storeKey, - } - return d.doJSON(ctx, http.MethodPost, d.driveURL("/api/v5/files/file"), body) -} - -func (d *Wps) put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { - if dstDir == nil || file == nil { - return errs.NotSupport - } - if up == nil { - up = func(float64) {} - } - node, err := d.resolvePath(ctx, dstDir.GetPath()) - if err != nil { - return err - } - if node.kind != "group" && node.kind != "folder" { - return errs.NotSupport - } - parentID := int64(0) - if node.file != nil && node.kind == "folder" { - parentID = node.file.ID - } - f, size, sha1Hex, sha256Hex, err := cacheAndHash(file, func(float64) {}) - if err != nil { - return err - } - if c, ok := f.(io.Closer); ok { - defer c.Close() - } - - // 在隐藏文件名前加_上传，这是WPS的限制，无法上传隐藏文件，也无法将任何文件重命名为隐藏文件，所有隐藏文件会被自动加上_ 上传 - // 甚至可以上传前缀是..的文件，但是单个点就是不行 - realName := file.GetName() - uploadName := realName - if strings.HasPrefix(realName, ".") { - uploadName = "_" + realName - } - - info, err := d.createUpload(ctx, node.group.GroupID, parentID, uploadName, size, sha1Hex, sha256Hex) - if err != nil { - return err - } - if _, err := f.Seek(0, io.SeekStart); err != nil { - return err - } - rf := driver.NewLimitedUploadFile(ctx, f) - prog := driver.NewProgress(size, model.UpdateProgressWithRange(up, 0, 1)) - - method := strings.ToUpper(strings.TrimSpace(info.Method)) - if method == "" { - method = http.MethodPut - } - - var req *http.Request - if method == http.MethodPost && len(info.Request.FormData) > 0 { - if size == 0 { - var buf bytes.Buffer - mw := multipart.NewWriter(&buf) - for k, v := range info.Request.FormData { - if err := mw.WriteField(k, v); err != nil { - return err - } - } - part, err := mw.CreateFormFile("file", uploadName) - if err != nil { - return err - } - if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { - return err - } - if err := mw.Close(); err != nil { - return err - } - req, err = http.NewRequestWithContext(ctx, method, info.URL, bytes.NewReader(buf.Bytes())) - if err != nil { - return err - } - for k, v := range info.Request.Headers { - req.Header.Set(k, v) - } - req.Header.Set("Content-Type", mw.FormDataContentType()) - req.ContentLength = int64(buf.Len()) - req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10)) - } else { - pr, pw := io.Pipe() - mw := multipart.NewWriter(pw) - req, err = http.NewRequestWithContext(ctx, method, info.URL, pr) - if err != nil { - return err - } - for k, v := range info.Request.Headers { - req.Header.Set(k, v) - } - req.Header.Set("Content-Type", mw.FormDataContentType()) - go func() { - for k, v := range info.Request.FormData { - if err := mw.WriteField(k, v); err != nil { - pw.CloseWithError(err) - return - } - } - part, err := mw.CreateFormFile("file", uploadName) - if err != nil { - pw.CloseWithError(err) - return - } - if _, err := io.Copy(part, io.TeeReader(rf, prog)); err != nil { - pw.CloseWithError(err) - return - } - if err := mw.Close(); err != nil { - pw.CloseWithError(err) - return - } - pw.Close() - }() +func unwrapWpsObj(obj model.Obj) (*Obj, error) { + for obj != nil { + if node, ok := obj.(*Obj); ok { + return node, nil } - } else { - var body = io.TeeReader(rf, prog) - if size == 0 { - body = bytes.NewReader(nil) - } - req, err = http.NewRequestWithContext(ctx, method, info.URL, body) - if err != nil { - return err - } - for k, v := range info.Request.Headers { - req.Header.Set(k, v) - } - req.ContentLength = size - req.Header.Set("Content-Length", strconv.FormatInt(size, 10)) - } - - c := *base.RestyClient.GetClient() - c.Timeout = 0 - resp, err := (&c).Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if !statusOK(resp.StatusCode, info.Response.ExpectCode) { - io.Copy(io.Discard, resp.Body) - return fmt.Errorf("http error: %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - - etag := normalizeETag(respArg(info.Response.ArgsETag, resp, body)) - if etag == "" { - etag = normalizeETag(resp.Header.Get("ETag")) - } - - key := strings.TrimSpace(respArg(info.Response.ArgsKey, resp, body)) - if key == "" { - key = strings.TrimSpace(resp.Header.Get("x-obs-save-key")) - } - - var pr uploadPutResp - sha1FromServer := "" - if err := json.Unmarshal(body, &pr); err == nil { - sha1FromServer = strings.TrimSpace(pr.NewFilename) - if sha1FromServer == "" { - sha1FromServer = strings.TrimSpace(pr.Sha1) - } - if etag == "" && pr.MD5 != "" { - etag = strings.TrimSpace(pr.MD5) - } - } - - if sha1FromServer == "" { - if v := extractXMLTag(string(body), "ETag"); v != "" { - sha1FromServer = v - if etag == "" { - etag = v - } - } - } - if sha1FromServer == "" && key != "" && len(key) == 40 { - sha1FromServer = key - } - if sha1FromServer == "" { - sha1FromServer = sha1Hex - } - - if etag == "" { - return fmt.Errorf("empty etag") - } - if sha1FromServer == "" { - return fmt.Errorf("empty sha1") - } - - store := strings.TrimSpace(info.Store) - commitKey := "" - if strings.TrimSpace(info.Response.ArgsKey) != "" { - commitKey = key - if commitKey == "" { - commitKey = sha1FromServer + unwrap, ok := obj.(model.ObjUnwrap) + if !ok { + break } + obj = unwrap.Unwrap() } - - if err := d.commitUpload(ctx, etag, commitKey, node.group.GroupID, parentID, uploadName, sha1FromServer, size, store); err != nil { - return err - } - - up(1) - return nil -} - -func (d *Wps) spaces(ctx context.Context) (*spacesResp, error) { - url := fmt.Sprintf("%s/api/v3/spaces", d.driveHost()+d.drivePrefix()) - var resp spacesResp - r, err := d.request(ctx).SetResult(&resp).SetError(&resp).Get(url) - if err != nil { - return nil, err - } - if r != nil && r.IsError() { - return nil, fmt.Errorf("http error: %d", r.StatusCode()) - } - return &resp, nil + return nil, fmt.Errorf("invalid object type") } diff --git a/go.mod b/go.mod index e751bcfe0..cf8089ca2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/OpenListTeam/OpenList/v4 -go 1.23.4 +go 1.24.0 + +toolchain go1.24.13 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 @@ -14,7 +16,7 @@ require ( github.com/OpenListTeam/wopan-sdk-go v0.1.5 github.com/ProtonMail/go-crypto v1.3.0 github.com/ProtonMail/gopenpgp/v2 v2.9.0 - github.com/SheltonZhu/115driver v1.1.1 + github.com/SheltonZhu/115driver v1.2.3 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/antchfx/htmlquery v1.3.5 github.com/antchfx/xpath v1.3.5 @@ -39,6 +41,7 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.4 github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 + github.com/glebarez/sqlite v1.11.0 github.com/go-resty/resty/v2 v2.16.5 github.com/go-webauthn/webauthn v0.13.4 github.com/golang-jwt/jwt/v4 v4.5.2 @@ -73,11 +76,11 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.6.0 github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 - golang.org/x/crypto v0.40.0 + golang.org/x/crypto v0.46.0 golang.org/x/image v0.29.0 - golang.org/x/net v0.42.0 - golang.org/x/oauth2 v0.30.0 - golang.org/x/time v0.12.0 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 + golang.org/x/time v0.14.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 @@ -87,8 +90,8 @@ require ( ) require ( - cloud.google.com/go/compute/metadata v0.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/BurntSushi/toml v1.6.0 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect @@ -102,10 +105,12 @@ require ( github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cronokirby/saferith v0.33.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.8.4 // indirect github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect github.com/geoffgarside/ber v1.2.0 // indirect + github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -121,15 +126,20 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/relvacode/iso8601 v1.6.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.uber.org/mock v0.5.0 // indirect golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect - golang.org/x/mod v0.27.0 // indirect + golang.org/x/mod v0.30.0 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect + modernc.org/libc v1.22.5 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/sqlite v1.23.1 // indirect ) require ( - github.com/OpenListTeam/115-sdk-go v0.2.2 + github.com/OpenListTeam/115-sdk-go v0.2.3 github.com/STARRY-S/zip v0.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/blevesearch/go-faiss v1.0.25 // indirect @@ -158,8 +168,7 @@ require ( github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/yuin/goldmark v1.7.13 - go4.org v0.0.0-20230225012048-214862532bf5 + go4.org v0.0.0-20260112195520-a5071408f32f resty.dev/v3 v3.0.0-beta.2 // indirect ) @@ -285,14 +294,14 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.4.0 // indirect golang.org/x/arch v0.18.0 // indirect - golang.org/x/sync v0.16.0 - golang.org/x/sys v0.34.0 - golang.org/x/term v0.33.0 // indirect - golang.org/x/text v0.27.0 - golang.org/x/tools v0.35.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.6 // indirect + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.40.0 + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 + golang.org/x/tools v0.39.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -303,4 +312,4 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -// replace github.com/OpenListTeam/115-sdk-go => ../../OpenListTeam/115-sdk-go +replace github.com/OpenListTeam/115-sdk-go => github.com/Ironboxplus/115-sdk-go v0.2.5 diff --git a/go.sum b/go.sum index 57964be4c..1a1288506 100644 --- a/go.sum +++ b/go.sum @@ -1,27 +1,10 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.53.0 h1:MZQCQQaRwOrAcuKjiHWHrgKykt4fZyuwF2dtiG3fGW8= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go/auth v0.16.2 h1:QvBAGFPLrDeoiNjyfVunhQ10HKNYuOwZ5noee0M5df4= -cloud.google.com/go/auth v0.16.2/go.mod h1:sRBas2Y1fB1vZTdurouM0AzuYQBMZinrUYL8EufhtEA= +cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= +cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0= +cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= -cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= @@ -34,11 +17,12 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZY github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= +github.com/Ironboxplus/115-sdk-go v0.2.5 h1:8giRpk9TwDT/5oe6F1H6h5+OwMMkM/vSYJqZomsmp20= +github.com/Ironboxplus/115-sdk-go v0.2.5/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/KarpelesLab/reflink v1.0.2 h1:hQ1aM3TmjU2kTNUx5p/HaobDoADYk+a6AuEinG4Cv88= github.com/KarpelesLab/reflink v1.0.2/go.mod h1:WGkTOKNjd1FsJKBw3mu4JvrPEDJyJJ+JPtxBkbPoCok= github.com/KirCute/zip v1.0.1 h1:L/tVZglOiDVKDi9Ud+fN49htgKdQ3Z0H80iX8OZk13c= @@ -47,8 +31,6 @@ github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7Y github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/OpenListTeam/115-sdk-go v0.2.2 h1:JCrGHqQjBX3laOA6Hw4CuBovSg7g+FC5s0LEAYsRciU= -github.com/OpenListTeam/115-sdk-go v0.2.2/go.mod h1:cfvitk2lwe6036iNi2h+iNxwxWDifKZsSvNtrur5BqU= github.com/OpenListTeam/go-cache v0.1.0 h1:eV2+FCP+rt+E4OCJqLUW7wGccWZNJMV0NNkh+uChbAI= github.com/OpenListTeam/go-cache v0.1.0/go.mod h1:AHWjKhNK3LE4rorVdKyEALDHoeMnP8SjiNyfVlB+Pz4= github.com/OpenListTeam/gsync v0.1.0 h1:ywzGybOvA3lW8K1BUjKZ2IUlT2FSlzPO4DOazfYXjcs= @@ -81,8 +63,8 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2 github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.1.1 h1:9EMhe2ZJflGiAaZbYInw2jqxTcqZNF+DtVDsEy70aFU= -github.com/SheltonZhu/115driver v1.1.1/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= +github.com/SheltonZhu/115driver v1.2.3 h1:94XMP/ey7VXIlpoBLIJHEoXu7N8YsELZlXVbxWcDDvk= +github.com/SheltonZhu/115driver v1.2.3/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= @@ -204,7 +186,6 @@ github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCN github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= @@ -225,12 +206,8 @@ github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQ github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= @@ -269,6 +246,8 @@ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cn github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= @@ -277,8 +256,6 @@ github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7 github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg= github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6 h1:q1b+gv6AG2TDPN+f0QAkbRrAvJ3ZosnwRLTKNxSXlaA= @@ -304,12 +281,14 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= +github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= +github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= +github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= @@ -347,32 +326,14 @@ github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXe github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0= github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -380,22 +341,17 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU= github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b h1:Qcx5LM0fSiks9uCyFZwDBUasd3lxd1RM0GYpL+Li5o4= +github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= -github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= -github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= +github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= @@ -418,8 +374,6 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= @@ -431,7 +385,6 @@ github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0 github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= @@ -477,8 +430,6 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg= @@ -497,11 +448,8 @@ github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -620,7 +568,6 @@ github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= @@ -635,16 +582,17 @@ github.com/rclone/rclone v1.70.3 h1:rg/WNh4DmSVZyKP2tHZ4lAaWEyMi7h/F0r7smOMA3IE= github.com/rclone/rclone v1.70.3/go.mod h1:nLyN+hpxAsQn9Rgt5kM774lcRDad82x/KqQeBZ83cMo= github.com/relvacode/iso8601 v1.6.0 h1:eFXUhMJN3Gz8Rcq82f9DTMW0svjtAVuIEULglM7QHTU= github.com/relvacode/iso8601 v1.6.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= @@ -720,46 +668,37 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= -github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3 h1:PSRwrE5QBufPnOjdgIkRs5KBV1Avq3SY8oksj2Z+k3o= github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3/go.mod h1:CKriYB8bkNgSbYUQF1khSpejKb5IsV6cR7MdaAR7Fc0= go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= -go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= -go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= +go4.org v0.0.0-20260112195520-a5071408f32f h1:ziUVAjmTPwQMBmYR1tbdRFJPtTcQUI12fH9QQjfb0Sw= +go4.org v0.0.0-20260112195520-a5071408f32f/go.mod h1:ZRJnO5ZI4zAwMFp+dS1+V6J6MSyAowhRqAE+DPa1Xp0= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc= golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -768,59 +707,23 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -832,42 +735,22 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -887,8 +770,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -899,11 +782,9 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -915,99 +796,42 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.236.0 h1:CAiEiDVtO4D/Qja2IA9VzlFrgPnK3XVMmRoJZlSWbc0= -google.golang.org/api v0.236.0/go.mod h1:X1WF9CU2oTc+Jml1tiIxGmWFK/UZezdqEu09gcxZAj4= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ= +google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= @@ -1032,17 +856,17 @@ gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDa gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 116b4cd35..74e218f8f 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -54,6 +54,7 @@ func InitConfig() { } } configPath = filepath.Clean(configPath) + conf.ConfigPath = configPath log.Infof("reading config file: %s", configPath) if !utils.Exists(configPath) { log.Infof("config file not exists, creating default config file") diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 7bff851de..d7fd8ea47 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -153,7 +153,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.SharePreviewDownloadByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.SharePreviewArchivesByDefault, Value: "false", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, - {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, // frontend {Key: conf.NonEFSZipEncoding, Value: "IBM437", Type: conf.TypeString, Group: model.PREVIEW}, // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index d97cb6796..7b91769f9 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -12,7 +12,6 @@ import ( log "github.com/sirupsen/logrus" "gorm.io/driver/mysql" "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" "gorm.io/gorm/schema" @@ -41,7 +40,7 @@ func InitDB() { var dB *gorm.DB var err error if flags.Dev { - dB, err = gorm.Open(sqlite.Open("file::memory:?cache=shared"), gormConfig) + dB, err = gorm.Open(openSQLite("file::memory:?cache=shared"), gormConfig) conf.Conf.Database.Type = "sqlite3" } else { database := conf.Conf.Database @@ -51,7 +50,7 @@ func InitDB() { if !(strings.HasSuffix(database.DBFile, ".db") && len(database.DBFile) > 3) { log.Fatalf("db name error.") } - dB, err = gorm.Open(sqlite.Open(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental", + dB, err = gorm.Open(openSQLite(fmt.Sprintf("%s?_journal=WAL&_vacuum=incremental", database.DBFile)), gormConfig) } case "mysql": diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go index c4a72a966..5d4c814dc 100644 --- a/internal/bootstrap/patch/all.go +++ b/internal/bootstrap/patch/all.go @@ -44,6 +44,7 @@ var UpgradePatches = []VersionPatches{ Version: "v4.1.9", Patches: []func(){ v4_1_9.EnableWebDavProxy, + v4_1_9.ResetSkipTlsVerify, }, }, } diff --git a/internal/bootstrap/patch/v4_1_9/skip_tls.go b/internal/bootstrap/patch/v4_1_9/skip_tls.go new file mode 100644 index 000000000..1d9858d95 --- /dev/null +++ b/internal/bootstrap/patch/v4_1_9/skip_tls.go @@ -0,0 +1,32 @@ +package v4_1_9 + +import ( + "os" + "strings" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +func ResetSkipTlsVerify() { + if !conf.Conf.TlsInsecureSkipVerify { + return + } + if !strings.HasPrefix(conf.Version, "v") { + return + } + + conf.Conf.TlsInsecureSkipVerify = false + + confBody, err := utils.Json.MarshalIndent(conf.Conf, "", " ") + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: marshal config error: %+v", err) + return + } + err = os.WriteFile(conf.ConfigPath, confBody, 0o777) + if err != nil { + utils.Log.Errorf("[ResetSkipTlsVerify] failed to rewrite config: update config struct error: %+v", err) + return + } + utils.Log.Infof("[ResetSkipTlsVerify] succeeded to set tls_insecure_skip_verify to false") +} diff --git a/internal/bootstrap/run.go b/internal/bootstrap/run.go index 6740dba65..ff02509ba 100644 --- a/internal/bootstrap/run.go +++ b/internal/bootstrap/run.go @@ -15,6 +15,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/db" "github.com/OpenListTeam/OpenList/v4/internal/fs" + "github.com/OpenListTeam/OpenList/v4/internal/frontend" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server" "github.com/OpenListTeam/OpenList/v4/server/middlewares" @@ -273,6 +274,7 @@ func Start() { func Shutdown(timeout time.Duration) { utils.Log.Println("Shutdown server...") + frontend.StopWatcher() fs.ArchiveContentUploadTaskManager.RemoveAll() ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() diff --git a/internal/bootstrap/sqlite_driver_glebarez.go b/internal/bootstrap/sqlite_driver_glebarez.go new file mode 100644 index 000000000..a45a8baeb --- /dev/null +++ b/internal/bootstrap/sqlite_driver_glebarez.go @@ -0,0 +1,12 @@ +//go:build !sqlite_cgo_compat && !(linux && (mips || mips64 || mips64le || mipsle || loong64)) && !(windows && 386) + +package bootstrap + +import ( + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func openSQLite(dsn string) gorm.Dialector { + return sqlite.Open(dsn) +} diff --git a/internal/bootstrap/sqlite_driver_gorm.go b/internal/bootstrap/sqlite_driver_gorm.go new file mode 100644 index 000000000..e69630eac --- /dev/null +++ b/internal/bootstrap/sqlite_driver_gorm.go @@ -0,0 +1,12 @@ +//go:build sqlite_cgo_compat || (linux && (mips || mips64 || mips64le || mipsle || loong64)) || (windows && 386) + +package bootstrap + +import ( + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func openSQLite(dsn string) gorm.Dialector { + return sqlite.Open(dsn) +} diff --git a/internal/conf/config.go b/internal/conf/config.go index c5ace8005..e0c7b52cf 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -118,6 +118,7 @@ type Config struct { TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` DistDir string `json:"dist_dir"` + FrontendRepo string `json:"frontend_repo" env:"FRONTEND_REPO"` Log LogConfig `json:"log" envPrefix:"LOG_"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` MaxBufferLimit int `json:"max_buffer_limitMB" env:"MAX_BUFFER_LIMIT_MB"` @@ -162,7 +163,8 @@ func DefaultConfig(dataDir string) *Config { Host: "http://localhost:7700", Index: "openlist", }, - BleveDir: indexDir, + BleveDir: indexDir, + FrontendRepo: FrontendRepoDefault, Log: LogConfig{ Enable: true, Name: logPath, @@ -182,7 +184,7 @@ func DefaultConfig(dataDir string) *Config { MmapThreshold: 4, MaxConnections: 0, MaxConcurrency: 64, - TlsInsecureSkipVerify: true, + TlsInsecureSkipVerify: false, Tasks: TasksConfig{ Download: TaskConfig{ Workers: 5, diff --git a/internal/conf/var.go b/internal/conf/var.go index 9a02eca26..30dd73de8 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -12,11 +12,13 @@ var ( GitCommit string = "unknown" Version string = "dev" WebVersion string = "rolling" + FrontendRepoDefault string = "OpenListTeam/OpenList-Frontend" ) var ( - Conf *Config - URL *url.URL + Conf *Config + URL *url.URL + ConfigPath string ) var SlicesMap = make(map[string][]string) diff --git a/internal/frontend/fetcher.go b/internal/frontend/fetcher.go new file mode 100644 index 000000000..363d5dfca --- /dev/null +++ b/internal/frontend/fetcher.go @@ -0,0 +1,501 @@ +package frontend + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/cmd/flags" + "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const ( + defaultFrontendRepo = "OpenListTeam/OpenList-Frontend" + versionFile = ".frontend_version" + distDirName = "dist" +) + +// FetchResult contains the result of a fetch operation +type FetchResult struct { + Version string + Downloaded bool + DistPath string +} + +// GetDistPath returns the path where dynamically fetched frontend dist is stored +func GetDistPath() string { + return filepath.Join(flags.DataDir, "frontend_dist") +} + +// GetVersionFilePath returns the path to the version tracking file +func GetVersionFilePath() string { + return filepath.Join(GetDistPath(), versionFile) +} + +// HasValidDist checks if the dynamic dist directory exists and has an index.html +func HasValidDist() bool { + distPath := GetDistPath() + _, err := os.Stat(filepath.Join(distPath, distDirName, "index.html")) + return err == nil +} + +// ReadCurrentVersion reads the currently cached version from disk +func ReadCurrentVersion() string { + data, err := os.ReadFile(GetVersionFilePath()) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +// writeVersion writes the version string to the version tracking file +func writeVersion(version string) error { + return os.WriteFile(GetVersionFilePath(), []byte(version), 0644) +} + +// FetchFromRolling downloads the frontend dist from the GitHub rolling release +func FetchFromRolling(ctx context.Context) (*FetchResult, error) { + return fetchFromTag(ctx, "rolling", "") +} + +// FetchFromLatest downloads the frontend dist from the GitHub latest release +func FetchFromLatest(ctx context.Context) (*FetchResult, error) { + return fetchFromTag(ctx, "", "") +} + +// githubRelease represents a GitHub release for JSON parsing +type githubRelease struct { + TagName string `json:"tag_name"` + Assets []struct { + BrowserDownloadURL string `json:"browser_download_url"` + Name string `json:"name"` + } `json:"assets"` + PublishedAt string `json:"published_at"` +} + +type githubRef struct { + Object struct { + Type string `json:"type"` + SHA string `json:"sha"` + } `json:"object"` +} + +type githubAnnotatedTag struct { + Object struct { + Type string `json:"type"` + SHA string `json:"sha"` + } `json:"object"` +} + +func getFrontendRepo() string { + if conf.Conf != nil && strings.TrimSpace(conf.Conf.FrontendRepo) != "" { + return strings.TrimSpace(conf.Conf.FrontendRepo) + } + return defaultFrontendRepo +} + +func shortHash(sha string) string { + const shortLen = 12 + if len(sha) > shortLen { + return sha[:shortLen] + } + return sha +} + +func versionIdentifier(tag, commitSHA, fallback string) string { + if strings.TrimSpace(commitSHA) != "" { + return fmt.Sprintf("%s@%s", tag, shortHash(commitSHA)) + } + if strings.TrimSpace(fallback) != "" { + return fallback + } + return tag +} + +func resolveTagCommitSHA(ctx context.Context, client *http.Client, baseURL, tag string) (string, error) { + if strings.TrimSpace(tag) == "" { + return "", fmt.Errorf("empty tag") + } + + apiBase := "https://api.github.com" + if baseURL != "" { + apiBase = strings.TrimRight(baseURL, "/") + } + + repo := getFrontendRepo() + refURL := fmt.Sprintf("%s/repos/%s/git/ref/tags/%s", apiBase, repo, url.PathEscape(tag)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, refURL, nil) + if err != nil { + return "", fmt.Errorf("create ref request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("fetch tag ref: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("tag ref API returned %d: %s", resp.StatusCode, string(body)) + } + + var ref githubRef + if err := json.NewDecoder(resp.Body).Decode(&ref); err != nil { + return "", fmt.Errorf("decode ref JSON: %w", err) + } + + switch ref.Object.Type { + case "commit": + if ref.Object.SHA == "" { + return "", fmt.Errorf("empty commit sha in ref response") + } + return ref.Object.SHA, nil + case "tag": + if ref.Object.SHA == "" { + return "", fmt.Errorf("empty tag sha in ref response") + } + tagObjURL := fmt.Sprintf("%s/repos/%s/git/tags/%s", apiBase, repo, ref.Object.SHA) + tagReq, err := http.NewRequestWithContext(ctx, http.MethodGet, tagObjURL, nil) + if err != nil { + return "", fmt.Errorf("create tag object request: %w", err) + } + tagReq.Header.Set("Accept", "application/vnd.github.v3+json") + tagReq.Header.Set("User-Agent", "OpenList") + + tagResp, err := client.Do(tagReq) + if err != nil { + return "", fmt.Errorf("fetch tag object: %w", err) + } + defer tagResp.Body.Close() + + if tagResp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(tagResp.Body) + return "", fmt.Errorf("tag object API returned %d: %s", tagResp.StatusCode, string(body)) + } + + var tagObj githubAnnotatedTag + if err := json.NewDecoder(tagResp.Body).Decode(&tagObj); err != nil { + return "", fmt.Errorf("decode tag object JSON: %w", err) + } + if tagObj.Object.SHA == "" { + return "", fmt.Errorf("empty object sha in tag object response") + } + return tagObj.Object.SHA, nil + default: + if ref.Object.SHA == "" { + return "", fmt.Errorf("unsupported ref object type %q with empty sha", ref.Object.Type) + } + return ref.Object.SHA, nil + } +} + +// fetchFromTag downloads frontend dist from a GitHub release tag. +// If baseURL is non-empty, it replaces api.github.com (used for testing). +func fetchFromTag(ctx context.Context, tag string, baseURL string) (*FetchResult, error) { + repo := getFrontendRepo() + var apiURL string + if baseURL != "" { + if tag == "" { + apiURL = fmt.Sprintf("%s/repos/%s/releases/latest", baseURL, repo) + } else { + apiURL = fmt.Sprintf("%s/repos/%s/releases/tags/%s", baseURL, repo, tag) + } + } else { + if tag == "" { + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/releases/latest", repo) + } else { + apiURL = fmt.Sprintf("https://api.github.com/repos/%s/releases/tags/%s", repo, tag) + } + } + + client := newHTTPClient() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("fetch release info: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("github API returned %d: %s", resp.StatusCode, string(body)) + } + + var release githubRelease + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + return nil, fmt.Errorf("decode release JSON: %w", err) + } + + // Find the dist tarball URL (non-lite) + var tarURL string + for _, asset := range release.Assets { + if strings.Contains(asset.Name, "openlist-frontend-dist") && + !strings.Contains(asset.Name, "lite") && + strings.HasSuffix(asset.Name, ".tar.gz") { + tarURL = asset.BrowserDownloadURL + break + } + } + if tarURL == "" { + return nil, fmt.Errorf("no frontend dist tarball found in release %s", release.TagName) + } + + commitSHA, err := resolveTagCommitSHA(ctx, client, baseURL, release.TagName) + if err != nil { + utils.Log.Warnf("[frontend] failed to resolve tag %s hash: %v", release.TagName, err) + } + + resolvedVersion := versionIdentifier(release.TagName, commitSHA, tarURL) + + // Use tag+commit-hash as the primary version identifier. + // For rolling releases the tag itself is static, but its target commit moves. + // If hash resolve fails, fallback to tarball URL so updates can still be detected. + currentVersion := ReadCurrentVersion() + if currentVersion == resolvedVersion && HasValidDist() { + utils.Log.Infof("[frontend] version %s already cached, skipping download", resolvedVersion) + return &FetchResult{ + Version: resolvedVersion, + Downloaded: false, + DistPath: filepath.Join(GetDistPath(), distDirName), + }, nil + } + + utils.Log.Infof("[frontend] downloading version %s from %s", resolvedVersion, tarURL) + if err := downloadAndExtract(ctx, client, tarURL); err != nil { + return nil, fmt.Errorf("download and extract: %w", err) + } + + if err := writeVersion(resolvedVersion); err != nil { + utils.Log.Warnf("[frontend] failed to write version file: %v", err) + } + + utils.Log.Infof("[frontend] successfully fetched version %s", resolvedVersion) + return &FetchResult{ + Version: resolvedVersion, + Downloaded: true, + DistPath: filepath.Join(GetDistPath(), distDirName), + }, nil +} + +func downloadAndExtract(ctx context.Context, client *http.Client, url string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("create download request: %w", err) + } + req.Header.Set("User-Agent", "OpenList") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("download tarball: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download returned status %d", resp.StatusCode) + } + + destDir := GetDistPath() + tmpDir := filepath.Join(destDir, ".tmp-"+fmt.Sprintf("%d", time.Now().UnixNano())) + if err := os.MkdirAll(tmpDir, 0755); err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + if err := extractTarGz(resp.Body, tmpDir); err != nil { + return fmt.Errorf("extract tar.gz: %w", err) + } + + // Determine the source directory: + // If the tarball contains a "dist" subdirectory, use it; + // otherwise, the files are at the root and we use tmpDir directly. + srcDir := tmpDir + if _, err := os.Stat(filepath.Join(tmpDir, distDirName)); err == nil { + srcDir = filepath.Join(tmpDir, distDirName) + } + + // Atomic swap: rename source to final + finalDir := filepath.Join(destDir, distDirName) + oldDir := filepath.Join(destDir, distDirName+".old") + // Remove previous backup if exists + os.RemoveAll(oldDir) + // Move current dist out of the way if it exists + os.Rename(finalDir, oldDir) + // Move new dist into place + if err := os.Rename(srcDir, finalDir); err != nil { + // Rollback + os.RemoveAll(finalDir) + os.Rename(oldDir, finalDir) + return fmt.Errorf("rename new dist: %w", err) + } + os.RemoveAll(oldDir) + + return nil +} + +func extractTarGz(r io.Reader, dest string) error { + gzr, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("gzip reader: %w", err) + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("tar next: %w", err) + } + + // Normalize: strip leading ./ so "./dist" becomes "dist" + name := strings.TrimPrefix(hdr.Name, "./") + if name == "" || name == "." { + continue // skip bare directory entry + } + + target := filepath.Join(dest, name) + + // Security: prevent path traversal + if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("path traversal detected: %s", hdr.Name) + } + + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return err + } + f.Close() + } + } + return nil +} + +// newHTTPClient creates an HTTP client that respects proxy configuration +func newHTTPClient() *http.Client { + transport := &http.Transport{} + if conf.Conf != nil && conf.Conf.ProxyAddress != "" { + if proxyURL := mustParseURL(conf.Conf.ProxyAddress); proxyURL != nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + } + return &http.Client{ + Transport: transport, + Timeout: 5 * time.Minute, + } +} + +func mustParseURL(raw string) *url.URL { + u, err := url.Parse(raw) + if err != nil { + utils.Log.Warnf("[frontend] invalid proxy URL %q: %v", raw, err) + return nil + } + return u +} + +// EnsureDist ensures a valid frontend dist is available, fetching if necessary. +// It first tries the dynamic dist, then falls back to fetching from GitHub. +// Returns the path to the dist directory, or empty string if no dist is available. +func EnsureDist(ctx context.Context) string { + // If user explicitly configured dist_dir, use that + if conf.Conf != nil && conf.Conf.DistDir != "" { + if _, err := os.Stat(filepath.Join(conf.Conf.DistDir, "index.html")); err == nil { + return conf.Conf.DistDir + } + } + + // Check if dynamic dist already exists + if HasValidDist() { + return filepath.Join(GetDistPath(), distDirName) + } + + // If auto-fetch is enabled (and WebVersion is rolling/beta/dev), try fetching + if shouldAutoFetch() { + utils.Log.Infof("[frontend] no local dist found, fetching from rolling release...") + result, err := FetchFromRolling(ctx) + if err != nil { + utils.Log.Warnf("[frontend] failed to fetch from rolling: %v", err) + // Fall through to return empty (embedded dist will be used as fallback) + return "" + } + return result.DistPath + } + + return "" +} + +func shouldAutoFetch() bool { + v := conf.WebVersion + return v == "" || v == "rolling" || v == "beta" || v == "dev" +} + +// Ensure the directory exists for the frontend dist +func init() { + _ = os.MkdirAll(GetDistPath(), 0755) +} + +// Ensure that the sync.Once pattern is used for the fetcher +var ( + fetchMu sync.Mutex + fetchDone bool + fetchResult string +) + +// EnsureDistOnce is a thread-safe version of EnsureDist that only fetches once per process. +// On failure, it does not lock the state so subsequent calls can retry. +func EnsureDistOnce(ctx context.Context) string { + fetchMu.Lock() + defer fetchMu.Unlock() + if fetchDone { + return fetchResult + } + result := EnsureDist(ctx) + if result != "" { + fetchResult = result + fetchDone = true + } + return result +} + +// ResetFetchState resets the fetch state (used for testing or re-fetch) +func ResetFetchState() { + fetchMu.Lock() + defer fetchMu.Unlock() + fetchDone = false + fetchResult = "" +} diff --git a/internal/frontend/fetcher_test.go b/internal/frontend/fetcher_test.go new file mode 100644 index 000000000..14f7a3391 --- /dev/null +++ b/internal/frontend/fetcher_test.go @@ -0,0 +1,480 @@ +package frontend + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +// createTestTarGz creates a tar.gz containing files with given names and contents +func createTestTarGz(t *testing.T, files map[string]string) []byte { + t.Helper() + pr, pw := io.Pipe() + go func() { + defer pw.Close() + gw := gzip.NewWriter(pw) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return + } + if _, err := tw.Write([]byte(content)); err != nil { + return + } + } + }() + data, err := io.ReadAll(pr) + if err != nil { + t.Fatalf("read tar.gz: %v", err) + } + return data +} + +func TestExtractTarGz(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "dist/index.html": "hello", + "dist/assets/app.js": "console.log('app')", + "dist/assets/style.css": "body {}", + "dist/images/logo.svg": "", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err != nil { + t.Fatalf("extractTarGz: %v", err) + } + + for name, expectedContent := range files { + path := filepath.Join(tmpDir, name) + data, err := os.ReadFile(path) + if err != nil { + t.Errorf("read %s: %v", name, err) + continue + } + if string(data) != expectedContent { + t.Errorf("content of %s: got %q, want %q", name, string(data), expectedContent) + } + } +} + +func TestExtractTarGzDotSlash(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "./dist/index.html": "dot-slash", + "./": "", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err != nil { + t.Fatalf("extractTarGz with ./ prefix: %v", err) + } + + data, err := os.ReadFile(filepath.Join(tmpDir, "dist", "index.html")) + if err != nil { + t.Fatalf("read dist/index.html: %v", err) + } + if string(data) != "dot-slash" { + t.Errorf("got %q, want dot-slash content", string(data)) + } +} + +func TestExtractTarGzPathTraversal(t *testing.T) { + tmpDir := t.TempDir() + files := map[string]string{ + "../../../etc/passwd": "root:x:0:0", + } + tarData := createTestTarGz(t, files) + + err := extractTarGz(strings.NewReader(string(tarData)), tmpDir) + if err == nil { + t.Fatal("expected error for path traversal, got nil") + } + if !strings.Contains(err.Error(), "path traversal") { + t.Errorf("expected path traversal error, got: %v", err) + } +} + +func TestHasValidDist(t *testing.T) { + if HasValidDist() { + t.Log("HasValidDist returned true (may have existing dist from previous runs)") + } +} + +func TestWriteAndReadVersion(t *testing.T) { + _ = os.MkdirAll(GetDistPath(), 0755) + versionPath := GetVersionFilePath() + + origData, origErr := os.ReadFile(versionPath) + defer func() { + if origErr == nil { + _ = os.WriteFile(versionPath, origData, 0644) + } else { + _ = os.Remove(versionPath) + } + }() + + testVersion := "v1.0.0-test" + if err := writeVersion(testVersion); err != nil { + t.Fatalf("writeVersion: %v", err) + } + + got := ReadCurrentVersion() + if got != testVersion { + t.Errorf("ReadCurrentVersion: got %q, want %q", got, testVersion) + } +} + +func TestShouldAutoFetch(t *testing.T) { + origVersion := conf.WebVersion + defer func() { conf.WebVersion = origVersion }() + + tests := []struct { + version string + want bool + }{ + {"", true}, + {"rolling", true}, + {"beta", true}, + {"dev", true}, + {"v3.0.0", false}, + {"latest", false}, + } + + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + conf.WebVersion = tt.version + if got := shouldAutoFetch(); got != tt.want { + t.Errorf("shouldAutoFetch(%q) = %v, want %v", tt.version, got, tt.want) + } + }) + } +} + +func TestFetchFromRollingIntegration(t *testing.T) { + files := map[string]string{ + "./dist/index.html": "integration", + "./dist/assets/test.js": "console.log('test')", + } + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/OpenListTeam/OpenList-Frontend/releases/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling-test", + "assets": [{ + "name": "openlist-frontend-dist.tar.gz", + "browser_download_url": "%s/download/frontend.tar.gz" + }] + }`, ts.URL))) + case "/repos/OpenListTeam/OpenList-Frontend/git/ref/tags/rolling-test": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "0123456789abcdef0123456789abcdef01234567" + } + }`)) + case "/download/frontend.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(200) + w.Write(tarData) + default: + w.WriteHeader(404) + } + })) + defer ts.Close() + + ResetFetchState() + + destDir := GetDistPath() + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + ctx := context.Background() + result, err := fetchFromTag(ctx, "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag: %v", err) + } + + if result.Version != "rolling-test@0123456789ab" { + t.Errorf("version: got %q, want %q", result.Version, "rolling-test@0123456789ab") + } + if !result.Downloaded { + t.Error("expected Downloaded=true") + } + + idx, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(idx) != "integration" { + t.Errorf("index.html: got %q", string(idx)) + } + + ver := ReadCurrentVersion() + expectedVer := "rolling-test@0123456789ab" + if ver != expectedVer { + t.Errorf("version file: got %q, want %q", ver, expectedVer) + } +} + +func TestLegacyConfigJSONGetsDefaultFrontendRepo(t *testing.T) { + cfg := conf.DefaultConfig("data") + if err := json.Unmarshal([]byte(`{"site_url":"https://example.com"}`), cfg); err != nil { + t.Fatalf("unmarshal legacy config: %v", err) + } + if cfg.FrontendRepo != defaultFrontendRepo { + t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, defaultFrontendRepo) + } +} + +func TestDefaultConfigUsesBuiltFrontendRepoDefault(t *testing.T) { + orig := conf.FrontendRepoDefault + conf.FrontendRepoDefault = "Ironboxplus/OpenList-Frontend" + defer func() { conf.FrontendRepoDefault = orig }() + + cfg := conf.DefaultConfig("data") + if cfg.FrontendRepo != "Ironboxplus/OpenList-Frontend" { + t.Fatalf("FrontendRepo: got %q, want %q", cfg.FrontendRepo, "Ironboxplus/OpenList-Frontend") + } +} + +func TestExistingConfigJSONKeepsFrontendRepo(t *testing.T) { + cfg := conf.DefaultConfig("data") + if err := json.Unmarshal([]byte(`{"frontend_repo":"Ironboxplus/OpenList-Frontend"}`), cfg); err != nil { + t.Fatalf("unmarshal config with frontend_repo: %v", err) + } + if cfg.FrontendRepo != "Ironboxplus/OpenList-Frontend" { + t.Fatalf("FrontendRepo: got %q", cfg.FrontendRepo) + } +} + +func TestFetchFromTagUsesConfiguredFrontendRepo(t *testing.T) { + origConf := conf.Conf + if origConf == nil { + conf.Conf = &conf.Config{} + } else { + confCopy := *origConf + conf.Conf = &confCopy + } + defer func() { conf.Conf = origConf }() + conf.Conf.FrontendRepo = "Ironboxplus/OpenList-Frontend" + + files := map[string]string{ + "./dist/index.html": "custom-repo", + } + tarData := createTestTarGz(t, files) + + var ts *httptest.Server + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/Ironboxplus/OpenList-Frontend/releases/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(fmt.Sprintf(`{ + "tag_name": "rolling-custom", + "assets": [{ + "name": "openlist-frontend-dist.tar.gz", + "browser_download_url": "%s/download/custom.tar.gz" + }] + }`, ts.URL))) + case "/repos/Ironboxplus/OpenList-Frontend/git/ref/tags/rolling-custom": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "fedcba9876543210fedcba9876543210fedcba98" + } + }`)) + case "/download/custom.tar.gz": + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(tarData) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + ResetFetchState() + _ = os.MkdirAll(GetDistPath(), 0o755) + _ = os.RemoveAll(filepath.Join(GetDistPath(), distDirName)) + _ = os.Remove(GetVersionFilePath()) + + result, err := fetchFromTag(context.Background(), "rolling", ts.URL) + if err != nil { + t.Fatalf("fetchFromTag(custom repo): %v", err) + } + if result.Version != "rolling-custom@fedcba987654" { + t.Fatalf("version: got %q, want %q", result.Version, "rolling-custom@fedcba987654") + } + + data, err := os.ReadFile(filepath.Join(result.DistPath, "index.html")) + if err != nil { + t.Fatalf("read index.html: %v", err) + } + if string(data) != "custom-repo" { + t.Fatalf("index.html: got %q", string(data)) + } +} + +func TestResolveTagCommitSHA_AnnotatedTag(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/repos/OpenListTeam/OpenList-Frontend/git/ref/tags/rolling": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{ + "object": { + "type": "tag", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + }`)) + case "/repos/OpenListTeam/OpenList-Frontend/git/tags/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{ + "object": { + "type": "commit", + "sha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + }`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + + sha, err := resolveTagCommitSHA(context.Background(), ts.Client(), ts.URL, "rolling") + if err != nil { + t.Fatalf("resolveTagCommitSHA: %v", err) + } + if sha != "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" { + t.Fatalf("sha: got %q, want %q", sha, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + } +} + +func TestVersionIdentifierFallback(t *testing.T) { + tests := []struct { + name string + tag string + sha string + fallback string + want string + }{ + {name: "hash preferred", tag: "rolling", sha: "0123456789abcdef", fallback: "fallback", want: "rolling@0123456789ab"}, + {name: "fallback url", tag: "rolling", sha: "", fallback: "http://example.com/dist.tar.gz", want: "http://example.com/dist.tar.gz"}, + {name: "tag only", tag: "rolling", sha: "", fallback: "", want: "rolling"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := versionIdentifier(tt.tag, tt.sha, tt.fallback) + if got != tt.want { + t.Fatalf("versionIdentifier() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestResolveTagCommitSHA_RealGitHubWithProxy10808(t *testing.T) { + if os.Getenv("OPENLIST_REAL_GITHUB_TEST") != "1" { + t.Skip("set OPENLIST_REAL_GITHUB_TEST=1 to run real GitHub integration test (proxy 127.0.0.1:10808 recommended)") + } + + if os.Getenv("HTTP_PROXY") == "" && os.Getenv("http_proxy") == "" { + _ = os.Setenv("HTTP_PROXY", "http://127.0.0.1:10808") + } + if os.Getenv("HTTPS_PROXY") == "" && os.Getenv("https_proxy") == "" { + _ = os.Setenv("HTTPS_PROXY", "http://127.0.0.1:10808") + } + + client := newHTTPClient() + sha, err := resolveTagCommitSHA(context.Background(), client, "", "rolling") + if err != nil { + t.Fatalf("resolveTagCommitSHA(real): %v", err) + } + + matched, _ := regexp.MatchString("^[0-9a-f]{40}$", sha) + if !matched { + t.Fatalf("sha format invalid: %q", sha) + } + + // Optional sanity: ensure API can fetch release JSON in real scenario + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "https://api.github.com/repos/OpenListTeam/OpenList-Frontend/releases/tags/rolling", nil) + if err != nil { + t.Fatalf("create release request: %v", err) + } + req.Header.Set("Accept", "application/vnd.github.v3+json") + req.Header.Set("User-Agent", "OpenList") + resp, err := client.Do(req) + if err != nil { + t.Fatalf("fetch release: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("release API status=%d body=%s", resp.StatusCode, string(body)) + } + + var release map[string]any + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + t.Fatalf("decode release: %v", err) + } + if release["tag_name"] == nil { + t.Fatalf("release tag_name missing") + } +} + +func TestEnsureDistOnceFailureDoesNotLock(t *testing.T) { + ResetFetchState() + _ = os.MkdirAll(GetDistPath(), 0755) + + destDir := GetDistPath() + os.RemoveAll(filepath.Join(destDir, distDirName)) + os.Remove(GetVersionFilePath()) + + origVersion := conf.WebVersion + conf.WebVersion = "v3.0.0" // shouldAutoFetch returns false + defer func() { conf.WebVersion = origVersion }() + + ctx := context.Background() + + result := EnsureDistOnce(ctx) + if result != "" { + t.Errorf("expected empty result, got %q", result) + } + + // Second call should also work (not locked by previous failure) + result2 := EnsureDistOnce(ctx) + if result2 != "" { + t.Errorf("expected empty result on retry, got %q", result2) + } +} diff --git a/internal/frontend/watcher.go b/internal/frontend/watcher.go new file mode 100644 index 000000000..1a0102fd1 --- /dev/null +++ b/internal/frontend/watcher.go @@ -0,0 +1,122 @@ +package frontend + +import ( + "context" + "sync" + "time" + + "github.com/OpenListTeam/OpenList/v4/pkg/utils" +) + +const defaultCheckInterval = 30 * time.Minute + +// Watcher periodically checks for new frontend versions and fetches them. +type Watcher struct { + interval time.Duration + stopCh chan struct{} + stopped bool + mu sync.Mutex + onUpdated func() +} + +// NewWatcher creates a new frontend watcher. +// onUpdated is called when a new version is fetched (used to reload static files). +func NewWatcher(onUpdated func()) *Watcher { + return &Watcher{ + interval: defaultCheckInterval, + stopCh: make(chan struct{}), + onUpdated: onUpdated, + } +} + +// SetInterval changes the check interval. Must be called before Start. +func (w *Watcher) SetInterval(d time.Duration) { + w.mu.Lock() + defer w.mu.Unlock() + w.interval = d +} + +// Start begins the periodic check loop in a background goroutine. +func (w *Watcher) Start() { + w.mu.Lock() + interval := w.interval + w.mu.Unlock() + + go func() { + utils.Log.Infof("[frontend] watcher started, checking every %s", interval) + // Check immediately on start, then periodically + w.check() + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-w.stopCh: + utils.Log.Infof("[frontend] watcher stopped") + return + case <-ticker.C: + w.check() + } + } + }() +} + +// Stop signals the watcher to stop. +func (w *Watcher) Stop() { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped { + return + } + w.stopped = true + close(w.stopCh) +} + +func (w *Watcher) check() { + if !shouldAutoFetch() { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + result, err := FetchFromRolling(ctx) + if err != nil { + utils.Log.Warnf("[frontend] watcher check failed: %v", err) + return + } + + if result.Downloaded { + utils.Log.Infof("[frontend] watcher fetched new version: %s", result.Version) + if w.onUpdated != nil { + w.onUpdated() + } + } +} + +// globalWatcher is the singleton watcher instance +var ( + globalWatcher *Watcher + watcherMu sync.Mutex +) + +// StartWatcher starts the global frontend watcher. +func StartWatcher(onUpdated func()) { + watcherMu.Lock() + defer watcherMu.Unlock() + if globalWatcher != nil { + return + } + globalWatcher = NewWatcher(onUpdated) + globalWatcher.Start() +} + +// StopWatcher stops the global frontend watcher. +func StopWatcher() { + watcherMu.Lock() + defer watcherMu.Unlock() + if globalWatcher != nil { + globalWatcher.Stop() + globalWatcher = nil + } +} diff --git a/internal/fs/copy_move.go b/internal/fs/copy_move.go index e78fc9be8..5ae92524e 100644 --- a/internal/fs/copy_move.go +++ b/internal/fs/copy_move.go @@ -17,6 +17,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/OpenListTeam/tache" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type taskType uint8 @@ -192,6 +193,20 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer dstActualPath := stdpath.Join(t.DstActualPath, srcObj.GetName()) task_group.TransferCoordinator.AppendPayload(t.groupID, task_group.DstPathToHook(dstActualPath)) + // Pre-create the destination directory first + t.Status = "ensuring destination directory exists" + if err := op.MakeDir(t.Ctx(), t.DstStorage, dstActualPath); err != nil { + log.Warnf("[copy_move] failed to ensure destination dir [%s]: %v, will continue", dstActualPath, err) + // Continue anyway - the directory might exist but Get failed due to cache issues + } + + // Pre-create subdirectories (up to 1 level deep) to avoid deep recursion issues + // Balances between reducing API calls and maintaining fault tolerance + t.Status = "pre-creating subdirectories" + if err := t.preCreateDirectoryTree(objs, t.SrcActualPath, dstActualPath, 1); err != nil { + log.Warnf("[copy_move] failed to pre-create directory tree: %v, will continue", err) + // Continue anyway - individual directories will be created on-demand + } existedObjs := make(map[string]bool) if t.TaskType == merge { dstObjs, err := op.List(t.Ctx(), t.DstStorage, dstActualPath, model.ListArgs{}) @@ -263,6 +278,88 @@ func (t *FileTransferTask) RunWithNextTaskCallback(f func(nextTask *FileTransfer return op.Put(context.WithValue(t.Ctx(), conf.SkipHookKey, struct{}{}), t.DstStorage, t.DstActualPath, ss, t.SetProgress) } +// preCreateDirectoryTree is a thin method wrapper that resolves the storage-bound +// makeDir / listSrc functions and delegates to the pure preCreateDirTreeFn helper. +func (t *FileTransferTask) preCreateDirectoryTree(objs []model.Obj, srcBasePath, dstBasePath string, maxDepth int) error { + makeDir := func(ctx context.Context, path string) error { + return op.MakeDir(ctx, t.DstStorage, path) + } + listSrc := func(ctx context.Context, path string) ([]model.Obj, error) { + return op.List(ctx, t.SrcStorage, path, model.ListArgs{}) + } + return preCreateDirTreeFn(t.Ctx(), objs, srcBasePath, dstBasePath, maxDepth, makeDir, listSrc) +} + +// preCreateDirTreeFn recursively scans source directory tree and pre-creates +// directories on destination up to maxDepth levels to avoid deep MakeDir recursion issues. +// +// - maxDepth=0 – only create dirs in the current objs list (no recursion) +// - maxDepth=1 – also recurse one level deeper, etc. +// - srcBasePath – current source directory path; passed explicitly through all +// recursion levels so that subdirSrcPath is always correct (do NOT use +// t.SrcActualPath, which is fixed at the top-level path). +// +// makeDir and listSrc are injected to enable testing without a real storage driver. +func preCreateDirTreeFn( + ctx context.Context, + objs []model.Obj, + srcBasePath, dstBasePath string, + maxDepth int, + makeDir func(context.Context, string) error, + listSrc func(context.Context, string) ([]model.Obj, error), +) error { + // First pass: create immediate subdirectories + var subdirs []model.Obj + for _, obj := range objs { + // Check for cancellation + if err := ctx.Err(); err != nil { + return err + } + + if obj.IsDir() { + subdirPath := stdpath.Join(dstBasePath, obj.GetName()) + if err := makeDir(ctx, subdirPath); err != nil { + log.Debugf("[copy_move] failed to pre-create dir [%s]: %v", subdirPath, err) + // Continue with other directories + } + subdirs = append(subdirs, obj) + // No explicit sleep here: drivers that have QPS limits (e.g. 115, BaiduNetDisk) + // implement WaitLimit via a token-bucket rate.Limiter and call it inside their + // MakeDir, so op.MakeDir already blocks at the correct per-driver rate. + } + } + + // Stop recursion if max depth reached + if maxDepth <= 0 { + return nil + } + + // Second pass: recursively scan and create nested subdirectories + for _, subdir := range subdirs { + if err := ctx.Err(); err != nil { + return err + } + + // Build paths relative to srcBasePath (NOT t.SrcActualPath) so that + // deeper recursion levels resolve to the correct source paths. + subdirSrcPath := stdpath.Join(srcBasePath, subdir.GetName()) + subdirDstPath := stdpath.Join(dstBasePath, subdir.GetName()) + + subObjs, err := listSrc(ctx, subdirSrcPath) + if err != nil { + log.Debugf("[copy_move] failed to list subdir [%s] for pre-creation: %v", subdirSrcPath, err) + continue // Skip this subdirectory, will handle when processing + } + + // Recursively create subdirectories with decreased depth + if err := preCreateDirTreeFn(ctx, subObjs, subdirSrcPath, subdirDstPath, maxDepth-1, makeDir, listSrc); err != nil { + return err + } + } + + return nil +} + var ( CopyTaskManager *tache.Manager[*FileTransferTask] MoveTaskManager *tache.Manager[*FileTransferTask] diff --git a/internal/fs/copy_move_test.go b/internal/fs/copy_move_test.go new file mode 100644 index 000000000..5c9f4b539 --- /dev/null +++ b/internal/fs/copy_move_test.go @@ -0,0 +1,317 @@ +package fs + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +// ---------- helpers ---------- + +func dirObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: true} +} + +func fileObj(name string) model.Obj { + return &model.Object{Name: name, IsFolder: false} +} + +// callRecorder records every path passed to makeDir and listSrc. +type callRecorder struct { + mu sync.Mutex + mkdirs []string + lists []string + // listReturns maps srcPath → objects to return (nil = empty) + listReturns map[string][]model.Obj + // mkdirErr maps dstPath → error to return + mkdirErr map[string]error +} + +func newRecorder() *callRecorder { + return &callRecorder{ + listReturns: make(map[string][]model.Obj), + mkdirErr: make(map[string]error), + } +} + +func (r *callRecorder) makeDir(_ context.Context, path string) error { + r.mu.Lock() + r.mkdirs = append(r.mkdirs, path) + err := r.mkdirErr[path] + r.mu.Unlock() + return err +} + +func (r *callRecorder) listSrc(_ context.Context, path string) ([]model.Obj, error) { + r.mu.Lock() + r.lists = append(r.lists, path) + objs := r.listReturns[path] + r.mu.Unlock() + return objs, nil +} + +func (r *callRecorder) hasMkdir(path string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range r.mkdirs { + if p == path { + return true + } + } + return false +} + +func (r *callRecorder) hasList(path string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range r.lists { + if p == path { + return true + } + } + return false +} + +// ---------- tests ---------- + +// TestPreCreateDirTreeFn_EmptyObjs: no objects → no calls at all. +func TestPreCreateDirTreeFn_EmptyObjs(t *testing.T) { + rec := newRecorder() + err := preCreateDirTreeFn(context.Background(), nil, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("expected 0 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + } + if len(rec.lists) != 0 { + t.Errorf("expected 0 List calls, got %d: %v", len(rec.lists), rec.lists) + } +} + +// TestPreCreateDirTreeFn_OnlyFiles: file objects only → zero MakeDir calls. +func TestPreCreateDirTreeFn_OnlyFiles(t *testing.T) { + objs := []model.Obj{fileObj("a.txt"), fileObj("b.txt")} + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("expected 0 MakeDir calls, got %d", len(rec.mkdirs)) + } +} + +// TestPreCreateDirTreeFn_FlatDirs_MaxDepth0: dirs present, maxDepth=0 → MakeDir +// called for each dir with correct dstPath, NO listSrc calls. +func TestPreCreateDirTreeFn_FlatDirs_MaxDepth0(t *testing.T) { + objs := []model.Obj{dirObj("subA"), fileObj("file.txt"), dirObj("subB")} + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst/parent", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !rec.hasMkdir("/dst/parent/subA") { + t.Error("expected MakeDir(/dst/parent/subA)") + } + if !rec.hasMkdir("/dst/parent/subB") { + t.Error("expected MakeDir(/dst/parent/subB)") + } + if rec.hasMkdir("/dst/parent/file.txt") { + t.Error("MakeDir must NOT be called for a file") + } + if len(rec.lists) != 0 { + t.Errorf("maxDepth=0 must not trigger any List calls, got: %v", rec.lists) + } +} + +// TestPreCreateDirTreeFn_Recursion_CorrectSrcPath is the regression test for the +// srcBasePath bug: with maxDepth=1 the recursive List must use the SUBDIR src path, +// not the original top-level srcBasePath. +func TestPreCreateDirTreeFn_Recursion_CorrectSrcPath(t *testing.T) { + // /src/parent contains [subA(dir), subB(dir)] + // /src/parent/subA contains [subA1(dir)] + // /src/parent/subB contains [] + topObjs := []model.Obj{dirObj("subA"), dirObj("subB")} + rec := newRecorder() + rec.listReturns["/src/parent/subA"] = []model.Obj{dirObj("subA1")} + rec.listReturns["/src/parent/subB"] = []model.Obj{} + + if err := preCreateDirTreeFn(context.Background(), topObjs, "/src/parent", "/dst/parent", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // ── first level dirs must be created + if !rec.hasMkdir("/dst/parent/subA") { + t.Error("expected MakeDir(/dst/parent/subA)") + } + if !rec.hasMkdir("/dst/parent/subB") { + t.Error("expected MakeDir(/dst/parent/subB)") + } + + // ── listSrc must use subdirSrcPath (NOT the whole /src/parent again) + if !rec.hasList("/src/parent/subA") { + t.Error("listSrc must be called with /src/parent/subA, got:", rec.lists) + } + if !rec.hasList("/src/parent/subB") { + t.Error("listSrc must be called with /src/parent/subB, got:", rec.lists) + } + // The original bug would have called listSrc("/src/parent/subA") as + // stdpath.Join(t.SrcActualPath, "subA") where t.SrcActualPath=="/src/parent", + // but in a deeper recursive call (e.g. maxDepth=2) it would have used + // the top-level path incorrectly; verify the nested mkdir used the right dst. + if !rec.hasMkdir("/dst/parent/subA/subA1") { + t.Error("expected MakeDir(/dst/parent/subA/subA1), got mkdirs:", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion: with maxDepth=1 recursion +// goes exactly one level. The nested list returns another dir, but since maxDepth +// reaches 0 that deeper dir must NOT be listed further. +func TestPreCreateDirTreeFn_MaxDepth1_NoFurtherRecursion(t *testing.T) { + topObjs := []model.Obj{dirObj("sub")} + rec := newRecorder() + // sub contains deeper, deeper contains deepest + rec.listReturns["/src/sub"] = []model.Obj{dirObj("deeper")} + rec.listReturns["/src/sub/deeper"] = []model.Obj{dirObj("deepest")} // should NOT be listed + + if err := preCreateDirTreeFn(context.Background(), topObjs, "/src", "/dst", 1, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !rec.hasMkdir("/dst/sub") { + t.Error("expected /dst/sub to be created") + } + if !rec.hasMkdir("/dst/sub/deeper") { + t.Error("expected /dst/sub/deeper to be created (within maxDepth=1)") + } + // deepest must NOT be created (would require maxDepth=2) + if rec.hasMkdir("/dst/sub/deeper/deepest") { + t.Error("/dst/sub/deeper/deepest must NOT be created at maxDepth=1") + } + // /src/sub/deeper must NOT be listed (we've hit maxDepth=0 at that point) + if rec.hasList("/src/sub/deeper") { + t.Error("/src/sub/deeper must NOT be listed when maxDepth reaches 0") + } +} + +// TestPreCreateDirTreeFn_ContextCancelled: context cancelled before processing → +// returns ctx.Err, makes zero or partial calls. +func TestPreCreateDirTreeFn_ContextCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // already cancelled + + objs := []model.Obj{dirObj("sub")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled, got: %v", err) + } + if len(rec.mkdirs) != 0 { + t.Errorf("no MakeDir should be called after cancellation, got: %v", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_ContextCancelledDuringRecursion: context is cancelled +// during the second-pass recursion loop. +func TestPreCreateDirTreeFn_ContextCancelledDuringRecursion(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + // Two dirs; cancel after the first List in recursion + callCount := 0 + listSrc := func(c context.Context, path string) ([]model.Obj, error) { + callCount++ + cancel() // cancel on first list call + return nil, nil + } + objs := []model.Obj{dirObj("sub1"), dirObj("sub2")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, listSrc) + if !errors.Is(err, context.Canceled) { + t.Errorf("expected context.Canceled after cancellation during recursion, got: %v", err) + } + if callCount > 1 { + t.Errorf("listSrc should have been called at most once before ctx.Err fired, got %d", callCount) + } +} + +// TestPreCreateDirTreeFn_MakeDirErrorNonFatal: a MakeDir failure on one dir must +// not stop processing of subsequent dirs. +func TestPreCreateDirTreeFn_MakeDirErrorNonFatal(t *testing.T) { + objs := []model.Obj{dirObj("subA"), dirObj("subB"), dirObj("subC")} + rec := newRecorder() + rec.mkdirErr["/dst/subA"] = errors.New("quota exceeded") + + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("error should not propagate from MakeDir failure: %v", err) + } + // All three must have been attempted despite the error on subA + for _, p := range []string{"/dst/subA", "/dst/subB", "/dst/subC"} { + if !rec.hasMkdir(p) { + t.Errorf("expected MakeDir(%s) to be called", p) + } + } +} + +// TestPreCreateDirTreeFn_ListErrorNonFatal: a List error for one subdir during +// recursion skips that subdir but continues with the rest. +func TestPreCreateDirTreeFn_ListErrorNonFatal(t *testing.T) { + objs := []model.Obj{dirObj("subA"), dirObj("subB")} + listCallCount := 0 + listSrc := func(_ context.Context, path string) ([]model.Obj, error) { + listCallCount++ + if path == "/src/subA" { + return nil, errors.New("I/O error") + } + return []model.Obj{dirObj("nested")}, nil + } + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 1, rec.makeDir, listSrc); err != nil { + t.Fatalf("List error must not be fatal: %v", err) + } + // subB's nested dir should still be processed despite subA's List failure + if !rec.hasMkdir("/dst/subB/nested") { + t.Error("expected /dst/subB/nested to be created despite subA list error, mkdirs:", rec.mkdirs) + } + if listCallCount != 2 { + t.Errorf("both subdirs must be attempted for listing, got %d calls", listCallCount) + } +} + +// TestPreCreateDirTreeFn_MixedObjs: mixed files and dirs; only dirs are processed. +func TestPreCreateDirTreeFn_MixedObjs(t *testing.T) { + objs := []model.Obj{ + fileObj("readme.md"), + dirObj("assets"), + fileObj("main.go"), + dirObj("pkg"), + } + rec := newRecorder() + if err := preCreateDirTreeFn(context.Background(), objs, "/src", "/dst", 0, rec.makeDir, rec.listSrc); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(rec.mkdirs) != 2 { + t.Errorf("expected exactly 2 MakeDir calls, got %d: %v", len(rec.mkdirs), rec.mkdirs) + } + if !rec.hasMkdir("/dst/assets") || !rec.hasMkdir("/dst/pkg") { + t.Errorf("unexpected mkdirs: %v", rec.mkdirs) + } +} + +// TestPreCreateDirTreeFn_Timeout: context with a very short deadline cancels execution. +func TestPreCreateDirTreeFn_Timeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + time.Sleep(5 * time.Millisecond) // ensure deadline has passed + + objs := []model.Obj{dirObj("sub")} + rec := newRecorder() + err := preCreateDirTreeFn(ctx, objs, "/src", "/dst", 1, rec.makeDir, rec.listSrc) + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("expected DeadlineExceeded, got: %v", err) + } +} diff --git a/internal/fs/list.go b/internal/fs/list.go index 1f92c7d46..113ba8231 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -2,13 +2,14 @@ package fs import ( "context" - "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/pkg/utils" + "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "path" ) // List files @@ -43,7 +44,29 @@ func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) om.InitHideReg(meta.Hide) } objs := om.Merge(_objs, virtualFiles...) - return objs, nil + objs, err = filterReadableObjs(objs, user, path, meta) + return objs, err +} + +func filterReadableObjs(objs []model.Obj, user *model.User, reqPath string, parentMeta *model.Meta) ([]model.Obj, error) { + var result []model.Obj + for _, obj := range objs { + var meta *model.Meta + objPath := path.Join(reqPath, obj.GetName()) + if obj.IsDir() { + var err error + meta, err = op.GetNearestMeta(objPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return result, err + } + } else { + meta = parentMeta + } + if common.CanRead(user, meta, objPath) { + result = append(result, obj) + } + } + return result, nil } func whetherHide(user *model.User, meta *model.Meta, path string) bool { @@ -60,7 +83,7 @@ func whetherHide(user *model.User, meta *model.Meta, path string) bool { return false } // if meta doesn't apply to sub_folder, don't hide - if !utils.PathEqual(meta.Path, path) && !meta.HSub { + if !common.MetaCoversPath(meta.Path, path, meta.HSub) { return false } // if is guest, hide diff --git a/internal/fs/list_test.go b/internal/fs/list_test.go new file mode 100644 index 000000000..ebaf4371e --- /dev/null +++ b/internal/fs/list_test.go @@ -0,0 +1,151 @@ +package fs + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestWhetherHide(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil user", + user: nil, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/folder", + want: false, + reason: "nil user (treated as admin) should not hide", + }, + { + name: "user with can_see_hides permission", + user: &model.User{ + Role: model.GENERAL, + Permission: 1, // bit 0 set = can see hides + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/folder", + want: false, + reason: "user with can_see_hides permission should not hide", + }, + { + name: "nil meta", + user: &model.User{ + Role: model.GUEST, + }, + meta: nil, + path: "/folder", + want: false, + reason: "nil meta should not hide", + }, + { + name: "empty hide string", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "", + HSub: true, + }, + path: "/folder", + want: false, + reason: "empty hide string should not hide", + }, + { + name: "exact path match with HSub=false", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: false, + }, + path: "/folder", + want: true, + reason: "exact path match should hide for guest", + }, + { + name: "sub path with HSub=true", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/folder/subfolder", + want: true, + reason: "sub path with HSub=true should hide for guest", + }, + { + name: "sub path with HSub=false", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: false, + }, + path: "/folder/subfolder", + want: false, + reason: "sub path with HSub=false should not hide", + }, + { + name: "non-sub path with HSub=true", + user: &model.User{ + Role: model.GUEST, + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/other", + want: false, + reason: "non-sub path should not hide even with HSub=true", + }, + { + name: "user without can_see_hides permission", + user: &model.User{ + Role: model.GENERAL, + Permission: 0, // bit 0 not set = cannot see hides + }, + meta: &model.Meta{ + Path: "/folder", + Hide: "secret", + HSub: true, + }, + path: "/folder", + want: true, + reason: "user without can_see_hides permission should hide", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := whetherHide(tt.user, tt.meta, tt.path) + if got != tt.want { + t.Errorf("whetherHide() = %v, want %v\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} diff --git a/internal/model/args.go b/internal/model/args.go index 073c94a63..d165908fb 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -25,6 +25,10 @@ type LinkArgs struct { Redirect bool } +// LinkRefresher is a callback function type for refreshing download links +// It returns a new Link and the associated object, or an error +type LinkRefresher func(ctx context.Context) (*Link, Obj, error) + type Link struct { URL string `json:"url"` // most common way Header http.Header `json:"header"` // needed header (for url) @@ -37,6 +41,10 @@ type Link struct { PartSize int `json:"part_size"` ContentLength int64 `json:"content_length"` // 转码视频、缩略图 + // Refresher is a callback to refresh the link when it expires during long downloads + // This field is not serialized and is optional - if nil, no refresh will be attempted + Refresher LinkRefresher `json:"-"` + utils.SyncClosers `json:"-"` // 如果SyncClosers中的资源被关闭后Link将不可用，则此值应为 true RequireReference bool `json:"-"` diff --git a/internal/model/meta.go b/internal/model/meta.go index 0446137a2..a105f38c6 100644 --- a/internal/model/meta.go +++ b/internal/model/meta.go @@ -1,16 +1,20 @@ package model type Meta struct { - ID uint `json:"id" gorm:"primaryKey"` - Path string `json:"path" gorm:"unique" binding:"required"` - Password string `json:"password"` - PSub bool `json:"p_sub"` - Write bool `json:"write"` - WSub bool `json:"w_sub"` - Hide string `json:"hide"` - HSub bool `json:"h_sub"` - Readme string `json:"readme"` - RSub bool `json:"r_sub"` - Header string `json:"header"` - HeaderSub bool `json:"header_sub"` + ID uint `json:"id" gorm:"primaryKey"` + Path string `json:"path" gorm:"unique" binding:"required"` + ReadUsers []uint `json:"read_users" gorm:"serializer:json"` + ReadUsersSub bool `json:"read_users_sub"` + WriteUsers []uint `json:"write_users" gorm:"serializer:json"` + WriteUsersSub bool `json:"write_users_sub"` + Password string `json:"password"` + PSub bool `json:"p_sub"` + Write bool `json:"write"` + WSub bool `json:"w_sub"` + Hide string `json:"hide"` + HSub bool `json:"h_sub"` + Readme string `json:"readme"` + RSub bool `json:"r_sub"` + Header string `json:"header"` + HeaderSub bool `json:"header_sub"` } diff --git a/internal/model/user.go b/internal/model/user.go index 640e3b2e3..61252ce95 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -20,7 +20,15 @@ const ( ADMIN ) -const StaticHashSalt = "https://github.com/alist-org/alist" +const ( + StaticHashSalt = "https://github.com/alist-org/alist" + + InvalidUsernameOrPassword = "Invalid username or password" + Invalid2FACode = "Invalid 2FA code" + TooManyAttempts = "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later." + GuestCannotUpdateProfile = "Guest user can not update profile" + GuestCannotGenerate2FA = "Guest user can not generate 2FA code" +) var LoginCache = cache.NewMemCache[int]() @@ -115,12 +123,12 @@ func (u *User) CanAddOfflineDownloadTasks() bool { return CanAddOfflineDownloadTasks(u.Permission) } -func CanWrite(permission int32) bool { +func CanWriteContent(permission int32) bool { return (permission>>3)&1 == 1 } -func (u *User) CanWrite() bool { - return CanWrite(u.Permission) +func (u *User) CanWriteContent() bool { + return CanWriteContent(u.Permission) } func CanRename(permission int32) bool { diff --git a/internal/net/oss.go b/internal/net/oss.go new file mode 100644 index 000000000..a897161f1 --- /dev/null +++ b/internal/net/oss.go @@ -0,0 +1,9 @@ +package net + +import "github.com/aliyun/aliyun-oss-go-sdk/oss" + +func NewOSSClient(endpoint, accessKeyID, accessKeySecret string, options ...oss.ClientOption) (*oss.Client, error) { + clientOptions := []oss.ClientOption{oss.HTTPClient(NewHttpClient())} + clientOptions = append(clientOptions, options...) + return oss.New(endpoint, accessKeyID, accessKeySecret, clientOptions...) +} diff --git a/internal/net/oss_test.go b/internal/net/oss_test.go new file mode 100644 index 000000000..9001cd39d --- /dev/null +++ b/internal/net/oss_test.go @@ -0,0 +1,54 @@ +package net + +import ( + "net/http" + "net/url" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/conf" +) + +func TestNewOSSClientUsesEnvironmentHTTPSProxy(t *testing.T) { + oldConf := conf.Conf + conf.Conf = conf.DefaultConfig("data") + defer func() { + conf.Conf = oldConf + }() + + t.Setenv("HTTP_PROXY", "") + t.Setenv("http_proxy", "") + t.Setenv("HTTPS_PROXY", "http://127.0.0.1:7890") + t.Setenv("https_proxy", "") + t.Setenv("NO_PROXY", "") + t.Setenv("no_proxy", "") + + client, err := NewOSSClient("https://oss-cn-hangzhou.aliyuncs.com", "test-access-key", "test-access-secret") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if client.HTTPClient == nil { + t.Fatal("expected OSS client to use a custom HTTP client") + } + + transport, ok := client.HTTPClient.Transport.(*http.Transport) + if !ok { + t.Fatalf("expected *http.Transport, got %T", client.HTTPClient.Transport) + } + + if transport.Proxy == nil { + t.Fatal("expected proxy function to be configured") + } + + req := &http.Request{URL: &url.URL{Scheme: "https", Host: "oss-cn-hangzhou.aliyuncs.com"}} + proxyURL, err := transport.Proxy(req) + if err != nil { + t.Fatalf("expected no proxy lookup error, got %v", err) + } + if proxyURL == nil { + t.Fatal("expected HTTPS proxy to be used") + } + if got, want := proxyURL.String(), "http://127.0.0.1:7890"; got != want { + t.Fatalf("expected proxy %q, got %q", want, got) + } +} diff --git a/internal/net/request_test.go b/internal/net/request_test.go index 00ba8a134..da16a3165 100644 --- a/internal/net/request_test.go +++ b/internal/net/request_test.go @@ -153,7 +153,7 @@ func (c *downloadCaptureClient) HttpRequest(ctx context.Context, params *HttpReq c.GetObjectInvocations++ - if ¶ms.Range != nil { + if params.Range.Length != 0 { c.RetrievedRanges = append(c.RetrievedRanges, fmt.Sprintf("%d-%d", params.Range.Start, params.Range.Length)) } diff --git a/internal/net/serve.go b/internal/net/serve.go index 6a20460b1..ee288b86a 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "mime/multipart" + stdnet "net" // 标准库net包，用于Dialer "net/http" "strconv" "strings" @@ -286,12 +287,20 @@ func NewHttpClient() *http.Client { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + // 快速连接超时：10秒建立连接，失败快速重试 + DialContext: (&stdnet.Dialer{ + Timeout: 10 * time.Second, // TCP握手超时 + KeepAlive: 30 * time.Second, // TCP keep-alive + }).DialContext, + // 响应头超时：15秒等待服务器响应头（平衡API调用与下载检测） + ResponseHeaderTimeout: 15 * time.Second, + // 允许长时间读取数据（无 IdleConnTimeout 限制） } SetProxyIfConfigured(transport) return &http.Client{ - Timeout: time.Hour * 48, + Timeout: time.Hour * 48, // 总超时保持48小时（允许大文件慢速下载） Transport: transport, } } diff --git a/internal/offline_download/115_open/client.go b/internal/offline_download/115_open/client.go index d12e02ec5..56669a674 100644 --- a/internal/offline_download/115_open/client.go +++ b/internal/offline_download/115_open/client.go @@ -2,8 +2,16 @@ package _115_open import ( "context" + "encoding/base32" + "encoding/hex" + "errors" "fmt" + "net/url" + "strconv" + "strings" + "time" + sdk "github.com/OpenListTeam/115-sdk-go" _115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/setting" @@ -12,11 +20,34 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" + log "github.com/sirupsen/logrus" ) type Open115 struct { } +type offlineTaskClient interface { + OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) + DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error +} + +type offlineTaskDetailClient interface { + OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) +} + +type offlineTaskLimiter interface { + WaitLimit(ctx context.Context) error +} + +func waitOfflineTaskLimit(ctx context.Context, client offlineTaskClient) error { + limiter, ok := client.(offlineTaskLimiter) + if !ok { + return nil + } + return limiter.WaitLimit(ctx) +} + func (o *Open115) Name() string { return "115 Open" } @@ -68,15 +99,473 @@ func (o *Open115) AddURL(args *tool.AddUrlArgs) (string, error) { if err != nil { return "", err } + log.Infof("[115_open] AddURL start: temp_dir=%q actual_path=%q parent_id=%q parent_name=%q url=%q", args.TempDir, actualPath, parentDir.GetID(), parentDir.GetName(), args.Url) + logOfflineURLDetails("[115_open] AddURL input", args.Url) - hashs, err := driver115Open.OfflineDownload(ctx, []string{args.Url}, parentDir) - if err != nil || len(hashs) < 1 { - return "", fmt.Errorf("failed to add offline download task: %w", err) + hashs, err := addOfflineDownloadTask(ctx, driver115Open, args.Url, parentDir) + if err != nil { + return "", err + } + + if len(hashs) < 1 { + return "", fmt.Errorf("failed to add offline download task: no task hash returned") } return hashs[0], nil } +func addOfflineDownloadTask(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, error) { + parentID, parentName := "", "" + if parentDir != nil { + parentID = parentDir.GetID() + parentName = parentDir.GetName() + } + log.Infof("[115_open] addOfflineDownloadTask: parent_id=%q parent_name=%q url=%q", parentID, parentName, url) + logOfflineURLDetails("[115_open] addOfflineDownloadTask target", url) + if err := preCleanDuplicateOfflineTasks(ctx, client, url); err != nil { + return nil, err + } + hashs, addItems, rawResp, err := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] addOfflineDownloadTask first attempt result: hashes=%v err=%v add_items=%d", hashs, err, len(addItems)) + if err == nil { + return hashs, nil + } + if !isDuplicateOfflineTaskError(err) { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] duplicate offline task detected, trying cleanup before retry") + if rawResp != "" { + log.Infof("[115_open] duplicate add response: %s", rawResp) + } + for _, item := range addItems { + log.Infof("[115_open] duplicate add item: state=%v code=%d info_hash=%q url=%q", item.State, item.Code, item.InfoHash, item.URL) + logOfflineURLDetails("[115_open] duplicate add item url", item.URL) + if item.InfoHash == "" { + log.Infof("[115_open] skipping add-response duplicate item: empty info_hash") + continue + } + if item.URL != "" && !offlineTaskURLMatches(item.URL, url) { + log.Infof("[115_open] skipping add-response duplicate item: url mismatch") + continue + } + log.Infof("[115_open] deleting duplicate task directly from add response: info_hash=%s url=%s", item.InfoHash, item.URL) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + if deleteErr := client.DeleteOfflineTask(ctx, item.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete duplicate task from add response failed: info_hash=%s err=%v", item.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task from add response: %w", deleteErr) + } + log.Infof("[115_open] delete duplicate task from add response success: info_hash=%s", item.InfoHash) + waitForOfflineTaskRemoval(ctx, client, item.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after add-response delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after add-response delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + return nil, fmt.Errorf("failed to add offline download task: %w", err) + } + log.Infof("[115_open] offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] matched duplicate offline task: info_hash=%s, name=%s", task.InfoHash, task.Name) + log.Infof("[115_open] deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, err + } + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return nil, fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + log.Infof("[115_open] delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + hashs, retryItems, retryRawResp, retryErr := offlineDownloadWithDetails(ctx, client, url, parentDir) + log.Infof("[115_open] retry add after matched delete: hashes=%v err=%v add_items=%d", hashs, retryErr, len(retryItems)) + if retryRawResp != "" { + log.Infof("[115_open] retry add raw response after matched delete: %s", retryRawResp) + } + err = retryErr + if err != nil { + return nil, fmt.Errorf("failed to add offline download task after removing duplicate: %w", err) + } + return hashs, nil + } + log.Warnf("[115_open] duplicate offline task detected but no matching task found in offline list") + return nil, fmt.Errorf("failed to add offline download task: %w", err) +} + +func preCleanDuplicateOfflineTasks(ctx context.Context, client offlineTaskClient, url string) error { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } + taskList, listErr := client.OfflineList(ctx) + if listErr != nil || taskList == nil { + log.Warnf("[115_open] pre-add offline list failed: err=%v", listErr) + return nil + } + log.Infof("[115_open] pre-add offline list returned %d tasks across %d pages", len(taskList.Tasks), taskList.PageCount) + deleted := 0 + for _, task := range taskList.Tasks { + matched, reason := offlineTaskMatchReason(task, url) + log.Infof("[115_open] pre-add duplicate candidate: info_hash=%s status=%d size=%d name=%q url=%q matched=%v reason=%s", task.InfoHash, task.Status, task.Size, task.Name, task.URL, matched, reason) + logOfflineURLDetails("[115_open] pre-add duplicate candidate url", task.URL) + if !matched { + continue + } + log.Infof("[115_open] pre-add deleting matched duplicate offline task: info_hash=%s status=%d size=%d", task.InfoHash, task.Status, task.Size) + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return err + } + if deleteErr := client.DeleteOfflineTask(ctx, task.InfoHash, false); deleteErr != nil { + log.Errorf("[115_open] pre-add delete matched duplicate offline task failed: info_hash=%s err=%v", task.InfoHash, deleteErr) + return fmt.Errorf("failed to delete duplicate offline download task: %w", deleteErr) + } + deleted++ + log.Infof("[115_open] pre-add delete matched duplicate offline task success: info_hash=%s", task.InfoHash) + waitForOfflineTaskRemoval(ctx, client, task.InfoHash) + } + if deleted == 0 { + log.Infof("[115_open] pre-add duplicate scan found no matches") + } + return nil +} + +func offlineDownloadWithDetails(ctx context.Context, client offlineTaskClient, url string, parentDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + return nil, nil, "", err + } + if detailClient, ok := client.(offlineTaskDetailClient); ok { + return detailClient.OfflineDownloadWithDetails(ctx, []string{url}, parentDir) + } + hashs, err := client.OfflineDownload(ctx, []string{url}, parentDir) + return hashs, nil, "", err +} + +func isDuplicateOfflineTaskError(err error) bool { + if err == nil { + return false + } + errStr := strings.ToLower(err.Error()) + return strings.Contains(errStr, "10008") || + strings.Contains(errStr, "重复") || + strings.Contains(errStr, "已存在") || + strings.Contains(errStr, "duplicate") +} + +func offlineTaskURLMatches(taskURL string, rawURL string) bool { + taskVariants := normalizedOfflineTaskURLVariants(taskURL) + rawVariants := normalizedOfflineTaskURLVariants(rawURL) + for candidate := range taskVariants { + if _, ok := rawVariants[candidate]; ok { + return true + } + } + return false +} + +func offlineTaskMatches(task sdk.OfflineTask, rawURL string) bool { + matched, _ := offlineTaskMatchReason(task, rawURL) + return matched +} + +func offlineTaskMatchReason(task sdk.OfflineTask, rawURL string) (bool, string) { + if offlineTaskURLMatches(task.URL, rawURL) { + return true, "url variants matched" + } + if httpURLMatches(task.URL, rawURL) { + return true, "http url host+path matched" + } + rawMagnet := parseMagnetBTIH(rawURL) + if rawMagnet != "" { + taskHash := normalizeInfoHash(task.InfoHash) + if taskHash != "" && taskHash == rawMagnet { + return true, "task info_hash matched raw magnet" + } + taskURLHash := parseMagnetBTIH(task.URL) + if taskURLHash != "" && taskURLHash == rawMagnet { + return true, "task url magnet hash matched" + } + return false, fmt.Sprintf("task magnet hash mismatch: task_info_hash=%q task_url_hash=%q raw_hash=%q", taskHash, taskURLHash, rawMagnet) + } + taskED2K, rawED2K := parseED2KLink(task.URL), parseED2KLink(rawURL) + if taskED2K != nil && rawED2K != nil { + if taskED2K.Hash == rawED2K.Hash { + if taskED2K.Size == rawED2K.Size { + return true, "task url ed2k hash matched" + } + return true, "task url ed2k hash matched despite size mismatch" + } + return false, fmt.Sprintf("task url ed2k mismatch: task=%s raw=%s", taskED2K.String(), rawED2K.String()) + } + if rawED2K == nil { + return false, "raw url is not ed2k and url variants did not match" + } + if normalizeOfflineTaskURL(task.InfoHash) == rawED2K.Hash { + if task.Size == rawED2K.Size { + return true, "task info_hash matched raw ed2k" + } + return true, "task info_hash matched raw ed2k despite size mismatch" + } + taskName := normalizeOfflineTaskURL(task.Name) + if taskName == normalizeOfflineTaskURL(rawED2K.Name) && task.Size == rawED2K.Size { + return true, "task name and size matched raw ed2k" + } + return false, fmt.Sprintf("task name/hash/size mismatch: task_name=%q raw_name=%q task_info_hash=%q raw_hash=%q task_size=%d raw_size=%d", taskName, normalizeOfflineTaskURL(rawED2K.Name), normalizeOfflineTaskURL(task.InfoHash), rawED2K.Hash, task.Size, rawED2K.Size) +} + +func normalizedOfflineTaskURLVariants(raw string) map[string]struct{} { + variants := map[string]struct{}{} + queue := []string{raw} + for len(queue) > 0 { + current := normalizeOfflineTaskURL(queue[0]) + queue = queue[1:] + if current == "" { + continue + } + if _, ok := variants[current]; ok { + continue + } + variants[current] = struct{}{} + if decoded, err := url.QueryUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + if decoded, err := url.PathUnescape(current); err == nil && decoded != current { + queue = append(queue, decoded) + } + } + return variants +} + +func httpURLMatches(taskURL, rawURL string) bool { + taskNormalized := normalizeHTTPURL(taskURL) + rawNormalized := normalizeHTTPURL(rawURL) + if taskNormalized == "" || rawNormalized == "" { + return false + } + return taskNormalized == rawNormalized +} + +func normalizeHTTPURL(raw string) string { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed == nil { + return "" + } + scheme := strings.ToLower(parsed.Scheme) + if scheme != "http" && scheme != "https" { + return "" + } + host := strings.ToLower(parsed.Host) + if host == "" { + return "" + } + path := strings.ToLower(parsed.Path) + path = strings.TrimSuffix(path, "/") + if path == "" { + path = "/" + } + return fmt.Sprintf("%s://%s%s", scheme, host, path) +} + +func normalizeOfflineTaskURL(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimSuffix(normalized, "/") + return strings.ToLower(normalized) +} + +type ed2kLink struct { + Name string + Size int64 + Hash string +} + +func parseED2KLink(raw string) *ed2kLink { + normalized := strings.TrimSpace(raw) + if !strings.HasPrefix(strings.ToLower(normalized), "ed2k://|file|") { + return nil + } + parts := strings.Split(normalized, "|") + if len(parts) < 6 { + return nil + } + name, err := url.PathUnescape(parts[2]) + if err != nil { + name = parts[2] + } + size, err := strconv.ParseInt(parts[3], 10, 64) + if err != nil { + return nil + } + return &ed2kLink{ + Name: normalizeOfflineTaskURL(name), + Size: size, + Hash: normalizeOfflineTaskURL(parts[4]), + } +} + +func parseMagnetBTIH(raw string) string { + if raw == "" { + return "" + } + lower := strings.ToLower(raw) + idx := strings.Index(lower, "btih:") + if idx == -1 { + return "" + } + candidate := raw[idx+len("btih:"):] + if candidate == "" { + return "" + } + for i, ch := range candidate { + if ch == '&' || ch == '#' || ch == '/' { + candidate = candidate[:i] + break + } + } + candidate = strings.TrimSpace(candidate) + if candidate == "" { + return "" + } + if decoded, err := url.QueryUnescape(candidate); err == nil { + candidate = decoded + } + return normalizeInfoHash(candidate) +} + +func normalizeInfoHash(raw string) string { + normalized := strings.TrimSpace(raw) + if normalized == "" { + return "" + } + normalized = strings.TrimPrefix(strings.ToLower(normalized), "urn:btih:") + if normalized == "" { + return "" + } + if len(normalized) == 40 && isHexString(normalized) { + return normalized + } + if len(normalized) == 32 && isBase32String(normalized) { + decoded, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(normalized)) + if err == nil && len(decoded) == 20 { + return hex.EncodeToString(decoded) + } + } + return normalized +} + +func isHexString(value string) bool { + for _, ch := range value { + if (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') { + continue + } + return false + } + return true +} + +func isBase32String(value string) bool { + for _, ch := range value { + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '2' && ch <= '7') { + continue + } + return false + } + return true +} + +func (e *ed2kLink) Equal(other *ed2kLink) bool { + if e == nil || other == nil { + return false + } + return e.Name == other.Name && e.Size == other.Size && e.Hash == other.Hash +} + +func (e *ed2kLink) String() string { + if e == nil { + return "" + } + return fmt.Sprintf("name=%q size=%d hash=%q", e.Name, e.Size, e.Hash) +} + +func logOfflineURLDetails(prefix string, raw string) { + if raw == "" { + log.Infof("%s details: raw is empty", prefix) + return + } + variants := normalizedOfflineTaskURLVariants(raw) + log.Infof("%s details: raw=%q normalized_variants=%v", prefix, raw, mapKeys(variants)) + if parsedMagnet := parseMagnetBTIH(raw); parsedMagnet != "" { + log.Infof("%s details: parsed_magnet_hash=%s", prefix, parsedMagnet) + } + if parsed := parseED2KLink(raw); parsed != nil { + log.Infof("%s details: parsed_ed2k=%s", prefix, parsed.String()) + } +} + +func mapKeys(values map[string]struct{}) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + return keys +} + +func waitForOfflineTaskRemoval(ctx context.Context, client offlineTaskClient, infoHash string) { + const maxChecks = 3 + for attempt := 1; attempt <= maxChecks; attempt++ { + if err := waitOfflineTaskLimit(ctx, client); err != nil { + log.Warnf("[115_open] post-delete wait limit failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } + taskList, err := client.OfflineList(ctx) + if err != nil { + log.Warnf("[115_open] post-delete check failed: info_hash=%s attempt=%d err=%v", infoHash, attempt, err) + return + } + stillExists := false + taskStatus := -999 + taskName := "" + for _, task := range taskList.Tasks { + if normalizeOfflineTaskURL(task.InfoHash) != normalizeOfflineTaskURL(infoHash) { + continue + } + stillExists = true + taskStatus = task.Status + taskName = task.Name + break + } + log.Infof("[115_open] post-delete check: info_hash=%s attempt=%d exists=%v status=%d name=%q task_count=%d", infoHash, attempt, stillExists, taskStatus, taskName, len(taskList.Tasks)) + if !stillExists { + return + } + if attempt < maxChecks { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + } + } + } +} + func (o *Open115) Remove(task *tool.DownloadTask) error { storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { @@ -124,13 +613,15 @@ func (o *Open115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Completed = t.IsDone() s.TotalBytes = t.Size if t.IsFailed() { - s.Err = fmt.Errorf(t.GetStatus()) + s.Err = errors.New(t.GetStatus()) } return s, nil } } - s.Err = fmt.Errorf("the task has been deleted") - return nil, nil + // 任务不在列表中，可能已完成或被删除 + s.Progress = 100 + s.Completed = true + return s, nil } var _ tool.Tool = (*Open115)(nil) diff --git a/internal/offline_download/115_open/client_test.go b/internal/offline_download/115_open/client_test.go new file mode 100644 index 000000000..2e8b94778 --- /dev/null +++ b/internal/offline_download/115_open/client_test.go @@ -0,0 +1,616 @@ +package _115_open + +import ( + "context" + "fmt" + "strings" + "testing" + + sdk "github.com/OpenListTeam/115-sdk-go" + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +type mockOfflineTaskClient struct { + offlineDownloadFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) + offlineDownloadWithDetailsFunc func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) + offlineListFunc func(ctx context.Context) (*sdk.OfflineTaskListResp, error) + deleteOfflineFunc func(ctx context.Context, infoHash string, deleteFiles bool) error + waitLimitFunc func(ctx context.Context) error + waitLimitCalls int +} + +func (m *mockOfflineTaskClient) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return m.offlineDownloadFunc(ctx, uris, dstDir) +} + +func (m *mockOfflineTaskClient) OfflineDownloadWithDetails(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + if m.offlineDownloadWithDetailsFunc == nil { + hashes, err := m.OfflineDownload(ctx, uris, dstDir) + return hashes, nil, "", err + } + return m.offlineDownloadWithDetailsFunc(ctx, uris, dstDir) +} + +func (m *mockOfflineTaskClient) OfflineList(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return m.offlineListFunc(ctx) +} + +func (m *mockOfflineTaskClient) DeleteOfflineTask(ctx context.Context, infoHash string, deleteFiles bool) error { + return m.deleteOfflineFunc(ctx, infoHash, deleteFiles) +} + +func (m *mockOfflineTaskClient) WaitLimit(ctx context.Context) error { + m.waitLimitCalls++ + if m.waitLimitFunc != nil { + return m.waitLimitFunc(ctx) + } + return nil +} + +func TestIsDuplicateOfflineTaskError(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "code 10008", err: fmt.Errorf("code: 10008"), want: true}, + {name: "chinese duplicate", err: fmt.Errorf("任务重复"), want: true}, + {name: "already exists", err: fmt.Errorf("任务已存在"), want: true}, + {name: "english duplicate", err: fmt.Errorf("duplicate task"), want: true}, + {name: "other", err: fmt.Errorf("network timeout"), want: false}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := isDuplicateOfflineTaskError(tc.err); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } +} + +func TestOfflineTaskURLMatches(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + taskURL string + rawURL string + want bool + }{ + { + name: "exact match", + taskURL: "ed2k://|file|test.avi|123|ABC|/", + rawURL: "ed2k://|file|test.avi|123|ABC|/", + want: true, + }, + { + name: "percent encoded file name", + taskURL: "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + rawURL: "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/", + want: true, + }, + { + name: "case and trailing slash normalized", + taskURL: "ED2K://|FILE|TEST.AVI|123|ABC|", + rawURL: "ed2k://|file|test.avi|123|abc|/", + want: true, + }, + { + name: "different link", + taskURL: "ed2k://|file|a.avi|123|ABC|/", + rawURL: "ed2k://|file|b.avi|123|ABC|/", + want: false, + }, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := offlineTaskURLMatches(tc.taskURL, tc.rawURL); got != tc.want { + t.Fatalf("want %v, got %v", tc.want, got) + } + }) + } +} + +func TestOfflineTaskMatches(t *testing.T) { + t.Parallel() + + t.Run("match ed2k by parsed fields when task url differs", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + InfoHash: "server-task-hash", + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected task to match by ed2k parsed fields") + } + }) + + t.Run("do not match different ed2k size", func(t *testing.T) { + t.Parallel() + + task := sdk.OfflineTask{ + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1, + } + rawURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + if offlineTaskMatches(task, rawURL) { + t.Fatal("expected task not to match") + } + }) + + t.Run("magnet still matches by url", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: rawURL, + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by url") + } + }) + + t.Run("match magnet by btih despite noisy tracker", func(t *testing.T) { + t.Parallel() + + rawURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + task := sdk.OfflineTask{ + InfoHash: "1234567890abcdef1234567890abcdef12345678", + URL: "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test&tr=%3C!DOCTYPE%20html%3E", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected magnet task to match by btih") + } + }) + + t.Run("match http by host and path", func(t *testing.T) { + t.Parallel() + + rawURL := "https://example.com/files/test.mp4" + task := sdk.OfflineTask{ + URL: "https://EXAMPLE.com/files/test.mp4?token=abc", + } + + if !offlineTaskMatches(task, rawURL) { + t.Fatal("expected http task to match by host and path") + } + }) +} + +func TestAddOfflineDownloadTask(t *testing.T) { + t.Parallel() + + const ( + testURL = "https://example.com/test.torrent" + firstHash = "hash-1" + staleHash = "hash-stale" + deleteError = "delete failed" + ) + + t.Run("success on first try", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("wait limit applied for add flow", func(t *testing.T) { + t.Parallel() + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return nil + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if client.waitLimitCalls < 2 { + t.Fatalf("want wait limit calls >= 2, got %d", client.waitLimitCalls) + } + }) + + t.Run("delete duplicate and retry", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: testURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate magnet and retry", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + magnetURL := "magnet:?xt=urn:btih:1234567890ABCDEF1234567890ABCDEF12345678&dn=test" + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: magnetURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, magnetURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry with decoded ed2k url", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + decodedURL := "ed2k://|file|[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: decodedURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate and retry with empty task url but matching name and size", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + encodedURL := "ed2k://|file|[AVS]Azumi%20Mizushima%20[ネオパンストフェティッシュ%20Ver.19%20水嶋あずみ](NOP-019)(2011.01.13).avi|1593601796|9E5CCC55541BD46EE8252BF100EFC46D|/" + + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + callCount++ + if callCount == 1 { + return nil, fmt.Errorf("code: 10008, message: 任务已存在，请勿输入重复的链接地址") + } + return []string{firstHash}, nil + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + { + InfoHash: staleHash, + Name: "[AVS]Azumi Mizushima [ネオパンストフェティッシュ Ver.19 水嶋あずみ](NOP-019)(2011.01.13).avi", + Size: 1593601796, + URL: "", + }, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, encodedURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 2 { + t.Fatalf("want 2 delete attempts (pre-add + duplicate), got %d", deleteCount) + } + if listCount < 2 { + t.Fatalf("want at least 2 offline list calls, got %d", listCount) + } + }) + + t.Run("delete duplicate directly from add response info hash", func(t *testing.T) { + t.Parallel() + + callCount := 0 + deleteCount := 0 + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadWithDetailsFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, []sdk.AddOfflineTaskURIsResp, string, error) { + callCount++ + if callCount == 1 { + return nil, []sdk.AddOfflineTaskURIsResp{ + {InfoHash: staleHash, URL: testURL}, + }, `{"state":false,"code":10008,"message":"任务已存在","data":[{"info_hash":"hash-stale","url":"` + testURL + `"}]}`, fmt.Errorf("code: 10008, message: 任务已存在") + } + return []string{firstHash}, nil, "", nil + }, + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("unexpected fallback OfflineDownload call") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + deleteCount++ + if infoHash != staleHash { + t.Fatalf("unexpected hash: %s", infoHash) + } + if deleteFiles { + t.Fatal("deleteFiles should be false") + } + return nil + }, + } + + hashes, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(hashes) != 1 || hashes[0] != firstHash { + t.Fatalf("unexpected hashes: %+v", hashes) + } + if callCount != 2 { + t.Fatalf("want 2 download attempts, got %d", callCount) + } + if deleteCount != 1 { + t.Fatalf("want 1 delete attempt, got %d", deleteCount) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("duplicate delete failure", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("duplicate task") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{ + Tasks: []sdk.OfflineTask{ + {InfoHash: staleHash, URL: testURL}, + }, + }, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + return fmt.Errorf(deleteError) + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), deleteError) { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) + + t.Run("non duplicate error is returned", func(t *testing.T) { + t.Parallel() + + listCount := 0 + client := &mockOfflineTaskClient{ + offlineDownloadFunc: func(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return nil, fmt.Errorf("network timeout") + }, + offlineListFunc: func(ctx context.Context) (*sdk.OfflineTaskListResp, error) { + listCount++ + return &sdk.OfflineTaskListResp{Tasks: nil}, nil + }, + deleteOfflineFunc: func(ctx context.Context, infoHash string, deleteFiles bool) error { + t.Fatal("DeleteOfflineTask should not be called") + return nil + }, + } + + _, err := addOfflineDownloadTask(context.Background(), client, testURL, nil) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "network timeout") { + t.Fatalf("unexpected error: %v", err) + } + if listCount < 1 { + t.Fatalf("want pre-add offline list call, got %d", listCount) + } + }) +} + +func TestOpen115BasicMethods(t *testing.T) { + t.Parallel() + + o := &Open115{} + + if o.Name() != "115 Open" { + t.Fatalf("unexpected name: %s", o.Name()) + } + if o.Items() != nil { + t.Fatal("Items should return nil") + } + msg, err := o.Init() + if err != nil { + t.Fatalf("unexpected init error: %v", err) + } + if msg != "ok" { + t.Fatalf("unexpected init message: %s", msg) + } +} diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 50a4f6343..477c26a51 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -32,6 +32,8 @@ type DownloadTask struct { callStatusRetried int } +var completedOfflineTaskCleanupDelay = time.Second + func (t *DownloadTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) @@ -97,16 +99,7 @@ outer: if t.tool.Name() == "ThunderX" { return nil } - if t.tool.Name() == "115 Cloud" { - // hack for 115 - <-time.After(time.Second * 1) - err := t.tool.Remove(t) - if err != nil { - log.Errorln(err.Error()) - } - return nil - } - if t.tool.Name() == "115 Open" { + if t.tool.Name() == "115 Cloud" || t.tool.Name() == "115 Open" { return nil } if t.tool.Name() == "123 Open" { @@ -147,11 +140,11 @@ func (t *DownloadTask) Update() (bool, error) { if err != nil { t.callStatusRetried++ log.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) + if t.callStatusRetried > 10 { + return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) + } return false, nil } - if t.callStatusRetried > 5 { - return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) - } t.callStatusRetried = 0 t.SetProgress(info.Progress) t.SetTotalBytes(info.TotalBytes) @@ -163,6 +156,13 @@ func (t *DownloadTask) Update() (bool, error) { } // if download completed if info.Completed { + // For 115, remove offline task record before transfer so it gets cleaned up even if transfer fails + if t.tool.Name() == "115 Cloud" || t.tool.Name() == "115 Open" { + <-time.After(completedOfflineTaskCleanupDelay) + if removeErr := t.tool.Remove(t); removeErr != nil { + log.Errorln(removeErr.Error()) + } + } err := t.Transfer() return true, errors.WithMessage(err, "failed to transfer file") } diff --git a/internal/offline_download/tool/download_test.go b/internal/offline_download/tool/download_test.go new file mode 100644 index 000000000..5303d35aa --- /dev/null +++ b/internal/offline_download/tool/download_test.go @@ -0,0 +1,90 @@ +package tool + +import ( + "context" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/errs" + "github.com/OpenListTeam/OpenList/v4/internal/model" + task2 "github.com/OpenListTeam/OpenList/v4/internal/task" +) + +type mockTool struct { + name string + addURLFunc func(args *AddUrlArgs) (string, error) + removeFunc func(task *DownloadTask) error + statusFunc func(task *DownloadTask) (*Status, error) + runFunc func(task *DownloadTask) error +} + +func (m *mockTool) Name() string { return m.name } + +func (m *mockTool) Items() []model.SettingItem { return nil } + +func (m *mockTool) Init() (string, error) { return "ok", nil } + +func (m *mockTool) IsReady() bool { return true } + +func (m *mockTool) AddURL(args *AddUrlArgs) (string, error) { + return m.addURLFunc(args) +} + +func (m *mockTool) Remove(task *DownloadTask) error { + return m.removeFunc(task) +} + +func (m *mockTool) Status(task *DownloadTask) (*Status, error) { + return m.statusFunc(task) +} + +func (m *mockTool) Run(task *DownloadTask) error { + return m.runFunc(task) +} + +func TestDownloadTaskRun_RemovesCompleted115OpenRecord(t *testing.T) { + previousDelay := completedOfflineTaskCleanupDelay + completedOfflineTaskCleanupDelay = 0 + defer func() { + completedOfflineTaskCleanupDelay = previousDelay + }() + + removeCount := 0 + tool := &mockTool{ + name: "115 Open", + addURLFunc: func(args *AddUrlArgs) (string, error) { + return "gid-1", nil + }, + removeFunc: func(task *DownloadTask) error { + removeCount++ + if task.GID != "gid-1" { + t.Fatalf("unexpected gid: %s", task.GID) + } + return nil + }, + statusFunc: func(task *DownloadTask) (*Status, error) { + return &Status{ + Completed: true, + Status: "completed", + }, nil + }, + runFunc: func(task *DownloadTask) error { + return errs.NotSupport + }, + } + + task := &DownloadTask{ + TaskExtension: task2.TaskExtension{}, + Url: "https://example.com/test.torrent", + DstDirPath: "/115", + TempDir: "/115", + tool: tool, + } + task.SetCtx(context.Background()) + + if err := task.Run(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if removeCount != 1 { + t.Fatalf("want 1 cleanup remove, got %d", removeCount) + } +} diff --git a/internal/op/fs.go b/internal/op/fs.go index 5116bbef5..534e73697 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -150,6 +150,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, IsFolder: true, Mask: model.Locked, + HashInfo: utils.NewHashInfo(nil, ""), }, nil case driver.IRootPath: return &model.Object{ @@ -158,6 +159,7 @@ func Get(ctx context.Context, storage driver.Driver, path string, excludeTempObj Modified: storage.GetStorage().Modified, Mask: model.Locked, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), }, nil } return nil, errors.New("please implement GetRooter or IRootPath or IRootId interface") @@ -247,6 +249,8 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li ol.link.SyncClosers.AcquireReference() || !ol.link.RequireReference { return ol.link, ol.obj, nil } + // SyncClosers 已关闭（文件句柄已关闭），删除缓存条目，重新获取 + Cache.linkCache.DeleteKey(key) } fn := func() (*objWithLink, error) { @@ -262,11 +266,37 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li if err != nil { return nil, errors.Wrapf(err, "failed get link") } + + // Set up link refresher for automatic refresh on expiry during long downloads + // This enables all download scenarios to handle link expiration gracefully + if link.Refresher == nil { + storageCopy := storage + pathCopy := path + argsCopy := args + link.Refresher = func(refreshCtx context.Context) (*model.Link, model.Obj, error) { + log.Infof("Refreshing download link for: %s", pathCopy) + // Get fresh link directly from storage, bypassing cache + file, err := GetUnwrap(refreshCtx, storageCopy, pathCopy) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file for refresh") + } + newLink, err := storageCopy.Link(refreshCtx, file, argsCopy) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to refresh link") + } + return newLink, file, nil + } + } + ol := &objWithLink{link: link, obj: file} if link.Expiration != nil { Cache.linkCache.SetTypeWithTTL(key, typeKey, ol, *link.Expiration) - } else { + } else if link.RequireReference { + // 本地文件等需要引用计数的链接，缓存与文件句柄生命周期绑定 Cache.linkCache.SetTypeWithExpirable(key, typeKey, ol, &link.SyncClosers) + } else { + // 不需要引用计数（如云盘链接无过期时间），使用默认 TTL，多客户端复用 + Cache.linkCache.SetType(key, typeKey, ol) } return ol, nil } @@ -326,9 +356,16 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { return nil, errors.WithMessagef(err, "failed to make parent dir [%s]", parentPath) } parentDir, err := GetUnwrap(ctx, storage, parentPath) - // this should not happen if err != nil { - return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + if errs.IsObjectNotFound(err) { + // Retry once after a short delay (handles cloud storage API sync delay) + log.Debugf("[op] parent dir [%s] not found immediately after creation, retrying...", parentPath) + time.Sleep(100 * time.Millisecond) + parentDir, err = GetUnwrap(ctx, storage, parentPath) + } + if err != nil { + return nil, errors.WithMessagef(err, "failed to get parent dir [%s]", parentPath) + } } if model.ObjHasMask(parentDir, model.NoWrite) { return nil, errors.WithStack(errs.PermissionDenied) @@ -358,6 +395,7 @@ func MakeDir(ctx context.Context, storage driver.Driver, path string) error { Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } dirCache.UpdateObject("", wrapObjName(storage, newObj)) @@ -682,6 +720,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod Modified: file.ModTime(), Ctime: file.CreateTime(), Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) @@ -750,6 +789,7 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url Modified: t, Ctime: t, Mask: model.Temp, + HashInfo: utils.NewHashInfo(nil, ""), } } newObj = wrapObjName(storage, newObj) diff --git a/internal/op/meta.go b/internal/op/meta.go index ed9e422a9..b7d867307 100644 --- a/internal/op/meta.go +++ b/internal/op/meta.go @@ -78,6 +78,7 @@ func UpdateMeta(u *model.Meta) error { return err } metaCache.Del(old.Path) + metaCache.Del(u.Path) return db.UpdateMeta(u) } diff --git a/internal/op/storage.go b/internal/op/storage.go index da4c84e31..2e93bf569 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -368,7 +368,9 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, }(d) select { case r := <-resultChan: - ret.StorageDetails = r + if r != nil { + ret.StorageDetails = r + } case <-time.After(time.Second): } return ret @@ -419,6 +421,7 @@ func getStorageVirtualFilesByPath(prefix string, rootCallback func(driver.Driver Name: name, Modified: v.GetStorage().Modified, IsFolder: true, + HashInfo: utils.NewHashInfo(nil, ""), } if !found { idx := len(files) diff --git a/internal/op/storage_test.go b/internal/op/storage_test.go index 2b191bd56..d7db25040 100644 --- a/internal/op/storage_test.go +++ b/internal/op/storage_test.go @@ -10,7 +10,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/pkg/utils" mapset "github.com/deckarep/golang-set/v2" - "gorm.io/driver/sqlite" + "github.com/glebarez/sqlite" "gorm.io/gorm" ) diff --git a/internal/stream/section_reader_prefetch_test.go b/internal/stream/section_reader_prefetch_test.go new file mode 100644 index 000000000..c4a4fb66f --- /dev/null +++ b/internal/stream/section_reader_prefetch_test.go @@ -0,0 +1,320 @@ +package stream + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + "time" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestDirectSectionReader_SeekablePrefetchPipeline(t *testing.T) { + data := []byte("abcdefgh") + + var ( + mu sync.Mutex + calls []http_range.Range + ) + secondStarted := make(chan struct{}) + releaseSecond := make(chan struct{}) + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + mu.Lock() + calls = append(calls, r) + mu.Unlock() + + if r.Start == 4 { + select { + case <-secondStarted: + default: + close(secondStarted) + } + <-releaseSecond + } + + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, context.Background(), data, rr) + ss, err := NewStreamSectionReader(seekable, 4, nil) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + b1, err := io.ReadAll(rd1) + if err != nil { + t.Fatalf("ReadAll first chunk error = %v", err) + } + if !bytes.Equal(b1, data[0:4]) { + t.Fatalf("first chunk = %q, want %q", b1, data[0:4]) + } + ss.FreeSectionReader(rd1) + + select { + case <-secondStarted: + case <-time.After(2 * time.Second): + t.Fatalf("prefetch for second chunk did not start asynchronously") + } + + type secondResult struct { + data []byte + err error + } + resCh := make(chan secondResult, 1) + go func() { + rd2, err := ss.GetSectionReader(4, 4) + if err != nil { + resCh <- secondResult{err: err} + return + } + b2, err := io.ReadAll(rd2) + ss.FreeSectionReader(rd2) + resCh <- secondResult{data: b2, err: err} + }() + + select { + case res := <-resCh: + t.Fatalf("second GetSectionReader returned too early: err=%v data=%q", res.err, res.data) + case <-time.After(100 * time.Millisecond): + // expected: wait prefetch completion + } + + close(releaseSecond) + res := <-resCh + if res.err != nil { + t.Fatalf("GetSectionReader(4,4) error = %v", res.err) + } + if !bytes.Equal(res.data, data[4:8]) { + t.Fatalf("second chunk = %q, want %q", res.data, data[4:8]) + } + + mu.Lock() + defer mu.Unlock() + if len(calls) != 2 { + t.Fatalf("RangeRead call count = %d, want 2", len(calls)) + } + if calls[0].Start != 0 || calls[1].Start != 4 { + t.Fatalf("RangeRead starts = [%d, %d], want [0, 4]", calls[0].Start, calls[1].Start) + } +} + +func TestDirectSectionReader_SeekablePrefetchMissFallsBackToSyncRead(t *testing.T) { + data := []byte("abcdefghijkl") + + var ( + mu sync.Mutex + calls []http_range.Range + ) + prefetchStarted := make(chan struct{}) + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + mu.Lock() + calls = append(calls, r) + mu.Unlock() + + if r.Start == 4 { + select { + case <-prefetchStarted: + default: + close(prefetchStarted) + } + } + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, context.Background(), data, rr) + ss, err := NewStreamSectionReader(seekable, 4, nil) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + _, _ = io.ReadAll(rd1) + ss.FreeSectionReader(rd1) + + select { + case <-prefetchStarted: + case <-time.After(2 * time.Second): + t.Fatalf("prefetch for second chunk did not start") + } + + rd3, err := ss.GetSectionReader(8, 4) + if err != nil { + t.Fatalf("GetSectionReader(8,4) error = %v", err) + } + b3, err := io.ReadAll(rd3) + if err != nil { + t.Fatalf("ReadAll third chunk error = %v", err) + } + ss.FreeSectionReader(rd3) + if !bytes.Equal(b3, data[8:12]) { + t.Fatalf("third chunk = %q, want %q", b3, data[8:12]) + } + + mu.Lock() + defer mu.Unlock() + if len(calls) != 3 { + t.Fatalf("RangeRead call count = %d, want 3", len(calls)) + } + starts := []int64{calls[0].Start, calls[1].Start, calls[2].Start} + if !((starts[0] == 0 && starts[1] == 4 && starts[2] == 8) || (starts[0] == 0 && starts[1] == 8 && starts[2] == 4)) { + t.Fatalf("unexpected RangeRead starts sequence: %v", starts) + } +} + +func TestDirectSectionReader_SeekablePrefetchDoesNotReportProgress(t *testing.T) { + data := []byte("abcdefgh") + + var progressCalls int + up := model.UpdateProgress(func(_ float64) { + progressCalls++ + }) + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, context.Background(), data, rr) + ss, err := NewStreamSectionReader(seekable, 4, &up) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + _, _ = io.ReadAll(rd1) + ss.FreeSectionReader(rd1) + + rd2, err := ss.GetSectionReader(4, 4) + if err != nil { + t.Fatalf("GetSectionReader(4,4) error = %v", err) + } + _, _ = io.ReadAll(rd2) + ss.FreeSectionReader(rd2) + + if progressCalls != 0 { + t.Fatalf("progress callback called %d times, want 0", progressCalls) + } +} + +func TestDirectSectionReader_SeekablePrefetchFailureIsObservable(t *testing.T) { + data := []byte("abcdefgh") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + started := make(chan struct{}) + var onceStarted sync.Once + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + if r.Start == 4 { + onceStarted.Do(func() { close(started) }) + <-ctx.Done() + return nil, ctx.Err() + } + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, ctx, data, rr) + ss, err := NewStreamSectionReader(seekable, 4, nil) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + _, _ = io.ReadAll(rd1) + ss.FreeSectionReader(rd1) + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatalf("prefetch did not start") + } + + cancel() + _, err = ss.GetSectionReader(4, 4) + if err == nil { + t.Fatalf("GetSectionReader(4,4) expected error, got nil") + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context canceled error, got: %v", err) + } +} + +func TestDirectSectionReader_SeekablePrefetchStopsOnContextCancel(t *testing.T) { + data := []byte("abcdefgh") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + started := make(chan struct{}) + exited := make(chan struct{}) + var onceStarted, onceExited sync.Once + + rr := RangeReaderFunc(func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { + if r.Start == 4 { + onceStarted.Do(func() { close(started) }) + <-ctx.Done() + onceExited.Do(func() { close(exited) }) + } + return io.NopCloser(bytes.NewReader(sliceForRange(data, r))), nil + }) + + seekable := newSeekableStreamForSectionTest(t, ctx, data, rr) + ss, err := NewStreamSectionReader(seekable, 4, nil) + if err != nil { + t.Fatalf("NewStreamSectionReader() error = %v", err) + } + + rd1, err := ss.GetSectionReader(0, 4) + if err != nil { + t.Fatalf("GetSectionReader(0,4) error = %v", err) + } + _, _ = io.ReadAll(rd1) + ss.FreeSectionReader(rd1) + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatalf("prefetch did not start") + } + + cancel() + + select { + case <-exited: + // pass: 预读协程在 ctx cancel 后退出 + case <-time.After(2 * time.Second): + t.Fatalf("prefetch goroutine did not exit after context cancel") + } +} + +func newSeekableStreamForSectionTest(t *testing.T, ctx context.Context, data []byte, rr model.RangeReaderIF) *SeekableStream { + t.Helper() + obj := &model.Object{Name: "section-test.bin", Size: int64(len(data))} + fs := &FileStream{Ctx: ctx, Obj: obj} + link := &model.Link{RangeReader: rr, ContentLength: int64(len(data))} + ss, err := NewSeekableStream(fs, link) + if err != nil { + t.Fatalf("NewSeekableStream() error = %v", err) + } + t.Cleanup(func() { + _ = ss.Close() + }) + return ss +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 4c8238100..7eec75dd9 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -211,7 +211,9 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { return io.NewSectionReader(f.GetFile(), httpRange.Start, httpRange.Length), nil } - cache, err := f.cache(httpRange.Start + httpRange.Length) + // 限制缓存大小，避免累积缓存整个文件 + maxCache := min(httpRange.Start+httpRange.Length, int64(conf.MaxBufferLimit)) + cache, err := f.cache(maxCache) if err != nil { return nil, err } @@ -224,31 +226,13 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { // 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大 // 确保指定大小的数据被缓存 +// 注意：此方法只缓存到 maxCacheSize，不会缓存整个文件 func (f *FileStream) cache(maxCacheSize int64) (model.File, error) { + // 限制缓存大小，避免超大文件占用过多资源 + // 如果需要缓存整个文件，应该显式调用 CacheFullAndWriter if maxCacheSize > int64(conf.MaxBufferLimit) { - size := f.GetSize() - reader := f.Reader - if f.peekBuff != nil { - size -= f.peekBuff.Size() - reader = f.oriReader - } - tmpF, err := utils.CreateTempFile(reader, size) - if err != nil { - return nil, err - } - f.Add(utils.CloseFunc(func() error { - return errors.Join(tmpF.Close(), os.RemoveAll(tmpF.Name())) - })) - if f.peekBuff != nil { - peekF, err := buffer.NewPeekFile(f.peekBuff, tmpF) - if err != nil { - return nil, err - } - f.Reader = peekF - return peekF, nil - } - f.Reader = tmpF - return tmpF, nil + // 不再创建整个文件的临时文件，只缓存到 MaxBufferLimit + maxCacheSize = int64(conf.MaxBufferLimit) } if f.peekBuff == nil { @@ -315,15 +299,9 @@ func NewSeekableStream(fs *FileStream, link *model.Link) (*SeekableStream, error if err != nil { return nil, err } - if _, ok := rr.(*model.FileRangeReader); ok { - var rc io.ReadCloser - rc, err = rr.RangeRead(fs.Ctx, http_range.Range{Length: -1}) - if err != nil { - return nil, err - } - fs.Reader = rc - fs.Add(rc) - } + // IMPORTANT: Do NOT create Reader early for FileRangeReader! + // Let generateReader() create it on-demand when actually needed for reading + // This prevents the Reader from being consumed by intermediate operations like hash calculation fs.size = size fs.Add(link) return &SeekableStream{FileStream: fs, rangeReader: rr}, nil diff --git a/internal/stream/util.go b/internal/stream/util.go index 6aa3dda5d..9c6c6357f 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -9,6 +9,9 @@ import ( "io" "net/http" "os" + "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/errs" @@ -21,13 +24,298 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + // 链接刷新相关常量 + MAX_LINK_REFRESH_COUNT = 50 // 下载链接最大刷新次数（支持长时间传输） + + // RangeRead 重试相关常量 + MAX_RANGE_READ_RETRY_COUNT = 5 // RangeRead 最大重试次数（从3增加到5） +) + type RangeReaderFunc func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) func (f RangeReaderFunc) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { return f(ctx, httpRange) } +// IsLinkExpiredError checks if the error indicates an expired download link +func IsLinkExpiredError(err error) bool { + if err == nil { + return false + } + + // Don't treat context cancellation as link expiration + // This happens when user pauses/seeks video or cancels download + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + + errStr := strings.ToLower(err.Error()) + + // Common expired link error keywords + expiredKeywords := []string{ + "expired", "invalid signature", "token expired", + "access denied", "forbidden", "unauthorized", + "link has expired", "url expired", "request has expired", + "signature expired", "accessdenied", "invalidtoken", + } + for _, keyword := range expiredKeywords { + if strings.Contains(errStr, keyword) { + return true + } + } + + // Check for HTTP status codes that typically indicate expired links + if statusErr, ok := errs.UnwrapOrSelf(err).(net.HttpStatusCodeError); ok { + code := int(statusErr) + // All 4xx client errors may indicate expired/invalid links + // 400 Bad Request, 401 Unauthorized, 403 Forbidden, 404 Not Found, 410 Gone, etc. + if code >= 400 && code < 500 { + return true + } + } + + return false +} + +// RefreshableRangeReader wraps a RangeReader with link refresh capability +type RefreshableRangeReader struct { + link *model.Link + size int64 + innerReader model.RangeReaderIF + mu sync.Mutex + refreshCount int // track refresh count to avoid infinite loops +} + +// NewRefreshableRangeReader creates a new RefreshableRangeReader +func NewRefreshableRangeReader(link *model.Link, size int64) *RefreshableRangeReader { + return &RefreshableRangeReader{ + link: link, + size: size, + } +} + +func (r *RefreshableRangeReader) getInnerReader() (model.RangeReaderIF, error) { + if r.innerReader != nil { + return r.innerReader, nil + } + + // Create inner reader without Refresher to avoid recursion + linkCopy := *r.link + linkCopy.Refresher = nil + + reader, err := GetRangeReaderFromLink(r.size, &linkCopy) + if err != nil { + return nil, err + } + r.innerReader = reader + return reader, nil +} + +func (r *RefreshableRangeReader) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + reader, err := r.getInnerReader() + r.mu.Unlock() + if err != nil { + return nil, err + } + + rc, err := reader.RangeRead(ctx, httpRange) + if err != nil { + // Check if we should try to refresh on initial connection error + if IsLinkExpiredError(err) && r.link.Refresher != nil { + rc, err = r.refreshAndRetry(ctx, httpRange) + } + if err != nil { + return nil, err + } + } + + // Wrap the ReadCloser with self-healing capability to detect 0-byte reads + // This handles cases where cloud providers return 200 OK but empty body for expired links + return &selfHealingReadCloser{ + ReadCloser: rc, + refresher: r, + ctx: ctx, + httpRange: httpRange, + firstRead: false, + closed: false, + }, nil +} + +func (r *RefreshableRangeReader) refreshAndRetry(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if err := r.doRefreshLocked(ctx); err != nil { + return nil, err + } + + reader, err := r.getInnerReader() + if err != nil { + return nil, err + } + return reader.RangeRead(ctx, httpRange) +} + +// doRefreshLocked 执行实际的刷新逻辑（需要持有锁） +func (r *RefreshableRangeReader) doRefreshLocked(ctx context.Context) error { + if r.refreshCount >= MAX_LINK_REFRESH_COUNT { + return fmt.Errorf("max refresh attempts (%d) reached", MAX_LINK_REFRESH_COUNT) + } + + log.Infof("Link expired, attempting to refresh...") + // Use independent context for refresh to prevent cancellation from affecting link refresh + refreshCtx := context.WithoutCancel(ctx) + newLink, _, refreshErr := r.link.Refresher(refreshCtx) + if refreshErr != nil { + return fmt.Errorf("failed to refresh link: %w", refreshErr) + } + + newLink.Refresher = r.link.Refresher + r.link = newLink + r.innerReader = nil + r.refreshCount++ + + log.Infof("Link refreshed successfully") + return nil +} + +// selfHealingReadCloser wraps an io.ReadCloser and automatically refreshes the link +// if the upstream reader dies before the requested range is fully delivered. +type selfHealingReadCloser struct { + io.ReadCloser + refresher *RefreshableRangeReader + ctx context.Context + httpRange http_range.Range + firstRead bool + bytesRead int64 + closed bool + mu sync.Mutex +} + +func (s *selfHealingReadCloser) Read(p []byte) (n int, err error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return 0, errors.New("read from closed reader") + } + + n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + wasFirstRead := !s.firstRead + s.firstRead = true + + // Detect 0-byte read on first attempt (indicates link may be expired but returned 200 OK) + if s.shouldReconnectAfterRead(wasFirstRead, n, err) { + if reconnectErr := s.reconnectFromCurrentOffsetLocked(); reconnectErr != nil { + log.Errorf("Failed to refresh link after interrupted read: %v", reconnectErr) + return n, err + } + + if n > 0 { + return n, nil + } + + n, err = s.ReadCloser.Read(p) + s.bytesRead += int64(n) + return n, err + } + + return n, err +} + +func (s *selfHealingReadCloser) shouldReconnectAfterRead(wasFirstRead bool, n int, err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + if s.remainingBytes() <= 0 { + return false + } + + if wasFirstRead && n == 0 && (err == io.EOF || err == io.ErrUnexpectedEOF) { + log.Warnf("Detected 0-byte read on first attempt, attempting to refresh link...") + return true + } + + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + log.Warnf("Detected interrupted read after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } + + errStr := strings.ToLower(err.Error()) + if strings.Contains(errStr, "connection reset by peer") { + log.Warnf("Detected upstream connection reset after %d bytes, attempting to refresh link...", s.bytesRead) + return true + } + + return false +} + +func (s *selfHealingReadCloser) reconnectFromCurrentOffsetLocked() error { + nextRange := s.httpRange + nextRange.Start += s.bytesRead + if nextRange.Length >= 0 { + nextRange.Length -= s.bytesRead + } + + s.refresher.mu.Lock() + refreshErr := s.refresher.doRefreshLocked(s.ctx) + if refreshErr != nil { + s.refresher.mu.Unlock() + return refreshErr + } + + reader, getErr := s.refresher.getInnerReader() + s.refresher.mu.Unlock() + if getErr != nil { + return getErr + } + + newRc, rangeErr := reader.RangeRead(s.ctx, nextRange) + if rangeErr != nil { + return rangeErr + } + + _ = s.ReadCloser.Close() + s.ReadCloser = newRc + log.Infof("Successfully refreshed link and reconnected from offset %d", nextRange.Start) + return nil +} + +func (s *selfHealingReadCloser) remainingBytes() int64 { + length := s.httpRange.Length + if length < 0 || s.httpRange.Start+length > s.refresher.size { + length = s.refresher.size - s.httpRange.Start + } + remaining := length - s.bytesRead + if remaining < 0 { + return 0 + } + return remaining +} + +func (s *selfHealingReadCloser) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.closed { + return nil + } + s.closed = true + return s.ReadCloser.Close() +} + func GetRangeReaderFromLink(size int64, link *model.Link) (model.RangeReaderIF, error) { + // If link has a Refresher, wrap with RefreshableRangeReader for automatic refresh on expiry + if link.Refresher != nil { + return NewRefreshableRangeReader(link, size), nil + } + if link.RangeReader != nil { if link.Concurrency < 1 && link.PartSize < 1 { return link.RangeReader, nil @@ -174,6 +462,120 @@ func CacheFullAndHash(stream model.FileStreamer, up *model.UpdateProgress, hashT return tmpF, hex.EncodeToString(h.Sum(nil)), nil } +// ReadFullWithRangeRead 使用 RangeRead 从文件流中读取数据到 buf +// file: 文件流 +// buf: 目标缓冲区 +// off: 读取的起始偏移量 +// 返回值: 实际读取的字节数和错误 +// 支持自动重试（最多5次），快速重试策略（1秒、2秒、3秒、4秒、5秒） +// 注意：链接刷新现在由 RefreshableRangeReader 内部的 selfHealingReadCloser 自动处理 +func ReadFullWithRangeRead(file model.FileStreamer, buf []byte, off int64) (int, error) { + length := int64(len(buf)) + var lastErr error + + // 重试最多 MAX_RANGE_READ_RETRY_COUNT 次 + for retry := 0; retry < MAX_RANGE_READ_RETRY_COUNT; retry++ { + reader, err := file.RangeRead(http_range.Range{Start: off, Length: length}) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return 0, err + } + lastErr = fmt.Errorf("RangeRead failed at offset %d: %w", off, err) + log.Debugf("RangeRead retry %d failed: %v", retry+1, lastErr) + // 快速重试：1秒、2秒、3秒、4秒、5秒（连接失败快速重试） + time.Sleep(time.Duration(retry+1) * time.Second) + continue + } + + n, err := io.ReadFull(reader, buf) + if closer, ok := reader.(io.Closer); ok { + closer.Close() + } + + if err == nil { + return n, nil + } + + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return n, err + } + + lastErr = fmt.Errorf("failed to read all data via RangeRead at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) + log.Debugf("RangeRead retry %d read failed: %v", retry+1, lastErr) + + // 快速重试：1秒、2秒、3秒、4秒、5秒（读取失败快速重试） + // 注意：0字节读取导致的链接过期现在由 selfHealingReadCloser 自动处理 + time.Sleep(time.Duration(retry+1) * time.Second) + } + + return 0, lastErr +} + +// StreamHashFile 流式计算文件哈希值，避免将整个文件加载到内存 +// file: 文件流 +// hashType: 哈希算法类型 +// progressWeight: 进度权重（0-100），用于计算整体进度 +// up: 进度回调函数 +func StreamHashFile(file model.FileStreamer, hashType *utils.HashType, progressWeight float64, up *model.UpdateProgress) (string, error) { + // 如果已经有完整缓存文件，直接使用 + if cache := file.GetFile(); cache != nil { + hashFunc := hashType.NewFunc() + cache.Seek(0, io.SeekStart) + _, err := io.Copy(hashFunc, cache) + if err != nil { + return "", err + } + if up != nil && progressWeight > 0 { + (*up)(progressWeight) + } + return hex.EncodeToString(hashFunc.Sum(nil)), nil + } + + hashFunc := hashType.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk + buf := make([]byte, chunkSize) + var offset int64 = 0 + + for offset < size { + readSize := chunkSize + if size-offset < chunkSize { + readSize = size - offset + } + + var n int + var err error + + // 对于 SeekableStream，优先使用 RangeRead 避免消耗 Reader + // 这样后续发送时 Reader 还能正常工作 + if _, ok := file.(*SeekableStream); ok { + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) + } else { + // 对于 FileStream，首先尝试顺序流读取（不消耗额外资源，适用于所有流类型） + n, err = io.ReadFull(file, buf[:readSize]) + if err != nil { + // 顺序流读取失败，尝试使用 RangeRead 重试（适用于 SeekableStream） + log.Warnf("StreamHashFile: sequential read failed at offset %d, retrying with RangeRead: %v", offset, err) + n, err = ReadFullWithRangeRead(file, buf[:readSize], offset) + } + } + + if err != nil { + return "", fmt.Errorf("calculate hash failed at offset %d: %w", offset, err) + } + + hashFunc.Write(buf[:n]) + offset += int64(n) + + if up != nil && progressWeight > 0 { + progress := progressWeight * float64(offset) / float64(size) + (*up)(progress) + } + } + + return hex.EncodeToString(hashFunc.Sum(nil)), nil +} + type StreamSectionReaderIF interface { // 线程不安全 GetSectionReader(off, length int64) (io.ReadSeeker, error) @@ -188,37 +590,9 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode } maxBufferSize = min(maxBufferSize, int(file.GetSize())) - if maxBufferSize > conf.MaxBufferLimit { - f, err := os.CreateTemp(conf.Conf.TempDir, "file-*") - if err != nil { - return nil, err - } - - if f.Truncate(file.GetSize()) != nil { - // fallback to full cache - _, _ = f.Close(), os.Remove(f.Name()) - cache, err := file.CacheFullAndWriter(up, nil) - if err != nil { - return nil, err - } - return &cachedSectionReader{cache}, nil - } - - ss := &fileSectionReader{file: file, temp: f} - ss.bufPool = &pool.Pool[*offsetWriterWithBase]{ - New: func() *offsetWriterWithBase { - base := ss.tempOffset - ss.tempOffset += int64(maxBufferSize) - return &offsetWriterWithBase{io.NewOffsetWriter(ss.temp, base), base} - }, - } - file.Add(utils.CloseFunc(func() error { - ss.bufPool.Reset() - return errors.Join(ss.temp.Close(), os.Remove(ss.temp.Name())) - })) - return ss, nil - } + // 始终使用 directSectionReader，只在内存中缓存当前分片 + // 避免创建临时文件导致中间文件增长到整个文件大小 ss := &directSectionReader{file: file} if conf.MmapThreshold > 0 && maxBufferSize >= conf.MmapThreshold { ss.bufPool = &pool.Pool[[]byte]{ @@ -243,6 +617,7 @@ func NewStreamSectionReader(file model.FileStreamer, maxBufferSize int, up *mode } file.Add(utils.CloseFunc(func() error { + ss.clearPrefetch() ss.bufPool.Reset() return nil })) @@ -319,10 +694,129 @@ type directSectionReader struct { file model.FileStreamer fileOffset int64 bufPool *pool.Pool[[]byte] + prefetchMu sync.Mutex + prefetch *sectionPrefetchTask } -// 线程不安全 +type sectionPrefetchTask struct { + off int64 + length int64 + buf []byte + n int + err error + ready chan struct{} +} + +func (ss *directSectionReader) recyclePrefetchTask(task *sectionPrefetchTask) { + if task == nil || task.buf == nil { + return + } + ss.bufPool.Put(task.buf[0:cap(task.buf)]) + task.buf = nil +} + +func (ss *directSectionReader) clearPrefetch() { + ss.prefetchMu.Lock() + task := ss.prefetch + ss.prefetch = nil + ss.prefetchMu.Unlock() + if task == nil { + return + } + go func(t *sectionPrefetchTask) { + <-t.ready + ss.recyclePrefetchTask(t) + }(task) +} + +func (ss *directSectionReader) takeMatchingPrefetch(off, length int64) *sectionPrefetchTask { + ss.prefetchMu.Lock() + task := ss.prefetch + if task != nil && task.off == off && task.length == length { + ss.prefetch = nil + ss.prefetchMu.Unlock() + return task + } + ss.prefetch = nil + ss.prefetchMu.Unlock() + if task != nil { + go func(t *sectionPrefetchTask) { + <-t.ready + ss.recyclePrefetchTask(t) + }(task) + } + return nil +} + +func (ss *directSectionReader) launchPrefetch(off, length int64) { + if length <= 0 { + return + } + tempBuf := ss.bufPool.Get() + if int64(cap(tempBuf)) < length { + tempBuf = make([]byte, length) + } + task := §ionPrefetchTask{ + off: off, + length: length, + buf: tempBuf, + ready: make(chan struct{}), + } + + ss.prefetchMu.Lock() + old := ss.prefetch + ss.prefetch = task + ss.prefetchMu.Unlock() + + if old != nil { + go func(t *sectionPrefetchTask) { + <-t.ready + ss.recyclePrefetchTask(t) + }(old) + } + + go func(t *sectionPrefetchTask) { + buf := t.buf[:int(t.length)] + n, err := ReadFullWithRangeRead(ss.file, buf, t.off) + if err != nil { + t.err = fmt.Errorf("RangeRead failed at offset %d: (expect=%d, actual=%d) %w", t.off, t.length, n, err) + } else if int64(n) != t.length { + t.err = fmt.Errorf("RangeRead failed at offset %d: (expect=%d, actual=%d)", t.off, t.length, n) + } else { + t.n = n + } + close(t.ready) + }(task) +} + +func (ss *directSectionReader) scheduleNextPrefetch(curOff, curLen int64) { + if _, ok := ss.file.(*SeekableStream); !ok { + return + } + nextOff := curOff + curLen + if nextOff >= ss.file.GetSize() { + ss.clearPrefetch() + return + } + nextLen := min(curLen, ss.file.GetSize()-nextOff) + if nextLen <= 0 { + ss.clearPrefetch() + return + } + ss.launchPrefetch(nextOff, nextLen) +} + +// 线程不安全（依赖调用方保证串行调用） +// 对于 SeekableStream：直接跳过（无需实际读取） +// 对于 FileStream：必须顺序读取并丢弃 func (ss *directSectionReader) DiscardSection(off int64, length int64) error { + // 对于 SeekableStream，直接跳过（RangeRead 支持随机访问，不需要实际读取） + if _, ok := ss.file.(*SeekableStream); ok { + ss.clearPrefetch() + return nil + } + + // 对于 FileStream，必须顺序读取并丢弃 if off != ss.fileOffset { return fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } @@ -339,19 +833,53 @@ type bufferSectionReader struct { buf []byte } -// 线程不安全 +// 线程不安全（依赖调用方保证串行调用） +// 对于 SeekableStream：使用 RangeRead，支持随机访问（续传场景可跳过已上传分片） +// 对于 FileStream：必须顺序读取 func (ss *directSectionReader) GetSectionReader(off, length int64) (io.ReadSeeker, error) { + tempBuf := ss.bufPool.Get() + if int64(cap(tempBuf)) < length { + tempBuf = make([]byte, length) + } + buf := tempBuf[:int(length)] + + // 对于 SeekableStream，直接使用 RangeRead（支持随机访问，适用于续传场景） + if _, ok := ss.file.(*SeekableStream); ok { + if task := ss.takeMatchingPrefetch(off, length); task != nil { + <-task.ready + if task.err != nil { + ss.recyclePrefetchTask(task) + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) + return nil, task.err + } + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) + ss.scheduleNextPrefetch(off, length) + return &bufferSectionReader{bytes.NewReader(task.buf[:int(length)]), task.buf}, nil + } + + n, err := ReadFullWithRangeRead(ss.file, buf, off) + if err != nil { + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) + return nil, fmt.Errorf("RangeRead failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) + } + ss.scheduleNextPrefetch(off, length) + return &bufferSectionReader{bytes.NewReader(buf), tempBuf}, nil + } + + // 对于 FileStream，必须顺序读取 if off != ss.fileOffset { + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) return nil, fmt.Errorf("stream not cached: request offset %d != current offset %d", off, ss.fileOffset) } - tempBuf := ss.bufPool.Get() - buf := tempBuf[:length] + n, err := io.ReadFull(ss.file, buf) - ss.fileOffset += int64(n) - if int64(n) != length { - return nil, fmt.Errorf("failed to read all data: (expect =%d, actual =%d) %w", length, n, err) + if err != nil { + ss.recyclePrefetchTask(§ionPrefetchTask{buf: tempBuf}) + return nil, fmt.Errorf("sequential read failed at offset %d: (expect=%d, actual=%d) %w", off, length, n, err) } - return &bufferSectionReader{bytes.NewReader(buf), buf}, nil + + ss.fileOffset = off + int64(n) + return &bufferSectionReader{bytes.NewReader(buf), tempBuf}, nil } func (ss *directSectionReader) FreeSectionReader(rs io.ReadSeeker) { if sr, ok := rs.(*bufferSectionReader); ok { diff --git a/internal/stream/util_test.go b/internal/stream/util_test.go new file mode 100644 index 000000000..6dc4c8a09 --- /dev/null +++ b/internal/stream/util_test.go @@ -0,0 +1,160 @@ +package stream + +import ( + "bytes" + "context" + "errors" + "io" + "sync" + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" +) + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: int64(len(data))}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != int64(len(data)-5) { + t.Fatalf("resumed range length = %d, want %d", resumedRanges[0].Length, len(data)-5) + } +} + +func TestRefreshableRangeReader_ReconnectsAfterMidStreamReset_UnboundedRange(t *testing.T) { + data := []byte("0123456789abcdef") + var refreshes int + var mu sync.Mutex + var resumedRanges []http_range.Range + + initial := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return newFlakyReadCloser(sliceForRange(data, httpRange), 5, errors.New("read tcp 127.0.0.1:443: read: connection reset by peer")), nil + }) + resumed := RangeReaderFunc(func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + mu.Lock() + resumedRanges = append(resumedRanges, httpRange) + mu.Unlock() + return io.NopCloser(bytes.NewReader(sliceForRange(data, httpRange))), nil + }) + + link := &model.Link{RangeReader: initial} + link.Refresher = func(ctx context.Context) (*model.Link, model.Obj, error) { + refreshes++ + return &model.Link{RangeReader: resumed}, nil, nil + } + + reader, err := NewRefreshableRangeReader(link, int64(len(data))).RangeRead(context.Background(), http_range.Range{Start: 0, Length: -1}) + if err != nil { + t.Fatalf("RangeRead() error = %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll() error = %v", err) + } + if !bytes.Equal(got, data) { + t.Fatalf("ReadAll() = %q, want %q", got, data) + } + if refreshes != 1 { + t.Fatalf("refreshes = %d, want 1", refreshes) + } + + mu.Lock() + defer mu.Unlock() + if len(resumedRanges) != 1 { + t.Fatalf("len(resumedRanges) = %d, want 1", len(resumedRanges)) + } + if resumedRanges[0].Start != 5 { + t.Fatalf("resumed range start = %d, want 5", resumedRanges[0].Start) + } + if resumedRanges[0].Length != -1 { + t.Fatalf("resumed range length = %d, want -1", resumedRanges[0].Length) + } +} + +type flakyReadCloser struct { + data []byte + failAfter int + failErr error + failed bool +} + +func newFlakyReadCloser(data []byte, failAfter int, failErr error) *flakyReadCloser { + return &flakyReadCloser{ + data: data, + failAfter: failAfter, + failErr: failErr, + } +} + +func (f *flakyReadCloser) Read(p []byte) (int, error) { + if f.failed { + return 0, io.EOF + } + if f.failAfter >= len(f.data) { + f.failed = true + n := copy(p, f.data) + return n, io.EOF + } + + n := copy(p, f.data[:f.failAfter]) + f.failed = true + return n, f.failErr +} + +func (f *flakyReadCloser) Close() error { + return nil +} + +func sliceForRange(data []byte, httpRange http_range.Range) []byte { + start := int(httpRange.Start) + length := int(httpRange.Length) + if httpRange.Length < 0 || httpRange.Start+httpRange.Length > int64(len(data)) { + length = len(data) - start + } + return data[start : start+length] +} diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index 596e61e54..c4b4e735f 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -90,6 +90,12 @@ var ( // SHA256 indicates SHA-256 support SHA256 = RegisterHash("sha256", "SHA-256", 64, sha256.New) + + // SHA1_128K is SHA1 of first 128KB, used by 115 driver for rapid upload + SHA1_128K = RegisterHash("sha1_128k", "SHA1-128K", 40, sha1.New) + + // PRE_HASH is SHA1 of first 1024 bytes, used by Aliyundrive for rapid upload + PRE_HASH = RegisterHash("pre_hash", "PRE-HASH", 40, sha1.New) ) // HashData get hash of one hashType diff --git a/public/dist/README.md b/public/dist/README.md deleted file mode 100644 index d8709fb57..000000000 --- a/public/dist/README.md +++ /dev/null @@ -1 +0,0 @@ -## Put dist of frontend here. \ No newline at end of file diff --git a/server/common/check.go b/server/common/check.go index 90074aeeb..27be3103c 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -2,6 +2,7 @@ package common import ( "path" + "slices" "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" @@ -17,24 +18,39 @@ func IsStorageSignEnabled(rawPath string) bool { return storage != nil && storage.GetStorage().EnableSign } -func CanWrite(meta *model.Meta, path string) bool { - if meta == nil || !meta.Write { +func CanRead(user *model.User, meta *model.Meta, path string) bool { + // nil user is treated as internal/system context and bypasses per-user read restrictions + if user == nil { + return true + } + if meta != nil && len(meta.ReadUsers) > 0 && !slices.Contains(meta.ReadUsers, user.ID) && MetaCoversPath(meta.Path, path, meta.ReadUsersSub) { return false } - return meta.WSub || meta.Path == path + return true } -func IsApply(metaPath, reqPath string, applySub bool) bool { - if utils.PathEqual(metaPath, reqPath) { +func CanWrite(user *model.User, meta *model.Meta, path string) bool { + // nil user is treated as internal/system context and bypasses per-user write restrictions + if user == nil { return true } - return utils.IsSubPath(metaPath, reqPath) && applySub + if meta != nil && len(meta.WriteUsers) > 0 && !slices.Contains(meta.WriteUsers, user.ID) && MetaCoversPath(meta.Path, path, meta.WriteUsersSub) { + return false + } + return true +} + +func CanWriteContentBypassUserPerms(meta *model.Meta, path string) bool { + if meta == nil || !meta.Write { + return false + } + return MetaCoversPath(meta.Path, path, meta.WSub) } func CanAccess(user *model.User, meta *model.Meta, reqPath string, password string) bool { // if the reqPath is in hide (only can check the nearest meta) and user can't see hides, can't access if meta != nil && !user.CanSeeHides() && meta.Hide != "" && - IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path + MetaCoversPath(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path for _, hide := range strings.Split(meta.Hide, "\n") { re := regexp2.MustCompile(hide, regexp2.None) if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { @@ -42,6 +58,9 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri } } } + if !CanRead(user, meta, reqPath) { + return false + } // if is not guest and can access without password if user.CanAccessWithoutPassword() { return true @@ -51,13 +70,20 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri return true } // if meta doesn't apply to sub_folder, can access - if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { + if !MetaCoversPath(meta.Path, reqPath, meta.PSub) { return true } // validate password return meta.Password == password } +func MetaCoversPath(metaPath, reqPath string, applyToSubFolder bool) bool { + if utils.PathEqual(metaPath, reqPath) { + return true + } + return utils.IsSubPath(metaPath, reqPath) && applyToSubFolder +} + // ShouldProxy TODO need optimize // when should be proxy? // 1. config.MustProxy() diff --git a/server/common/check_test.go b/server/common/check_test.go index 33114603b..18abca8e9 100644 --- a/server/common/check_test.go +++ b/server/common/check_test.go @@ -1,24 +1,986 @@ package common -import "testing" +import ( + "testing" -func TestIsApply(t *testing.T) { - datas := []struct { + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestCoversPath(t *testing.T) { + tests := []struct { + name string metaPath string reqPath string applySub bool - result bool + want bool }{ { + name: "exact path match with applySub=false", + metaPath: "/folder", + reqPath: "/folder", + applySub: false, + want: true, + }, + { + name: "exact path match with applySub=true", + metaPath: "/folder", + reqPath: "/folder", + applySub: true, + want: true, + }, + { + name: "sub path with applySub=true", + metaPath: "/folder", + reqPath: "/folder/subfolder", + applySub: true, + want: true, + }, + { + name: "sub path with applySub=false", + metaPath: "/folder", + reqPath: "/folder/subfolder", + applySub: false, + want: false, + }, + { + name: "non-sub path with applySub=true", + metaPath: "/folder", + reqPath: "/other", + applySub: true, + want: false, + }, + { + name: "non-sub path with applySub=false", + metaPath: "/folder", + reqPath: "/other", + applySub: false, + want: false, + }, + { + name: "root path covers all with applySub=true", metaPath: "/", - reqPath: "/test", + reqPath: "/any/deep/path", + applySub: true, + want: true, + }, + { + name: "root path exact match", + metaPath: "/", + reqPath: "/", + applySub: false, + want: true, + }, + { + name: "deep sub path with applySub=true", + metaPath: "/folder", + reqPath: "/folder/sub1/sub2/file.txt", applySub: true, - result: true, + want: true, + }, + { + name: "sibling paths with applySub=true", + metaPath: "/folder1", + reqPath: "/folder2", + applySub: true, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := MetaCoversPath(tt.metaPath, tt.reqPath, tt.applySub) + if got != tt.want { + t.Errorf("MetaCoversPath(%q, %q, %v) = %v, want %v", + tt.metaPath, tt.reqPath, tt.applySub, got, tt.want) + } + }) + } +} + +func TestCanWriteContentIgnoringUserPerms(t *testing.T) { + tests := []struct { + name string + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil meta", + meta: nil, + path: "/any", + want: false, + reason: "nil meta should deny write", + }, + { + name: "meta.Write=false", + meta: &model.Meta{ + Path: "/folder", + Write: false, + }, + path: "/folder", + want: false, + reason: "Write=false should deny write", + }, + { + name: "exact path match with WSub=false", + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: false, + }, + path: "/folder", + want: true, + reason: "exact path match should allow write", + }, + { + name: "sub path with WSub=true", + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: true, + }, + path: "/folder/subfolder", + want: true, + reason: "sub path with WSub=true should allow write", + }, + { + name: "sub path with WSub=false (BEHAVIOR CHANGE)", + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: false, + }, + path: "/folder/subfolder", + want: false, + reason: "sub path with WSub=false should deny write (fixed bug)", + }, + { + name: "non-sub path with WSub=true", + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: true, + }, + path: "/other", + want: false, + reason: "non-sub path should deny write even with WSub=true", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CanWriteContentBypassUserPerms(tt.meta, tt.path) + if got != tt.want { + t.Errorf("CanWriteContentBypassUserPerms() = %v, want %v\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} + +func TestCanRead(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil user should allow access", + user: nil, + meta: nil, + path: "/any", + want: true, + reason: "nil user represents internal/system context and bypasses per-user read restrictions", + }, + { + name: "nil meta should allow access", + user: &model.User{ + ID: 1, + }, + meta: nil, + path: "/any", + want: true, + reason: "nil meta means no restrictions", + }, + { + name: "empty ReadUsers list should allow access", + user: &model.User{ + ID: 1, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{}, + }, + path: "/folder", + want: true, + reason: "empty ReadUsers means no user-level restrictions", + }, + { + name: "user in ReadUsers list with exact path match", + user: &model.User{ + ID: 1, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: false, + }, + path: "/folder", + want: true, + reason: "user ID 1 is in ReadUsers list", + }, + { + name: "user not in ReadUsers list with exact path match", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: false, + }, + path: "/folder", + want: false, + reason: "user ID 5 is not in ReadUsers list and path matches", + }, + { + name: "user not in ReadUsers list with ReadUsersSub=true for sub path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: true, + }, + path: "/folder/subfolder", + want: false, + reason: "user ID 5 is not in ReadUsers list and ReadUsersSub applies to sub paths", + }, + { + name: "user not in ReadUsers list with ReadUsersSub=false for sub path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: false, + }, + path: "/folder/subfolder", + want: true, + reason: "ReadUsersSub=false means restriction doesn't apply to sub paths", + }, + { + name: "user in ReadUsers list with ReadUsersSub=true for sub path", + user: &model.User{ + ID: 2, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: true, + }, + path: "/folder/subfolder/deep", + want: true, + reason: "user ID 2 is in ReadUsers list so can access sub paths", + }, + { + name: "user not in ReadUsers list for different path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: false, + }, + path: "/other", + want: true, + reason: "meta path doesn't match request path, so restriction doesn't apply", + }, + { + name: "root level restriction with ReadUsersSub=true", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/", + ReadUsers: []uint{1, 2, 3}, + ReadUsersSub: true, + }, + path: "/any/deep/path", + want: false, + reason: "root level restriction with ReadUsersSub affects all paths", }, } - for i, data := range datas { - if IsApply(data.metaPath, data.reqPath, data.applySub) != data.result { - t.Errorf("TestIsApply %d failed", i) - } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CanRead(tt.user, tt.meta, tt.path) + if got != tt.want { + t.Errorf("CanRead() = %v, want %v\nReason: %s\nUser ID: %v, Meta: %+v, Path: %s", + got, tt.want, tt.reason, getUserID(tt.user), tt.meta, tt.path) + } + }) + } +} + +func TestCanWrite(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil user should allow access", + user: nil, + meta: nil, + path: "/any", + want: true, + reason: "nil user represents internal/system context and bypasses per-user write restrictions", + }, + { + name: "nil meta should allow access", + user: &model.User{ + ID: 1, + }, + meta: nil, + path: "/any", + want: true, + reason: "nil meta means no restrictions", + }, + { + name: "empty WriteUsers list should allow access", + user: &model.User{ + ID: 1, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{}, + }, + path: "/folder", + want: true, + reason: "empty WriteUsers means no user-level restrictions", + }, + { + name: "user in WriteUsers list with exact path match", + user: &model.User{ + ID: 1, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "user ID 1 is in WriteUsers list", + }, + { + name: "user not in WriteUsers list with exact path match", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: false, + }, + path: "/folder", + want: false, + reason: "user ID 5 is not in WriteUsers list and path matches", + }, + { + name: "user not in WriteUsers list with WriteUsersSub=true for sub path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: true, + }, + path: "/folder/subfolder", + want: false, + reason: "user ID 5 is not in WriteUsers list and WriteUsersSub applies to sub paths", + }, + { + name: "user not in WriteUsers list with WriteUsersSub=false for sub path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: false, + }, + path: "/folder/subfolder", + want: true, + reason: "WriteUsersSub=false means restriction doesn't apply to sub paths", + }, + { + name: "user in WriteUsers list with WriteUsersSub=true for sub path", + user: &model.User{ + ID: 2, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: true, + }, + path: "/folder/subfolder/deep", + want: true, + reason: "user ID 2 is in WriteUsers list so can write to sub paths", + }, + { + name: "user not in WriteUsers list for different path", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 2, 3}, + WriteUsersSub: false, + }, + path: "/other", + want: true, + reason: "meta path doesn't match request path, so restriction doesn't apply", + }, + { + name: "multiple users with mixed permissions", + user: &model.User{ + ID: 10, + }, + meta: &model.Meta{ + Path: "/folder", + WriteUsers: []uint{1, 5, 10, 15}, + WriteUsersSub: true, + }, + path: "/folder/file.txt", + want: true, + reason: "user ID 10 is in WriteUsers list", + }, + { + name: "write restriction at root level", + user: &model.User{ + ID: 5, + }, + meta: &model.Meta{ + Path: "/", + WriteUsers: []uint{1}, + WriteUsersSub: true, + }, + path: "/any/path", + want: false, + reason: "only user ID 1 can write when root has WriteUsers restriction", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CanWrite(tt.user, tt.meta, tt.path) + if got != tt.want { + t.Errorf("CanWrite() = %v, want %v\nReason: %s\nUser ID: %v, Meta: %+v, Path: %s", + got, tt.want, tt.reason, getUserID(tt.user), tt.meta, tt.path) + } + }) + } +} + +func TestCanAccessWithReadPermissions(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + reqPath string + password string + want bool + reason string + }{ + { + name: "user with read permission and correct password", + user: &model.User{ + ID: 1, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2}, + ReadUsersSub: true, + Password: "secret", + PSub: true, + }, + reqPath: "/folder/file.txt", + password: "secret", + want: true, + reason: "user in ReadUsers list with correct password", + }, + { + name: "user without read permission even with correct password", + user: &model.User{ + ID: 5, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2}, + ReadUsersSub: true, + Password: "secret", + PSub: true, + }, + reqPath: "/folder/file.txt", + password: "secret", + want: false, + reason: "user not in ReadUsers list, should be denied before password check", + }, + { + name: "user with read permission but wrong password", + user: &model.User{ + ID: 1, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2}, + ReadUsersSub: true, + Password: "secret", + PSub: true, + }, + reqPath: "/folder/file.txt", + password: "wrong", + want: false, + reason: "user in ReadUsers list but wrong password", + }, + { + name: "user without read permission and no password", + user: &model.User{ + ID: 5, + Role: model.GENERAL, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + ReadUsers: []uint{1, 2}, + ReadUsersSub: true, + }, + reqPath: "/folder/file.txt", + password: "", + want: false, + reason: "user not in ReadUsers list should be denied", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CanAccess(tt.user, tt.meta, tt.reqPath, tt.password) + if got != tt.want { + t.Errorf("CanAccess() = %v, want %v\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} + +// Helper function to safely get user ID +func getUserID(user *model.User) uint { + if user == nil { + return 0 + } + return user.ID +} + +// TestWritePermissionCombinations tests the combined permission check logic +// that is actually used in the codebase: +// +// if !user.CanWriteContent() && !CanWriteContentBypassUserPerms(meta, path) { +// deny +// } +// if !CanWrite(user, meta, path) { +// deny +// } +// +// This ensures the three-layer permission system works correctly: +// 1. User-level global write permission (CanWriteContent) +// 2. Meta-level global write permission (CanWriteContentBypassUserPerms) +// 3. Meta-level user whitelist (CanWrite) +func TestWritePermissionCombinations(t *testing.T) { + tests := []struct { + name string + user *model.User + meta *model.Meta + path string + want bool + reason string + checkFirstLayer bool // whether first layer should pass + checkSecondLayer bool // whether second layer should pass + expectedDenyReason string + }{ + // === Scenario 1: User has global write permission === + { + name: "user has CanWriteContent + in WriteUsers whitelist", + user: &model.User{ + ID: 1, + Permission: 1 << 3, // CanWriteContent = true + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, + WriteUsers: []uint{1}, + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "user has global write permission AND is in whitelist", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "user has CanWriteContent but NOT in WriteUsers whitelist", + user: &model.User{ + ID: 1, + Permission: 1 << 3, // CanWriteContent = true + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, + WriteUsers: []uint{2, 3}, // user 1 not in list + WriteUsersSub: false, + }, + path: "/folder", + want: false, + reason: "even with global write permission, must pass whitelist check", + checkFirstLayer: true, + checkSecondLayer: false, + expectedDenyReason: "whitelist check failed", + }, + + // === Scenario 2: User lacks global permission but meta.Write=true === + { + name: "no CanWriteContent + meta.Write=true + in WriteUsers", + user: &model.User{ + ID: 1, + Permission: 0, // CanWriteContent = false + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, // bypass enabled + WSub: false, + WriteUsers: []uint{1}, + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "meta.Write bypasses user permission check, and user is in whitelist", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "no CanWriteContent + meta.Write=true + NOT in WriteUsers (KEY TEST)", + user: &model.User{ + ID: 5, + Permission: 0, // CanWriteContent = false + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, // bypass enabled + WSub: false, + WriteUsers: []uint{1, 2, 3}, // user 5 not in list + WriteUsersSub: false, + }, + path: "/folder", + want: false, + reason: "CRITICAL: meta.Write cannot bypass whitelist check (new behavior)", + checkFirstLayer: true, + checkSecondLayer: false, + expectedDenyReason: "whitelist check failed even with meta.Write=true", + }, + + // === Scenario 3: Both checks fail === + { + name: "no CanWriteContent + meta.Write=false", + user: &model.User{ + ID: 1, + Permission: 0, // CanWriteContent = false + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, // no bypass + WriteUsers: []uint{1}, + WriteUsersSub: false, + }, + path: "/folder", + want: false, + reason: "denied at first layer: no global permission and no bypass", + checkFirstLayer: false, + checkSecondLayer: false, + expectedDenyReason: "first layer check failed", + }, + + // === Scenario 4: Empty WriteUsers (no whitelist restriction) === + { + name: "user has CanWriteContent + empty WriteUsers", + user: &model.User{ + ID: 1, + Permission: 1 << 3, // CanWriteContent = true + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, + WriteUsers: []uint{}, // empty = no restriction + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "empty WriteUsers means no whitelist restriction", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "no CanWriteContent + meta.Write=true + empty WriteUsers", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: false, + WriteUsers: []uint{}, // empty = no restriction + WriteUsersSub: false, + }, + path: "/folder", + want: true, + reason: "meta.Write bypasses first check, empty whitelist passes second", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + + // === Scenario 5: Nil meta (no restrictions) === + { + name: "user has CanWriteContent + nil meta", + user: &model.User{ + ID: 1, + Permission: 1 << 3, + }, + meta: nil, + path: "/folder", + want: true, + reason: "nil meta means no restrictions", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "no CanWriteContent + nil meta", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: nil, + path: "/folder", + want: false, + reason: "nil meta cannot bypass lack of user permission", + checkFirstLayer: false, + checkSecondLayer: true, // would pass if first layer passed + expectedDenyReason: "first layer check failed", + }, + + // === Scenario 6: Sub-directory inheritance === + { + name: "meta.Write with WSub=true for subdirectory", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: true, // applies to subdirectories + WriteUsers: []uint{1}, + WriteUsersSub: true, + }, + path: "/folder/subfolder", + want: true, + reason: "WSub=true applies meta.Write to subdirectories", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "meta.Write with WSub=false for subdirectory", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/folder", + Write: true, + WSub: false, // does NOT apply to subdirectories + WriteUsers: []uint{1}, + WriteUsersSub: false, + }, + path: "/folder/subfolder", + want: false, + reason: "WSub=false means meta.Write doesn't apply to subdirectories", + checkFirstLayer: false, + checkSecondLayer: true, + expectedDenyReason: "first layer check failed (WSub=false)", + }, + { + name: "WriteUsersSub=false for subdirectory bypasses whitelist", + user: &model.User{ + ID: 5, // not in WriteUsers + Permission: 1 << 3, + }, + meta: &model.Meta{ + Path: "/folder", + Write: false, + WriteUsers: []uint{1, 2}, + WriteUsersSub: false, // whitelist does NOT apply to subdirectories + }, + path: "/folder/subfolder", + want: true, + reason: "WriteUsersSub=false means whitelist doesn't apply to subdirectories", + checkFirstLayer: true, + checkSecondLayer: true, // passes because restriction doesn't apply + expectedDenyReason: "", + }, + + // === Scenario 7: Root level restriction === + { + name: "root level meta.Write with user in whitelist", + user: &model.User{ + ID: 1, + Permission: 0, + }, + meta: &model.Meta{ + Path: "/", + Write: true, + WSub: true, + WriteUsers: []uint{1}, + WriteUsersSub: true, + }, + path: "/any/deep/path", + want: true, + reason: "root level permissions apply to all paths", + checkFirstLayer: true, + checkSecondLayer: true, + expectedDenyReason: "", + }, + { + name: "root level restriction denies non-whitelisted user", + user: &model.User{ + ID: 5, + Permission: 1 << 3, // has global permission + }, + meta: &model.Meta{ + Path: "/", + Write: false, + WriteUsers: []uint{1, 2}, + WriteUsersSub: true, + }, + path: "/any/path", + want: false, + reason: "root level whitelist restricts all paths", + checkFirstLayer: true, + checkSecondLayer: false, + expectedDenyReason: "not in root level whitelist", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Simulate the actual permission check logic + firstLayerPass := tt.user.CanWriteContent() || CanWriteContentBypassUserPerms(tt.meta, tt.path) + secondLayerPass := CanWrite(tt.user, tt.meta, tt.path) + + // Verify our understanding of each layer + if firstLayerPass != tt.checkFirstLayer { + t.Errorf("First layer check mismatch: got %v, expected %v\n"+ + "CanWriteContent()=%v, CanWriteContentBypassUserPerms()=%v", + firstLayerPass, tt.checkFirstLayer, + tt.user.CanWriteContent(), CanWriteContentBypassUserPerms(tt.meta, tt.path)) + } + + if firstLayerPass && secondLayerPass != tt.checkSecondLayer { + t.Errorf("Second layer check mismatch: got %v, expected %v\n"+ + "CanWrite()=%v", + secondLayerPass, tt.checkSecondLayer, + CanWrite(tt.user, tt.meta, tt.path)) + } + + // Final result + got := firstLayerPass && secondLayerPass + + if got != tt.want { + t.Errorf("Permission check failed:\n"+ + " Result: %v, want %v\n"+ + " Reason: %s\n"+ + " First layer (CanWriteContent || CanWriteContentBypassUserPerms): %v\n"+ + " Second layer (CanWrite): %v\n"+ + " User: ID=%d, Permission=%d, CanWriteContent=%v\n"+ + " Meta: Path=%s, Write=%v, WSub=%v, WriteUsers=%v, WriteUsersSub=%v\n"+ + " Check Path: %s", + got, tt.want, + tt.reason, + firstLayerPass, + secondLayerPass, + tt.user.ID, tt.user.Permission, tt.user.CanWriteContent(), + getMetaPath(tt.meta), getMetaWrite(tt.meta), getMetaWSub(tt.meta), + getMetaWriteUsers(tt.meta), getMetaWriteUsersSub(tt.meta), + tt.path) + } + }) + } +} + +// Helper functions to safely extract meta fields +func getMetaPath(meta *model.Meta) string { + if meta == nil { + return "nil" + } + return meta.Path +} + +func getMetaWrite(meta *model.Meta) bool { + if meta == nil { + return false + } + return meta.Write +} + +func getMetaWSub(meta *model.Meta) bool { + if meta == nil { + return false + } + return meta.WSub +} + +func getMetaWriteUsers(meta *model.Meta) []uint { + if meta == nil { + return nil + } + return meta.WriteUsers +} + +func getMetaWriteUsersSub(meta *model.Meta) bool { + if meta == nil { + return false } + return meta.WriteUsersSub } diff --git a/server/ftp/fsmanage.go b/server/ftp/fsmanage.go index 48f72794e..3e98d6d14 100644 --- a/server/ftp/fsmanage.go +++ b/server/ftp/fsmanage.go @@ -15,20 +15,23 @@ import ( func Mkdir(ctx context.Context, path string) error { user := ctx.Value(conf.UserKey).(*model.User) + if !user.CanFTPManage() { + return errs.PermissionDenied + } reqPath, err := user.JoinPath(path) if err != nil { return err } - if !user.CanWrite() || !user.CanFTPManage() { - meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return err - } - } - if !common.CanWrite(meta, reqPath) { - return errs.PermissionDenied - } + parentPath := stdpath.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + return errs.PermissionDenied + } + if !common.CanWrite(user, parentMeta, parentPath) { + return errs.PermissionDenied } return fs.MakeDir(ctx, reqPath) } @@ -42,6 +45,13 @@ func Remove(ctx context.Context, path string) error { if err != nil { return err } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + if !common.CanWrite(user, meta, reqPath) { + return errs.PermissionDenied + } if err = RemoveStage(reqPath); !errors.Is(err, errs.ObjectNotFound) { return err } @@ -60,8 +70,12 @@ func Rename(ctx context.Context, oldPath, newPath string) error { } srcDir, srcBase := stdpath.Split(srcPath) dstDir, dstBase := stdpath.Split(dstPath) + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } if srcDir == dstDir { - if !user.CanRename() || !user.CanFTPManage() { + if !user.CanRename() || !user.CanFTPManage() || !common.CanWrite(user, dstMeta, dstDir) { return errs.PermissionDenied } if err = MoveStage(srcPath, dstPath); !errors.Is(err, errs.ObjectNotFound) { @@ -69,7 +83,11 @@ func Rename(ctx context.Context, oldPath, newPath string) error { } return fs.Rename(ctx, srcPath, dstBase) } else { - if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) { + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + if !user.CanMove() || !user.CanFTPManage() || (srcBase != dstBase && !user.CanRename()) || !common.CanWrite(user, srcMeta, srcDir) || !common.CanWrite(user, dstMeta, dstDir) { return errs.PermissionDenied } if err = MoveStage(srcPath, dstPath); !errors.Is(err, errs.ObjectNotFound) { diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index 9080bae17..54a3de8f2 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -27,10 +27,8 @@ type FileDownloadProxy struct { func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownloadProxy, error) { user := ctx.Value(conf.UserKey).(*model.User) meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return nil, err - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err } ctx = context.WithValue(ctx, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, ctx.Value(conf.MetaPassKey).(string)) { @@ -121,10 +119,8 @@ func Stat(ctx context.Context, path string) (os.FileInfo, error) { return nil, err } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return nil, err - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err } ctx = context.WithValue(ctx, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, ctx.Value(conf.MetaPassKey).(string)) { @@ -147,10 +143,8 @@ func List(ctx context.Context, path string) ([]os.FileInfo, error) { return nil, err } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return nil, err - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err } ctx = context.WithValue(ctx, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, ctx.Value(conf.MetaPassKey).(string)) { diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index c549a1943..7a96a4f65 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -33,14 +33,18 @@ type FileUploadProxy struct { func uploadAuth(ctx context.Context, path string) error { user := ctx.Value(conf.UserKey).(*model.User) - meta, err := op.GetNearestMeta(stdpath.Dir(path)) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return err - } + if !user.CanFTPManage() { + return errs.PermissionDenied + } + parentPath := stdpath.Dir(path) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + return errs.PermissionDenied } - if !(common.CanAccess(user, meta, path, ctx.Value(conf.MetaPassKey).(string)) && - ((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) { + if !common.CanWrite(user, parentMeta, parentPath) { return errs.PermissionDenied } return nil diff --git a/server/handles/archive.go b/server/handles/archive.go index 56418de26..d46f83c86 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -101,11 +101,9 @@ func FsArchiveMeta(c *gin.Context, req *ArchiveMetaReq, user *model.User) { return } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { @@ -186,11 +184,9 @@ func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { return } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { @@ -231,7 +227,7 @@ func FsArchiveList(c *gin.Context, req *ArchiveListReq, user *model.User) { type ArchiveDecompressReq struct { SrcDir string `json:"src_dir" form:"src_dir"` DstDir string `json:"dst_dir" form:"dst_dir"` - Name []string `json:"name" form:"name"` + Names []string `json:"name" form:"name"` ArchivePass string `json:"archive_pass" form:"archive_pass"` InnerPath string `json:"inner_path" form:"inner_path"` CacheFull bool `json:"cache_full" form:"cache_full"` @@ -250,8 +246,8 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcPaths := make([]string, 0, len(req.Name)) - for _, name := range req.Name { + srcPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) if err != nil { common.ErrorResp(c, err, 403) @@ -264,6 +260,15 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, err, 403) return } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, dstMeta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } tasks := make([]task.TaskExtensionInfo, 0, len(srcPaths)) for _, srcPath := range srcPaths { t, e := fs.ArchiveDecompress(c.Request.Context(), srcPath, dstDir, model.ArchiveDecompressArgs{ diff --git a/server/handles/auth.go b/server/handles/auth.go index 35776ba6a..780069091 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -45,27 +45,28 @@ func loginHash(c *gin.Context, req *LoginReq) { ip := c.ClientIP() count, ok := model.LoginCache.Get(ip) if ok && count >= model.DefaultMaxAuthRetries { - common.ErrorStrResp(c, "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.", 429) + common.ErrorStrResp(c, model.TooManyAttempts, 429) model.LoginCache.Expire(ip, model.DefaultLockDuration) return } // check username user, err := op.GetUserByName(req.Username) if err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, model.InvalidUsernameOrPassword, 401) model.LoginCache.Set(ip, count+1) return } // check 2FA if user.OtpSecret != "" { if !totp.Validate(req.OtpCode, user.OtpSecret) { - common.ErrorStrResp(c, "Invalid 2FA code", 402) + // 402 - need opt + common.ErrorStrResp(c, model.Invalid2FACode, 402) model.LoginCache.Set(ip, count+1) return } @@ -73,7 +74,7 @@ func loginHash(c *gin.Context, req *LoginReq) { // generate token token, err := common.GenerateToken(user) if err != nil { - common.ErrorResp(c, err, 400, true) + common.ErrorResp(c, err, 500, true) return } common.SuccessResp(c, gin.H{"token": token}) @@ -107,7 +108,7 @@ func UpdateCurrent(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not update profile", 403) + common.ErrorStrResp(c, model.GuestCannotUpdateProfile, 403) return } user.Username = req.Username @@ -125,7 +126,7 @@ func UpdateCurrent(c *gin.Context) { func Generate2FA(c *gin.Context) { user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } key, err := totp.Generate(totp.GenerateOpts{ @@ -164,11 +165,11 @@ func Verify2FA(c *gin.Context) { } user := c.Request.Context().Value(conf.UserKey).(*model.User) if user.IsGuest() { - common.ErrorStrResp(c, "Guest user can not generate 2FA code", 403) + common.ErrorStrResp(c, model.GuestCannotGenerate2FA, 403) return } if !totp.Validate(req.Code, req.Secret) { - common.ErrorStrResp(c, "Invalid 2FA code", 400) + common.ErrorStrResp(c, model.Invalid2FACode, 400) return } user.OtpSecret = req.Secret diff --git a/server/handles/down.go b/server/handles/down.go index d4d634cbe..50025a0b6 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -1,11 +1,8 @@ package handles import ( - "bytes" "errors" - "fmt" stdpath "path" - "strconv" "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/driver" @@ -17,9 +14,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" - "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" - "github.com/yuin/goldmark" ) func Down(c *gin.Context) { @@ -115,33 +110,7 @@ func proxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) { link = common.ProxyRange(c, link, file.GetSize()) } Writer := &common.WrittenResponseWriter{ResponseWriter: c.Writer} - raw, _ := strconv.ParseBool(c.DefaultQuery("raw", "false")) - if utils.Ext(file.GetName()) == "md" && setting.GetBool(conf.FilterReadMeScripts) && !raw { - buf := bytes.NewBuffer(make([]byte, 0, file.GetSize())) - w := &common.InterceptResponseWriter{ResponseWriter: Writer, Writer: buf} - err = common.Proxy(w, c.Request, link, file) - if err == nil && buf.Len() > 0 { - if c.Writer.Status() < 200 || c.Writer.Status() > 300 { - c.Writer.Write(buf.Bytes()) - return - } - - var html bytes.Buffer - if err = goldmark.Convert(buf.Bytes(), &html); err != nil { - err = fmt.Errorf("markdown conversion failed: %w", err) - } else { - buf.Reset() - err = bluemonday.UGCPolicy().SanitizeReaderToWriter(&html, buf) - if err == nil { - Writer.Header().Set("Content-Length", strconv.FormatInt(int64(buf.Len()), 10)) - Writer.Header().Set("Content-Type", "text/html; charset=utf-8") - _, err = utils.CopyWithBuffer(Writer, buf) - } - } - } - } else { - err = common.Proxy(Writer, c.Request, link, file) - } + err = common.Proxy(Writer, c.Request, link, file) if err == nil { return } diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 162419f7b..28588d668 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -22,6 +22,7 @@ type RecursiveMoveReq struct { ConflictPolicy string `json:"conflict_policy"` } +// FsRecursiveMove recursively moves files (individual item permission checks skipped for performance). func FsRecursiveMove(c *gin.Context) { var req RecursiveMoveReq if err := c.ShouldBind(&req); err != nil { @@ -39,20 +40,31 @@ func FsRecursiveMove(c *gin.Context) { common.ErrorResp(c, err, 403) return } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, srcMeta, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + common.GinWithValue(c, conf.MetaKey, srcMeta) + dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - - meta, err := op.GetNearestMeta(srcDir) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, dstMeta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } - common.GinWithValue(c, conf.MetaKey, meta) rootFiles, err := fs.List(c.Request.Context(), srcDir, &fs.ListArgs{}) if err != nil { @@ -143,6 +155,7 @@ type BatchRenameReq struct { } `json:"rename_objects"` } +// FsBatchRename performs batch rename (individual item permission checks skipped for performance). func FsBatchRename(c *gin.Context) { var req BatchRenameReq if err := c.ShouldBind(&req); err != nil { @@ -162,11 +175,13 @@ func FsBatchRename(c *gin.Context) { } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } common.GinWithValue(c, conf.MetaKey, meta) for _, renameObject := range req.RenameObjects { @@ -193,6 +208,7 @@ type RegexRenameReq struct { NewNameRegex string `json:"new_name_regex"` } +// FsRegexRename renames files by regex (individual item permission checks skipped for performance). func FsRegexRename(c *gin.Context) { var req RegexRenameReq if err := c.ShouldBind(&req); err != nil { @@ -212,11 +228,13 @@ func FsRegexRename(c *gin.Context) { } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } common.GinWithValue(c, conf.MetaKey, meta) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 8247fa8cb..9ead8d60a 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -6,18 +6,18 @@ import ( "strings" "github.com/OpenListTeam/OpenList/v4/internal/conf" - "github.com/OpenListTeam/OpenList/v4/internal/task" - "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/sign" + "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/pkg/generic" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" + log "github.com/sirupsen/logrus" ) type MkdirOrLinkReq struct { @@ -36,18 +36,19 @@ func FsMkdir(c *gin.Context) { common.ErrorResp(c, err, 403) return } - if !user.CanWrite() { - meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } - } - if !common.CanWrite(meta, reqPath) { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } + parentPath := stdpath.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if !common.CanWrite(user, parentMeta, parentPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } if err := fs.MakeDir(c.Request.Context(), reqPath); err != nil { common.ErrorResp(c, err, 500) @@ -65,6 +66,7 @@ type MoveCopyReq struct { Merge bool `json:"merge"` } +// FsMove performs batch move (individual item permission checks skipped for performance). func FsMove(c *gin.Context) { var req MoveCopyReq if err := c.ShouldBind(&req); err != nil { @@ -85,31 +87,62 @@ func FsMove(c *gin.Context) { common.ErrorResp(c, err, 403) return } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, srcMeta, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, dstMeta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } - var validNames []string - if !req.Overwrite { - for _, name := range req.Names { - if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil && !req.SkipExisting { - common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + validPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + // ensure req.Names is not a relative path + srcPath := stdpath.Join(req.SrcDir, name) + srcPath, err = user.JoinPath(srcPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !req.Overwrite { + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) return - } else if res == nil { - validNames = append(validNames, name) + } + if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { + if !req.SkipExisting { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + return + } else { + continue + } } } - } else { - validNames = req.Names + validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo - for i, name := range validNames { - t, err := fs.Move(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(validNames) > i+1) + for i, p := range validPaths { + t, err := fs.Move(c.Request.Context(), p, dstDir, len(validPaths) > i+1) if t != nil { addedTasks = append(addedTasks, t) } @@ -132,6 +165,7 @@ func FsMove(c *gin.Context) { } } +// FsCopy performs batch copy (individual item permission checks skipped for performance). func FsCopy(c *gin.Context) { var req MoveCopyReq if err := c.ShouldBind(&req); err != nil { @@ -152,39 +186,66 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, err, 403) return } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanRead(user, srcMeta, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, dstMeta, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } - var validNames []string - if !req.Overwrite { - for _, name := range req.Names { - if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + validPaths := make([]string, 0, len(req.Names)) + for _, name := range req.Names { + // ensure req.Names is not a relative path + srcPath := stdpath.Join(req.SrcDir, name) + srcPath, err = user.JoinPath(srcPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !req.Overwrite { + base := stdpath.Base(srcPath) + if base == "." || base == "/" { + common.ErrorStrResp(c, fmt.Sprintf("invalid file name [%s]", name), 400) + return + } + if res, _ := fs.Get(c.Request.Context(), stdpath.Join(dstDir, base), &fs.GetArgs{NoLog: true}); res != nil { if !req.SkipExisting && !req.Merge { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return - } else if req.Merge && res.IsDir() { - validNames = append(validNames, name) + } else if !req.Merge || !res.IsDir() { + continue } - } else { - validNames = append(validNames, name) } } - } else { - validNames = req.Names + validPaths = append(validPaths, srcPath) } // Create all tasks immediately without any synchronous validation // All validation will be done asynchronously in the background var addedTasks []task.TaskExtensionInfo - for i, name := range validNames { + for i, p := range validPaths { var t task.TaskExtensionInfo if req.Merge { - t, err = fs.Merge(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(validNames) > i+1) + t, err = fs.Merge(c.Request.Context(), p, dstDir, len(validPaths) > i+1) } else { - t, err = fs.Copy(c.Request.Context(), stdpath.Join(srcDir, name), dstDir, len(validNames) > i+1) + t, err = fs.Copy(c.Request.Context(), p, dstDir, len(validPaths) > i+1) } if t != nil { addedTasks = append(addedTasks, t) @@ -233,6 +294,16 @@ func FsRename(c *gin.Context) { common.ErrorResp(c, err, 403) return } + parentPath := stdpath.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, parentMeta, parentPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) if dstPath != reqPath { @@ -261,6 +332,7 @@ type RemoveReq struct { Names []string `json:"names"` } +// FsRemove performs batch remove (individual item permission checks skipped for performance). func FsRemove(c *gin.Context) { var req RemoveReq if err := c.ShouldBind(&req); err != nil { @@ -276,13 +348,34 @@ func FsRemove(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - reqDir, err := user.JoinPath(req.Dir) + reqPath, err := user.JoinPath(req.Dir) if err != nil { common.ErrorResp(c, err, 403) return } - for _, name := range req.Names { - err := fs.Remove(c.Request.Context(), stdpath.Join(reqDir, name)) + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + for i, name := range req.Names { + fullPath := stdpath.Join(reqPath, name) + if !strings.HasPrefix(fullPath+"/", reqPath+"/") { + log.Warnf("FsRemove: path traversal attempt skipped: %s (dir: %s)\n", name, req.Dir) + req.Names[i] = "" + continue + } + req.Names[i] = fullPath + } + for _, path := range req.Names { + if path == "" { + continue + } + err := fs.Remove(c.Request.Context(), path) if err != nil { common.ErrorResp(c, err, 500) return @@ -296,6 +389,7 @@ type RemoveEmptyDirectoryReq struct { SrcDir string `json:"src_dir"` } +// FsRemoveEmptyDirectory recursively removes empty directories (individual item permission checks skipped for performance). func FsRemoveEmptyDirectory(c *gin.Context) { var req RemoveEmptyDirectoryReq if err := c.ShouldBind(&req); err != nil { @@ -315,11 +409,13 @@ func FsRemoveEmptyDirectory(c *gin.Context) { } meta, err := op.GetNearestMeta(srcDir) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } common.GinWithValue(c, conf.MetaKey, meta) diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 886da9dc9..8a67e4e59 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -47,13 +47,14 @@ type ObjResp struct { } type FsListResp struct { - Content []ObjResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` - DirectUploadTools []string `json:"direct_upload_tools,omitempty"` + Content []ObjResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + WriteContentBypass bool `json:"write_content_bypass"` + Provider string `json:"provider"` + DirectUploadTools []string `json:"direct_upload_tools,omitempty"` } func FsListSplit(c *gin.Context) { @@ -83,18 +84,17 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { return } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } - if !user.CanWrite() && !common.CanWrite(meta, reqPath) && req.Refresh { + canWriteContentAtPath := common.CanWrite(user, meta, reqPath) && (user.CanWriteContent() || common.CanWriteContentBypassUserPerms(meta, reqPath)) + if req.Refresh && !canWriteContentAtPath { common.ErrorStrResp(c, "Refresh without permission", 403) return } @@ -109,19 +109,20 @@ func FsList(c *gin.Context, req *ListReq, user *model.User) { total, objs := pagination(objs, &req.PageReq) provider := "unknown" var directUploadTools []string - if user.CanWrite() { + if canWriteContentAtPath { if storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}); err == nil { directUploadTools = op.GetDirectUploadTools(storage) } } common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), - Total: int64(total), - Readme: getReadme(meta, reqPath), - Header: getHeader(meta, reqPath), - Write: user.CanWrite() || common.CanWrite(meta, reqPath), - Provider: provider, - DirectUploadTools: directUploadTools, + Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), + Total: int64(total), + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Write: common.CanWrite(user, meta, reqPath), + WriteContentBypass: common.CanWriteContentBypassUserPerms(meta, reqPath), + Provider: provider, + DirectUploadTools: directUploadTools, }) } @@ -147,11 +148,9 @@ func FsDirs(c *gin.Context) { reqPath = tmp } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { @@ -186,14 +185,14 @@ func filterDirs(objs []model.Obj) []DirResp { } func getReadme(meta *model.Meta, path string) string { - if meta != nil && (utils.PathEqual(meta.Path, path) || meta.RSub) { + if meta != nil && common.MetaCoversPath(meta.Path, path, meta.RSub) { return meta.Readme } return "" } func getHeader(meta *model.Meta, path string) string { - if meta != nil && (utils.PathEqual(meta.Path, path) || meta.HeaderSub) { + if meta != nil && common.MetaCoversPath(meta.Path, path, meta.HeaderSub) { return meta.Header } return "" @@ -206,7 +205,7 @@ func isEncrypt(meta *model.Meta, path string) bool { if meta == nil || meta.Password == "" { return false } - if !utils.PathEqual(meta.Path, path) && !meta.PSub { + if !common.MetaCoversPath(meta.Path, path, meta.PSub) { return false } return true @@ -231,6 +230,10 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { for _, obj := range objs { thumb, _ := model.GetThumb(obj) mountDetails, _ := model.GetStorageDetails(obj) + hashInfo := obj.GetHash().Export() + if hashInfo == nil { + hashInfo = make(map[*utils.HashType]string) + } resp = append(resp, ObjResp{ Name: obj.GetName(), Size: obj.GetSize(), @@ -238,7 +241,7 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { Modified: obj.ModTime(), Created: obj.CreateTime(), HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), + HashInfo: hashInfo, Sign: common.Sign(obj, parent, encrypt), Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), @@ -288,11 +291,9 @@ func FsGet(c *gin.Context, req *FsGetReq, user *model.User) { return } meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, reqPath, req.Password) { @@ -414,11 +415,9 @@ func FsOther(c *gin.Context) { return } meta, err := op.GetNearestMeta(req.Path) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500) + return } common.GinWithValue(c, conf.MetaKey, meta) if !common.CanAccess(user, meta, req.Path, req.Password) { diff --git a/server/handles/fsread_test.go b/server/handles/fsread_test.go new file mode 100644 index 000000000..3947ae27f --- /dev/null +++ b/server/handles/fsread_test.go @@ -0,0 +1,255 @@ +package handles + +import ( + "testing" + + "github.com/OpenListTeam/OpenList/v4/internal/model" +) + +func TestGetReadme(t *testing.T) { + tests := []struct { + name string + meta *model.Meta + path string + want string + reason string + }{ + { + name: "nil meta", + meta: nil, + path: "/any", + want: "", + reason: "nil meta should return empty", + }, + { + name: "exact path match with RSub=false", + meta: &model.Meta{ + Path: "/folder", + Readme: "Welcome", + RSub: false, + }, + path: "/folder", + want: "Welcome", + reason: "exact path should show readme", + }, + { + name: "sub path with RSub=true", + meta: &model.Meta{ + Path: "/folder", + Readme: "Welcome", + RSub: true, + }, + path: "/folder/subfolder", + want: "Welcome", + reason: "sub path with RSub=true should show readme", + }, + { + name: "sub path with RSub=false", + meta: &model.Meta{ + Path: "/folder", + Readme: "Welcome", + RSub: false, + }, + path: "/folder/subfolder", + want: "", + reason: "sub path with RSub=false should not show readme", + }, + { + name: "non-sub path with RSub=true (BEHAVIOR CHANGE - BUG FIX)", + meta: &model.Meta{ + Path: "/folder", + Readme: "Welcome", + RSub: true, + }, + path: "/other", + want: "", + reason: "non-sub path should not show readme even with RSub=true (fixed bug)", + }, + { + name: "root readme applies to all with RSub=true", + meta: &model.Meta{ + Path: "/", + Readme: "Global Info", + RSub: true, + }, + path: "/any/path", + want: "Global Info", + reason: "root readme with RSub=true should apply to all paths", + }, + { + name: "empty readme", + meta: &model.Meta{ + Path: "/folder", + Readme: "", + RSub: true, + }, + path: "/folder", + want: "", + reason: "empty readme should return empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getReadme(tt.meta, tt.path) + if got != tt.want { + t.Errorf("getReadme() = %q, want %q\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} + +func TestGetHeader(t *testing.T) { + tests := []struct { + name string + meta *model.Meta + path string + want string + reason string + }{ + { + name: "nil meta", + meta: nil, + path: "/any", + want: "", + reason: "nil meta should return empty", + }, + { + name: "exact path match with HeaderSub=false", + meta: &model.Meta{ + Path: "/folder", + Header: "Custom Header", + HeaderSub: false, + }, + path: "/folder", + want: "Custom Header", + reason: "exact path should show header", + }, + { + name: "sub path with HeaderSub=true", + meta: &model.Meta{ + Path: "/folder", + Header: "Custom Header", + HeaderSub: true, + }, + path: "/folder/subfolder", + want: "Custom Header", + reason: "sub path with HeaderSub=true should show header", + }, + { + name: "sub path with HeaderSub=false", + meta: &model.Meta{ + Path: "/folder", + Header: "Custom Header", + HeaderSub: false, + }, + path: "/folder/subfolder", + want: "", + reason: "sub path with HeaderSub=false should not show header", + }, + { + name: "non-sub path with HeaderSub=true (BEHAVIOR CHANGE - BUG FIX)", + meta: &model.Meta{ + Path: "/folder", + Header: "Custom Header", + HeaderSub: true, + }, + path: "/other", + want: "", + reason: "non-sub path should not show header even with HeaderSub=true (fixed bug)", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getHeader(tt.meta, tt.path) + if got != tt.want { + t.Errorf("getHeader() = %q, want %q\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} + +func TestIsEncrypt(t *testing.T) { + tests := []struct { + name string + meta *model.Meta + path string + want bool + reason string + }{ + { + name: "nil meta", + meta: nil, + path: "/any", + want: false, + reason: "nil meta should not be encrypted", + }, + { + name: "empty password", + meta: &model.Meta{ + Path: "/folder", + Password: "", + }, + path: "/folder", + want: false, + reason: "empty password should not be encrypted", + }, + { + name: "exact path match with PSub=false", + meta: &model.Meta{ + Path: "/folder", + Password: "secret", + PSub: false, + }, + path: "/folder", + want: true, + reason: "exact path with password should be encrypted", + }, + { + name: "sub path with PSub=true", + meta: &model.Meta{ + Path: "/folder", + Password: "secret", + PSub: true, + }, + path: "/folder/subfolder", + want: true, + reason: "sub path with PSub=true should be encrypted", + }, + { + name: "sub path with PSub=false", + meta: &model.Meta{ + Path: "/folder", + Password: "secret", + PSub: false, + }, + path: "/folder/subfolder", + want: false, + reason: "sub path with PSub=false should not be encrypted", + }, + { + name: "non-sub path with PSub=true", + meta: &model.Meta{ + Path: "/folder", + Password: "secret", + PSub: true, + }, + path: "/other", + want: false, + reason: "non-sub path should not be encrypted even with PSub=true", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isEncrypt(tt.meta, tt.path) + if got != tt.want { + t.Errorf("isEncrypt() = %v, want %v\nReason: %s", + got, tt.want, tt.reason) + } + }) + } +} diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 0f46398cd..54cdb4fee 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -93,6 +93,12 @@ func FsStream(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := c.GetHeader("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) @@ -190,6 +196,12 @@ func FsForm(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + if sha1_128k := c.GetHeader("X-File-Sha1-128k"); sha1_128k != "" { + h[utils.SHA1_128K] = sha1_128k + } + if preHash := c.GetHeader("X-File-Pre-Hash"); preHash != "" { + h[utils.PRE_HASH] = preHash + } mimetype := file.Header.Get("Content-Type") if len(mimetype) == 0 { mimetype = utils.GetMimeType(name) diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 153b27293..32fa64a42 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -12,12 +12,14 @@ import ( "github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser" "github.com/OpenListTeam/OpenList/v4/drivers/thunderx" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool" "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/task" "github.com/OpenListTeam/OpenList/v4/server/common" "github.com/gin-gonic/gin" + "github.com/pkg/errors" ) type SetAria2Req struct { @@ -448,6 +450,7 @@ func SetThunderBrowser(c *gin.Context) { case *thunder_browser.ThunderBrowser, *thunder_browser.ThunderBrowserExpert: default: common.ErrorStrResp(c, "unsupported storage driver for offline download, only ThunderBrowser is supported", 400) + return } } items := []model.SettingItem{ @@ -498,6 +501,15 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 403) return } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + if !common.CanWrite(user, meta, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } var tasks []task.TaskExtensionInfo for _, url := range req.Urls { // Filter out empty lines and whitespace-only strings diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index a36e79d38..4baabf6c1 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -256,11 +256,13 @@ func OIDCLoginCallback(c *gin.Context) { user, err = autoRegister(userID, userID, err) if err != nil { common.ErrorResp(c, err, 400) + return } } token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) + return } if useCompatibility { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) @@ -427,6 +429,7 @@ func SSOLoginCallback(c *gin.Context) { token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) + return } if usecompatibility { c.Redirect(302, common.GetApiUrl(c)+"/@login?token="+token) diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index c7ad4edfe..b2a0fbfb4 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -130,17 +130,20 @@ func BeginAuthnRegistration(c *gin.Context) { authnInstance, err := authn.NewAuthnInstance(c) if err != nil { common.ErrorResp(c, err, 400) + return } options, sessionData, err := authnInstance.BeginRegistration(user) if err != nil { common.ErrorResp(c, err, 400) + return } val, err := json.Marshal(sessionData) if err != nil { common.ErrorResp(c, err, 400) + return } common.SuccessResp(c, gin.H{ diff --git a/server/middlewares/down.go b/server/middlewares/down.go index cb87eb3c3..c1f81b54b 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -25,11 +25,9 @@ func Down(verifyFunc func(string, string) error) func(c *gin.Context) { return func(c *gin.Context) { rawPath := c.Request.Context().Value(conf.PathKey).(string) meta, err := op.GetNearestMeta(rawPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorPage(c, err, 500, true) - return - } + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorPage(c, err, 500, true) + return } common.GinWithValue(c, conf.MetaKey, meta) // verify sign diff --git a/server/middlewares/fsup.go b/server/middlewares/fsup.go index 08b160ee5..d99e62aea 100644 --- a/server/middlewares/fsup.go +++ b/server/middlewares/fsup.go @@ -15,7 +15,6 @@ import ( func FsUp(c *gin.Context) { path := c.GetHeader("File-Path") - password := c.GetHeader("Password") path, err := url.PathUnescape(path) if err != nil { common.ErrorResp(c, err, 400) @@ -28,15 +27,19 @@ func FsUp(c *gin.Context) { common.ErrorResp(c, err, 403) return } - meta, err := op.GetNearestMeta(stdpath.Dir(path)) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - c.Abort() - return - } + parentPath := stdpath.Dir(path) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + c.Abort() + return + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + c.Abort() + return } - if !(common.CanAccess(user, meta, path, password) && (user.CanWrite() || common.CanWrite(meta, stdpath.Dir(path)))) { + if !common.CanWrite(user, parentMeta, parentPath) { common.ErrorResp(c, errs.PermissionDenied, 403) c.Abort() return diff --git a/server/static/static.go b/server/static/static.go index 29f97ff74..3850b6d11 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -1,6 +1,7 @@ package static import ( + "context" "encoding/json" "errors" "fmt" @@ -8,10 +9,14 @@ import ( "io/fs" "net/http" "os" + "path/filepath" "strings" + "sync" + "time" "github.com/OpenListTeam/OpenList/v4/drivers/base" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/frontend" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/pkg/utils" "github.com/OpenListTeam/OpenList/v4/public" @@ -32,21 +37,60 @@ type Manifest struct { Icons []ManifestIcon `json:"icons"` } -var static fs.FS +// reloadableFS wraps fs.FS with thread-safe swapping. +// This allows gin StaticFS routes (which capture the fs.FS at registration time) +// to serve updated files after a watcher-triggered reload. +type reloadableFS struct { + mu sync.RWMutex + current fs.FS +} + +func (r *reloadableFS) Open(name string) (fs.File, error) { + r.mu.RLock() + current := r.current + r.mu.RUnlock() + return current.Open(name) +} + +func (r *reloadableFS) swap(f fs.FS) { + r.mu.Lock() + r.current = f + r.mu.Unlock() +} + +var staticFS = &reloadableFS{} func initStatic() { utils.Log.Debug("Initializing static file system...") - if conf.Conf.DistDir == "" { - dist, err := fs.Sub(public.Public, "dist") - if err != nil { - utils.Log.Fatalf("failed to read dist dir: %v", err) - } - static = dist - utils.Log.Debug("Using embedded dist directory") + // 1. User explicitly configured dist_dir + if conf.Conf.DistDir != "" { + staticFS.swap(os.DirFS(conf.Conf.DistDir)) + utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) + return + } + // 2. Try dynamic dist (fetched from rolling release) + if frontend.HasValidDist() { + distPath := filepath.Join(frontend.GetDistPath(), "dist") + staticFS.swap(os.DirFS(distPath)) + utils.Log.Infof("Using dynamically fetched dist: %s", distPath) + return + } + // 3. Try auto-fetching from rolling (short timeout to avoid blocking startup) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + distPath := frontend.EnsureDistOnce(ctx) + cancel() + if distPath != "" { + staticFS.swap(os.DirFS(distPath)) + utils.Log.Infof("Using auto-fetched dist: %s", distPath) return } - static = os.DirFS(conf.Conf.DistDir) - utils.Log.Infof("Using custom dist directory: %s", conf.Conf.DistDir) + // 4. Final fallback to embedded dist + dist, err := fs.Sub(public.Public, "dist") + if err != nil { + utils.Log.Fatalf("failed to read dist dir: %v", err) + } + staticFS.swap(dist) + utils.Log.Debug("Using embedded dist directory") } func replaceStrings(content string, replacements map[string]string) string { @@ -74,7 +118,7 @@ func initIndex(siteConfig SiteConfig) { utils.Log.Info("Successfully fetched index.html from CDN") } else { utils.Log.Debug("Reading index.html from static files system...") - indexFile, err := static.Open("index.html") + indexFile, err := staticFS.Open("index.html") if err != nil { if errors.Is(err, fs.ErrNotExist) { utils.Log.Fatalf("index.html not exist, you may forget to put dist of frontend to public/dist") @@ -131,13 +175,21 @@ func UpdateIndex() { utils.Log.Debug("Index.html update completed") } +// ReloadStatic reloads the static files from disk (called by the watcher after an update) +func ReloadStatic() { + utils.Log.Info("[static] reloading static files after frontend update...") + siteConfig := getSiteConfig() + initStatic() + initIndex(siteConfig) +} + func ManifestJSON(c *gin.Context) { // Get site configuration to ensure consistent base path handling siteConfig := getSiteConfig() - + // Get site title from settings siteTitle := setting.GetStr(conf.SiteTitle) - + // Get logo from settings, use the first line (light theme logo) logoSetting := setting.GetStr(conf.Logo) logoUrl := strings.Split(logoSetting, "\n")[0] @@ -167,7 +219,7 @@ func ManifestJSON(c *gin.Context) { c.Header("Content-Type", "application/json") c.Header("Cache-Control", "public, max-age=3600") // cache for 1 hour - + if err := json.NewEncoder(c.Writer).Encode(manifest); err != nil { utils.Log.Errorf("Failed to encode manifest.json: %v", err) c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to generate manifest"}) @@ -180,8 +232,12 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { siteConfig := getSiteConfig() initStatic() initIndex(siteConfig) + + // Start the frontend watcher for periodic updates + frontend.StartWatcher(ReloadStatic) + folders := []string{"assets", "images", "streamer", "static"} - + if conf.Conf.Cdn == "" { utils.Log.Debug("Setting up static file serving...") r.Use(func(c *gin.Context) { @@ -192,7 +248,7 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { } }) for _, folder := range folders { - sub, err := fs.Sub(static, folder) + sub, err := fs.Sub(staticFS, folder) if err != nil { utils.Log.Fatalf("can't find folder: %s", folder) } diff --git a/server/webdav.go b/server/webdav.go index 789236b8b..a949068f0 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -117,22 +117,22 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!user.CanWebdavManage() || !user.CanWrite()) { + if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "MOVE" && (!user.CanWebdavManage() || (!user.CanMove() && !user.CanRename())) { + if c.Request.Method == "MOVE" && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "COPY" && (!user.CanWebdavManage() || !user.CanCopy()) { + if c.Request.Method == "COPY" && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "DELETE" && (!user.CanWebdavManage() || !user.CanRemove()) { + if c.Request.Method == "DELETE" && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return @@ -143,6 +143,11 @@ func WebDAVAuth(c *gin.Context) { return } common.GinWithValue(c, conf.UserKey, user) + if user.IsGuest() { + common.GinWithValue(c, conf.MetaPassKey, password) + } else { + common.GinWithValue(c, conf.MetaPassKey, "") + } c.Next() } diff --git a/server/webdav/file.go b/server/webdav/file.go index debfcfe9e..ea6099735 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -11,9 +11,12 @@ import ( "path/filepath" "github.com/OpenListTeam/OpenList/v4/internal/conf" + "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" + "github.com/OpenListTeam/OpenList/v4/server/common" + "github.com/pkg/errors" ) // slashClean is equivalent to but slightly more efficient than @@ -26,6 +29,7 @@ func slashClean(name string) string { } // moveFiles moves files and/or directories from src to dst. +// Individual item permission checks are skipped for performance reasons. // // See section 9.9.4 for when various HTTP status codes apply. func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int, err error) { @@ -40,6 +44,17 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int if srcName != dstName && !user.CanRename() { return http.StatusForbidden, nil } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, srcMeta, srcDir) || !common.CanWrite(user, dstMeta, dstDir) { + return http.StatusForbidden, nil + } if srcDir == dstDir { err = fs.Rename(ctx, src, dstName) } else { @@ -59,10 +74,30 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int } // copyFiles copies files and/or directories from src to dst. +// Individual item permission checks are skipped for performance reasons. // // See section 9.8.5 for when various HTTP status codes apply. func copyFiles(ctx context.Context, src, dst string, overwrite bool) (status int, err error) { + srcDir := path.Dir(src) dstDir := path.Dir(dst) + user := ctx.Value(conf.UserKey).(*model.User) + if !user.CanCopy() { + return http.StatusForbidden, nil + } + srcMeta, err := op.GetNearestMeta(srcDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanRead(user, srcMeta, srcDir) { + return http.StatusForbidden, nil + } + dstMeta, err := op.GetNearestMeta(dstDir) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, dstMeta, dstDir) { + return http.StatusForbidden, nil + } _, err = fs.Copy(context.WithValue(ctx, conf.NoTaskKey, struct{}{}), src, dstDir) if err != nil { return http.StatusInternalServerError, err diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 504c5fc1d..06d1431ac 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -7,7 +7,6 @@ package webdav // import "golang.org/x/net/webdav" import ( "context" - "errors" "fmt" "io" "net/http" @@ -20,8 +19,10 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/conf" "github.com/OpenListTeam/OpenList/v4/internal/net" + "github.com/OpenListTeam/OpenList/v4/internal/op" "github.com/OpenListTeam/OpenList/v4/internal/setting" "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/pkg/errors" "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/fs" @@ -200,7 +201,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err } allow := "OPTIONS, LOCK, PUT, MKCOL" if fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil { @@ -226,10 +227,18 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta // TODO: check locks for read-only access?? ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) + password, _ := ctx.Value(conf.MetaPassKey).(string) reqPath, err = user.JoinPath(reqPath) if err != nil { return http.StatusForbidden, err } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanAccess(user, meta, reqPath, password) { + return http.StatusForbidden, errs.PermissionDenied + } fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { return http.StatusNotFound, err @@ -294,9 +303,12 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i ctx := r.Context() user := ctx.Value(conf.UserKey).(*model.User) + if !user.CanRemove() { + return http.StatusForbidden, nil + } reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err } // TODO: return MultiStatus where appropriate. @@ -309,6 +321,14 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i } return http.StatusMethodNotAllowed, err } + parentPath := path.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } if err := fs.Remove(ctx, reqPath); err != nil { return http.StatusMethodNotAllowed, err } @@ -363,6 +383,17 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if setting.GetBool(conf.IgnoreSystemFiles) && utils.IsSystemFile(obj.Name) { return http.StatusForbidden, errs.IgnoredSystemFile } + parentPath := path.Dir(reqPath) + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } + if !common.CanWrite(user, parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } fsStream := &stream.FileStream{ Obj: &obj, Reader: r.Body, @@ -407,7 +438,7 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err } if r.ContentLength > 0 { @@ -421,13 +452,23 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in } // RFC 4918 9.3.1 // 409 (Conflict) The server MUST NOT create those intermediate collections automatically. - reqDir := path.Dir(reqPath) - if _, err := fs.Get(ctx, reqDir, &fs.GetArgs{}); err != nil { + parentPath := path.Dir(reqPath) + if _, err := fs.Get(ctx, parentPath, &fs.GetArgs{}); err != nil { if errs.IsObjectNotFound(err) { return http.StatusConflict, err } return http.StatusMethodNotAllowed, err } + parentMeta, err := op.GetNearestMeta(parentPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } + if !common.CanWrite(user, parentMeta, parentPath) { + return http.StatusForbidden, errs.PermissionDenied + } if err := fs.MakeDir(ctx, reqPath); err != nil { if os.IsNotExist(err) { return http.StatusConflict, err @@ -471,11 +512,11 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status user := ctx.Value(conf.UserKey).(*model.User) src, err = user.JoinPath(src) if err != nil { - return 403, err + return http.StatusForbidden, err } dst, err = user.JoinPath(dst) if err != nil { - return 403, err + return http.StatusForbidden, err } if r.Method == "COPY" { @@ -572,7 +613,14 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus } reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, meta, reqPath) { + return http.StatusForbidden, errs.PermissionDenied } ld = LockDetails{ Root: reqPath, @@ -630,6 +678,24 @@ func (h *Handler) handleUnlock(w http.ResponseWriter, r *http.Request) (status i } t = t[1 : len(t)-1] + reqPath, status, err := h.stripPrefix(r.URL.Path) + if err != nil { + return status, err + } + ctx := r.Context() + user := ctx.Value(conf.UserKey).(*model.User) + reqPath, err = user.JoinPath(reqPath) + if err != nil { + return http.StatusForbidden, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, meta, reqPath) { + return http.StatusForbidden, errs.PermissionDenied + } + switch err = h.LockSystem.Unlock(time.Now(), t); err { case nil: return http.StatusNoContent, err @@ -653,9 +719,17 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status userAgent := r.Header.Get("User-Agent") ctx = context.WithValue(ctx, conf.UserAgentKey, userAgent) user := ctx.Value(conf.UserKey).(*model.User) + password, _ := ctx.Value(conf.MetaPassKey).(string) reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanAccess(user, meta, reqPath, password) { + return http.StatusForbidden, errs.PermissionDenied } fi, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { @@ -734,7 +808,14 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu user := ctx.Value(conf.UserKey).(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return http.StatusForbidden, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return http.StatusInternalServerError, err + } + if !common.CanWrite(user, meta, reqPath) { + return http.StatusForbidden, errs.PermissionDenied } if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err != nil { if errs.IsObjectNotFound(err) {