diff --git a/.github/workflows/test_docker.yml b/.github/workflows/test_docker.yml index aa6fe8966..33c644c90 100644 --- a/.github/workflows/test_docker.yml +++ b/.github/workflows/test_docker.yml @@ -1,5 +1,4 @@ name: Beta Release (Docker) - on: workflow_dispatch: push: @@ -7,51 +6,51 @@ on: - main pull_request: branches: - - main + - fix # 👈 允许你的 fix 分支触发 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: - DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }} - GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }} - IMAGE_NAME: openlist-git - IMAGE_NAME_DOCKERHUB: openlist + GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名，防止推错地方 + IMAGE_NAME: openlist REGISTRY: ghcr.io ARTIFACT_NAME: 'binaries_docker_release' - RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons. - IMAGE_PUSH: ${{ github.event_name == 'push' }} + # 👇 关键修改：只保留 linux/amd64，删掉后面一长串 + RELEASE_PLATFORMS: 'linux/amd64' + # 👇 关键修改：强制允许推送，不用管是不是 push 事件 + IMAGE_PUSH: 'true' IMAGE_TAGS_BETA: | type=ref,event=pr type=raw,value=beta,enable={{is_default_branch}} jobs: build_binary: - name: Build Binaries for Docker Release (Beta) + name: Build Binaries (x64 Only) runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-go@v5 with: go-version: '1.25.0' + # 即使只构建 x64，我们也需要 musl 工具链（因为 BuildDockerMultiplatform 默认会检查它） - name: Cache Musl id: cache-musl uses: actions/cache@v4 with: path: build/musl-libs key: docker-musl-libs-v2 - - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' run: bash build.sh prepare docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build go binary (beta) + - name: Build go binary + # 这里还是跑 docker-multiplatform，虽然会多编译一些架构，但这是兼容 Dockerfile 路径最稳妥的方法 run: bash build.sh beta docker-multiplatform env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -69,12 +68,13 @@ jobs: release_docker: needs: build_binary - name: Release Docker image (Beta) + name: Release Docker (x64) runs-on: ubuntu-latest permissions: packages: write strategy: matrix: + # 你可以选择只构建 latest，或者保留全部变体 image: ["latest", "ffmpeg", "aria2", "aio"] include: - image: "latest" @@ -102,46 +102,32 @@ jobs: with: name: ${{ env.ARTIFACT_NAME }} path: 'build/' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + # 👇 只保留 GitHub 登录，删除了 DockerHub 登录 - name: Login to GitHub Container Registry - if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to DockerHub Container Registry - if: env.IMAGE_PUSH == 'true' - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: | ${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }} - ${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }} tags: ${{ env.IMAGE_TAGS_BETA }} - flavor: | - ${{ matrix.tag_favor }} + flavor: ${{ matrix.tag_favor }} - name: Build and push - id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci - push: ${{ env.IMAGE_PUSH == 'true' }} + push: true build-args: | BASE_IMAGE_TAG=${{ matrix.base_image_tag }} ${{ matrix.build_arg }} diff --git a/build.sh b/build.sh index 26e5a301b..0e8f4b85d 100644 --- a/build.sh +++ b/build.sh @@ -186,8 +186,8 @@ BuildDockerMultiplatform() { docker_lflags="--extldflags '-static -fpic' $ldflags" export CGO_ENABLED=1 - OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds - CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds + OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds + CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} @@ -205,14 +205,14 @@ BuildDockerMultiplatform() { GO_ARM=(6 7) export GOOS=linux export GOARCH=arm - for i in "${!DOCKER_ARM_ARCHES[@]}"; do - docker_arch=${DOCKER_ARM_ARCHES[$i]} - cgo_cc=${CGO_ARGS[$i]} - export GOARM=${GO_ARM[$i]} - export CC=${cgo_cc} - echo "building for $docker_arch" - go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . - done + # for i in "${!DOCKER_ARM_ARCHES[@]}"; do + # docker_arch=${DOCKER_ARM_ARCHES[$i]} + # cgo_cc=${CGO_ARGS[$i]} + # export GOARM=${GO_ARM[$i]} + # export CC=${cgo_cc} + # echo "building for $docker_arch" + # go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter . + # done } BuildRelease() { diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index ac75e51d7..611ec9e35 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -2,7 +2,9 @@ package _123_open import ( "context" + "encoding/hex" "fmt" + "io" "strconv" "time" @@ -10,7 +12,7 @@ import ( "github.com/OpenListTeam/OpenList/v4/internal/errs" "github.com/OpenListTeam/OpenList/v4/internal/model" "github.com/OpenListTeam/OpenList/v4/internal/op" - "github.com/OpenListTeam/OpenList/v4/internal/stream" + "github.com/OpenListTeam/OpenList/v4/pkg/http_range" "github.com/OpenListTeam/OpenList/v4/pkg/utils" ) @@ -156,21 +158,66 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { } func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // 1. 创建文件 + // 1. 准备参数 // parentFileID 父目录id，上传到根目录时填写 0 parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) if err != nil { return nil, fmt.Errorf("parse parentFileID error: %v", err) } - // etag 文件md5 + + // 1. 检查是否有etag（注意：etag默认是不可靠的） etag := file.GetHash().GetHash(utils.MD5) - if len(etag) < utils.MD5.Width { - _, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5) + hasEtag := len(etag) > 0 + + var createResp *UploadCreateResp // 用于最后的complete调用 + + if hasEtag { + // 有etag（但不可靠）：先尝试用它秒传，如果失败则流式计算真实MD5 + createResp, err = d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err } + // 是否秒传 + if createResp.Data.Reuse { + // 秒传成功才会返回正确的 FileID，否则为 0 + if createResp.Data.FileID != 0 { + return File{ + FileName: file.GetName(), + Size: file.GetSize(), + FileId: createResp.Data.FileID, + Type: 2, + Etag: etag, + }, nil + } + } + // 秒传失败，说明etag不可靠，需要流式计算真实MD5 + // 注意：此时不能上传，因为etag可能是错的 } - createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + + // 没有etag或秒传失败：流式计算MD5 + md5Hash := utils.MD5.NewFunc() + size := file.GetSize() + chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk for MD5 calculation + var offset int64 = 0 + for offset < size { + readSize := min(chunkSize, size-offset) + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize}) + if err != nil { + return nil, fmt.Errorf("range read for MD5 calculation failed: %w", err) + } + if _, err := io.Copy(md5Hash, reader); err != nil { + return nil, fmt.Errorf("calculate MD5 failed: %w", err) + } + offset += readSize + + progress := 40 * float64(offset) / float64(size) + up(progress) + } + + etag = hex.EncodeToString(md5Hash.Sum(nil)) + + // 创建上传任务（使用真实计算的MD5） + createResp, err = d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) if err != nil { return nil, err } @@ -188,13 +235,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre } } - // 2. 上传分片 - err = d.Upload(ctx, file, createResp, up) + // 上传分片（不需要计算哈希了，已经计算过了） + uploadProgress := func(p float64) { + up(40 + p*0.5) + } + _, err = d.Upload(ctx, file, createResp, uploadProgress, false) if err != nil { return nil, err } - // 3. 上传完毕 + // 2. 合并分片/完成上传 for range 60 { uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) // 返回错误代码未知，如：20103，文档也没有具体说 diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go index 9bfc081a7..65ae8d5ed 100644 --- a/drivers/123_open/upload.go +++ b/drivers/123_open/upload.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "hash" "io" "mime/multipart" "net/http" @@ -42,14 +43,21 @@ func (d *Open123) create(parentFileID int64, filename string, etag string, size } // 上传分片 V2 -func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createResp *UploadCreateResp, up driver.UpdateProgress) error { +// 如果needHash为true，会在上传过程中计算文件的MD5哈希，并通过返回值返回 +func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createResp *UploadCreateResp, up driver.UpdateProgress, needHash bool) (string, error) { uploadDomain := createResp.Data.Servers[0] size := file.GetSize() chunkSize := createResp.Data.SliceSize ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up) if err != nil { - return err + return "", err + } + + // 如果需要计算哈希，创建哈希计算器 + var md5Hash hash.Hash + if needHash { + md5Hash = utils.MD5.NewFunc() } uploadNums := (size + chunkSize - 1) / chunkSize @@ -88,6 +96,15 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes reader.Seek(0, io.SeekStart) } + // 如果需要计算整体文件哈希，在这里将分片数据写入哈希计算器 + if needHash { + _, err = io.Copy(md5Hash, reader) + if err != nil { + return err + } + reader.Seek(0, io.SeekStart) + } + b.Reset() w := multipart.NewWriter(b) // 添加表单字段 @@ -160,10 +177,14 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes } if err := threadG.Wait(); err != nil { - return err + return "", err } - return nil + // 如果需要计算哈希，返回计算结果 + if needHash { + return fmt.Sprintf("%x", md5Hash.Sum(nil)), nil + } + return "", nil } // 上传完毕