Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 16 additions & 30 deletions .github/workflows/test_docker.yml
Original file line number Diff line number Diff line change
@@ -1,57 +1,56 @@
name: Beta Release (Docker)

on:
workflow_dispatch:
push:
branches:
- main
pull_request:
branches:
- main
- fix # 👈 允许你的 fix 分支触发

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

env:
DOCKERHUB_ORG_NAME: ${{ vars.DOCKERHUB_ORG_NAME || 'openlistteam' }}
GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'openlistteam' }}
IMAGE_NAME: openlist-git
IMAGE_NAME_DOCKERHUB: openlist
GHCR_ORG_NAME: ${{ vars.GHCR_ORG_NAME || 'ironboxplus' }} # 👈 最好改成你的用户名,防止推错地方
IMAGE_NAME: openlist
REGISTRY: ghcr.io
ARTIFACT_NAME: 'binaries_docker_release'
RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/ppc64le,linux/riscv64,linux/loong64' ### Temporarily disable Docker builds for linux/s390x architectures for unknown reasons.
IMAGE_PUSH: ${{ github.event_name == 'push' }}
# 👇 关键修改:只保留 linux/amd64,删掉后面一长串
RELEASE_PLATFORMS: 'linux/amd64'
# 👇 关键修改:强制允许推送,不用管是不是 push 事件
IMAGE_PUSH: 'true'
IMAGE_TAGS_BETA: |
type=ref,event=pr
type=raw,value=beta,enable={{is_default_branch}}

jobs:
build_binary:
name: Build Binaries for Docker Release (Beta)
name: Build Binaries (x64 Only)
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- uses: actions/setup-go@v5
with:
go-version: '1.25.0'

# 即使只构建 x64,我们也需要 musl 工具链(因为 BuildDockerMultiplatform 默认会检查它)
- name: Cache Musl
id: cache-musl
uses: actions/cache@v4
with:
path: build/musl-libs
key: docker-musl-libs-v2

- name: Download Musl Library
if: steps.cache-musl.outputs.cache-hit != 'true'
run: bash build.sh prepare docker-multiplatform
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Build go binary (beta)
- name: Build go binary
# 这里还是跑 docker-multiplatform,虽然会多编译一些架构,但这是兼容 Dockerfile 路径最稳妥的方法
run: bash build.sh beta docker-multiplatform
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -69,12 +68,13 @@ jobs:

release_docker:
needs: build_binary
name: Release Docker image (Beta)
name: Release Docker (x64)
runs-on: ubuntu-latest
permissions:
packages: write
strategy:
matrix:
# 你可以选择只构建 latest,或者保留全部变体
image: ["latest", "ffmpeg", "aria2", "aio"]
include:
- image: "latest"
Expand Down Expand Up @@ -102,46 +102,32 @@ jobs:
with:
name: ${{ env.ARTIFACT_NAME }}
path: 'build/'

- name: Set up QEMU
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

# 👇 只保留 GitHub 登录,删除了 DockerHub 登录
- name: Login to GitHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Login to DockerHub Container Registry
if: env.IMAGE_PUSH == 'true'
uses: docker/login-action@v3
with:
username: ${{ vars.DOCKERHUB_ORG_NAME_BACKUP || env.DOCKERHUB_ORG_NAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: |
${{ env.REGISTRY }}/${{ env.GHCR_ORG_NAME }}/${{ env.IMAGE_NAME }}
${{ env.DOCKERHUB_ORG_NAME }}/${{ env.IMAGE_NAME_DOCKERHUB }}
tags: ${{ env.IMAGE_TAGS_BETA }}
flavor: |
${{ matrix.tag_favor }}
flavor: ${{ matrix.tag_favor }}

- name: Build and push
id: docker_build
uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile.ci
push: ${{ env.IMAGE_PUSH == 'true' }}
push: true
build-args: |
BASE_IMAGE_TAG=${{ matrix.base_image_tag }}
${{ matrix.build_arg }}
Expand Down
20 changes: 10 additions & 10 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -186,8 +186,8 @@ BuildDockerMultiplatform() {
docker_lflags="--extldflags '-static -fpic' $ldflags"
export CGO_ENABLED=1

OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-riscv64 linux-ppc64le linux-loong64) ## Disable linux-s390x builds
CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc loongarch64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds
OS_ARCHES=(linux-amd64) ## Disable linux-s390x builds
CGO_ARGS=(x86_64-linux-musl-gcc) ## Disable s390x-linux-musl-gcc builds
for i in "${!OS_ARCHES[@]}"; do
os_arch=${OS_ARCHES[$i]}
cgo_cc=${CGO_ARGS[$i]}
Expand All @@ -205,14 +205,14 @@ BuildDockerMultiplatform() {
GO_ARM=(6 7)
export GOOS=linux
export GOARCH=arm
for i in "${!DOCKER_ARM_ARCHES[@]}"; do
docker_arch=${DOCKER_ARM_ARCHES[$i]}
cgo_cc=${CGO_ARGS[$i]}
export GOARM=${GO_ARM[$i]}
export CC=${cgo_cc}
echo "building for $docker_arch"
go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter .
done
# for i in "${!DOCKER_ARM_ARCHES[@]}"; do
# docker_arch=${DOCKER_ARM_ARCHES[$i]}
# cgo_cc=${CGO_ARGS[$i]}
# export GOARM=${GO_ARM[$i]}
# export CC=${cgo_cc}
# echo "building for $docker_arch"
# go build -o build/${docker_arch%%-*}/${docker_arch##*-}/"$appName" -ldflags="$docker_lflags" -tags=jsoniter .
# done
}

BuildRelease() {
Expand Down
68 changes: 59 additions & 9 deletions drivers/123_open/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,17 @@ package _123_open

import (
"context"
"encoding/hex"
"fmt"
"io"
"strconv"
"time"

"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)

Expand Down Expand Up @@ -156,21 +158,66 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error {
}

func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
// 1. 创建文件
// 1. 准备参数
// parentFileID 父目录id,上传到根目录时填写 0
parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64)
if err != nil {
return nil, fmt.Errorf("parse parentFileID error: %v", err)
}
// etag 文件md5

// 1. 检查是否有etag(注意:etag默认是不可靠的)
etag := file.GetHash().GetHash(utils.MD5)
if len(etag) < utils.MD5.Width {
_, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5)
hasEtag := len(etag) > 0

var createResp *UploadCreateResp // 用于最后的complete调用

if hasEtag {
// 有etag(但不可靠):先尝试用它秒传,如果失败则流式计算真实MD5
createResp, err = d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false)
if err != nil {
return nil, err
}
// 是否秒传
if createResp.Data.Reuse {
// 秒传成功才会返回正确的 FileID,否则为 0
if createResp.Data.FileID != 0 {
return File{
FileName: file.GetName(),
Size: file.GetSize(),
FileId: createResp.Data.FileID,
Type: 2,
Etag: etag,
}, nil
}
}
// 秒传失败,说明etag不可靠,需要流式计算真实MD5
// 注意:此时不能上传,因为etag可能是错的
}
createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false)

// 没有etag或秒传失败:流式计算MD5
md5Hash := utils.MD5.NewFunc()
size := file.GetSize()
chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk for MD5 calculation
var offset int64 = 0
for offset < size {
readSize := min(chunkSize, size-offset)
reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize})
if err != nil {
return nil, fmt.Errorf("range read for MD5 calculation failed: %w", err)
}
if _, err := io.Copy(md5Hash, reader); err != nil {
return nil, fmt.Errorf("calculate MD5 failed: %w", err)
}
offset += readSize

progress := 40 * float64(offset) / float64(size)
up(progress)
}

etag = hex.EncodeToString(md5Hash.Sum(nil))

// 创建上传任务(使用真实计算的MD5)
createResp, err = d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false)
if err != nil {
return nil, err
}
Expand All @@ -188,13 +235,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
}
}

// 2. 上传分片
err = d.Upload(ctx, file, createResp, up)
// 上传分片(不需要计算哈希了,已经计算过了)
uploadProgress := func(p float64) {
up(40 + p*0.5)
}
_, err = d.Upload(ctx, file, createResp, uploadProgress, false)
if err != nil {
return nil, err
}

// 3. 上传完毕
// 2. 合并分片/完成上传
for range 60 {
uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID)
// 返回错误代码未知,如:20103,文档也没有具体说
Expand Down
29 changes: 25 additions & 4 deletions drivers/123_open/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"hash"
"io"
"mime/multipart"
"net/http"
Expand Down Expand Up @@ -42,14 +43,21 @@ func (d *Open123) create(parentFileID int64, filename string, etag string, size
}

// 上传分片 V2
func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createResp *UploadCreateResp, up driver.UpdateProgress) error {
// 如果needHash为true,会在上传过程中计算文件的MD5哈希,并通过返回值返回
func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createResp *UploadCreateResp, up driver.UpdateProgress, needHash bool) (string, error) {
uploadDomain := createResp.Data.Servers[0]
size := file.GetSize()
chunkSize := createResp.Data.SliceSize

ss, err := stream.NewStreamSectionReader(file, int(chunkSize), &up)
if err != nil {
return err
return "", err
}

// 如果需要计算哈希,创建哈希计算器
var md5Hash hash.Hash
if needHash {
md5Hash = utils.MD5.NewFunc()
}

uploadNums := (size + chunkSize - 1) / chunkSize
Expand Down Expand Up @@ -88,6 +96,15 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
reader.Seek(0, io.SeekStart)
}

// 如果需要计算整体文件哈希,在这里将分片数据写入哈希计算器
if needHash {
_, err = io.Copy(md5Hash, reader)
if err != nil {
return err
}
reader.Seek(0, io.SeekStart)
}

b.Reset()
w := multipart.NewWriter(b)
// 添加表单字段
Expand Down Expand Up @@ -160,10 +177,14 @@ func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createRes
}

if err := threadG.Wait(); err != nil {
return err
return "", err
}

return nil
// 如果需要计算哈希,返回计算结果
if needHash {
return fmt.Sprintf("%x", md5Hash.Sum(nil)), nil
}
return "", nil
}

// 上传完毕
Expand Down