Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ Thank you for your support and understanding of the OpenList project.

## Demo

N/A (to be rebuilt)
- 🌎 [Global Demo](https://demo.oplist.org)
- 🇨🇳 [CN Demo](https://demo.oplist.org.cn)

## Discussion

Expand Down
5 changes: 3 additions & 2 deletions README_cn.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,14 @@ OpenList 是一个由 OpenList 团队独立维护的开源项目,遵循 AGPL-3

## 演示

N/A(待重建)
- 🇨🇳 [国内演示站](https://demo.oplist.org.cn)
- 🌎 [海外演示站](https://demo.oplist.org)

## 讨论

如有一般性问题请前往 [*Discussions*](https://github.com/OpenListTeam/OpenList/discussions) 讨论区,***Issues* 仅用于错误报告和功能请求。**

## 赞助
## 赞助者

[![VPS.Town](https://vps.town/static/images/sponsor.png)](https://vps.town "VPS.Town - Trust, Effortlessly. Your Cloud, Reimagined.")

Expand Down
3 changes: 2 additions & 1 deletion README_ja.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ OpenListプロジェクトへのご支援とご理解をありがとうござい

## デモ

N/A(再構築中)
- 🌎 [グローバルデモ](https://demo.oplist.org)
- 🇨🇳 [CNデモ](https://demo.oplist.org.cn)

## ディスカッション

Expand Down
3 changes: 2 additions & 1 deletion README_nl.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,8 @@ Dank u voor uw ondersteuning en begrip

## Demo

N.v.t. (wordt opnieuw opgebouwd)
- 🌎 [Global Demo](https://demo.oplist.org)
- 🇨🇳 [CN Demo](https://demo.oplist.org.cn)

## Discussie

Expand Down
9 changes: 9 additions & 0 deletions cmd/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,19 @@ import (
"path/filepath"
"strconv"

"github.com/OpenListTeam/OpenList/v4/internal/bootstrap"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
log "github.com/sirupsen/logrus"
)

func Init() {
bootstrap.Init()
}

func Release() {
bootstrap.Release()
}

var pid = -1
var pidFile string

Expand Down
41 changes: 31 additions & 10 deletions drivers/123_open/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,17 @@ package _123_open

import (
"context"
"encoding/hex"
"fmt"
"io"
"strconv"
"time"

"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
"github.com/OpenListTeam/OpenList/v4/internal/stream"
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)

Expand Down Expand Up @@ -156,20 +158,36 @@ func (d *Open123) Remove(ctx context.Context, obj model.Obj) error {
}

func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
// 1. 创建文件
// 1. 准备参数
// parentFileID 父目录id,上传到根目录时填写 0
parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64)
if err != nil {
return nil, fmt.Errorf("parse parentFileID error: %v", err)
}
// etag 文件md5
etag := file.GetHash().GetHash(utils.MD5)
if len(etag) < utils.MD5.Width {
_, etag, err = stream.CacheFullAndHash(file, &up, utils.MD5)

// 1. 流式计算MD5
md5Hash := utils.MD5.NewFunc()
size := file.GetSize()
chunkSize := int64(10 * 1024 * 1024) // 10MB per chunk for MD5 calculation
var offset int64 = 0
for offset < size {
readSize := min(chunkSize, size-offset)
reader, err := file.RangeRead(http_range.Range{Start: offset, Length: readSize})
if err != nil {
return nil, err
return nil, fmt.Errorf("range read for MD5 calculation failed: %w", err)
}
if _, err := io.Copy(md5Hash, reader); err != nil {
return nil, fmt.Errorf("calculate MD5 failed: %w", err)
}
offset += readSize

progress := 40 * float64(offset) / float64(size)
up(progress)
}

etag := hex.EncodeToString(md5Hash.Sum(nil))

// 2. 创建上传任务
createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false)
if err != nil {
return nil, err
Expand All @@ -188,13 +206,16 @@ func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStre
}
}

// 2. 上传分片
err = d.Upload(ctx, file, createResp, up)
// 3. 上传分片
uploadProgress := func(p float64) {
up(40 + p*0.6)
}
err = d.Upload(ctx, file, createResp, uploadProgress)
if err != nil {
return nil, err
}

// 3. 上传完毕
// 4. 合并分片/完成上传
for range 60 {
uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID)
// 返回错误代码未知,如:20103,文档也没有具体说
Expand Down
3 changes: 3 additions & 0 deletions drivers/strm/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,9 @@ func (d *Strm) Init(ctx context.Context) error {
d.PathPrefix = "/d"
d.Version = 5
}
if len(d.SaveLocalMode) == 0 {
d.SaveLocalMode = SaveLocalInsertMode
}
return nil
}

Expand Down
60 changes: 51 additions & 9 deletions drivers/strm/hook.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
package strm

import (
"bytes"
"context"
"crypto/sha256"
"errors"
"io"
"os"
stdpath "path"
"strings"
Expand Down Expand Up @@ -90,6 +93,9 @@ func RemoveStrm(dstPath string, d *Strm) {

func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath string) {
if !obj.IsDir() {
if utils.Exists(localPath) && driver.SaveLocalMode == SaveLocalInsertMode {
return
}
link, err := driver.Link(ctx, obj, model.LinkArgs{})
if err != nil {
log.Warnf("failed to generate strm of obj %s: failed to link: %v", localPath, err)
Expand All @@ -111,6 +117,20 @@ func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath st
return
}
defer rc.Close()
same, err := isSameContent(localPath, size, rc)
if err != nil {
log.Warnf("failed to compare content of obj %s: %v", localPath, err)
return
}
if same {
return
}
rc, err = rrf.RangeRead(ctx, http_range.Range{Length: -1})
if err != nil {
log.Warnf("failed to generate strm of obj %s: failed to reread range: %v", localPath, err)
return
}
defer rc.Close()
file, err := utils.CreateNestedFile(localPath)
if err != nil {
log.Warnf("failed to generate strm of obj %s: failed to create local file: %v", localPath, err)
Expand All @@ -123,7 +143,38 @@ func generateStrm(ctx context.Context, driver *Strm, obj model.Obj, localPath st
}
}

func isSameContent(localPath string, size int64, rc io.Reader) (bool, error) {
info, err := os.Stat(localPath)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}

if info.Size() != size {
return false, nil
}
localFile, err := os.Open(localPath)
if err != nil {
return false, err
}
defer localFile.Close()
h1 := sha256.New()
h2 := sha256.New()
if _, err := io.Copy(h1, localFile); err != nil {
return false, err
}
if _, err := io.Copy(h2, rc); err != nil {
return false, err
}
return bytes.Equal(h1.Sum(nil), h2.Sum(nil)), nil
}

func deleteExtraFiles(driver *Strm, localPath string, objs []model.Obj) {
if driver.SaveLocalMode != SaveLocalSyncMode {
return
}
localFiles, err := getLocalFiles(localPath)
if err != nil {
log.Errorf("Failed to read local files from %s: %v", localPath, err)
Expand All @@ -145,15 +196,6 @@ func deleteExtraFiles(driver *Strm, localPath string, objs []model.Obj) {

for _, localFile := range localFiles {
if _, exists := objsSet[localFile]; !exists {
ext := utils.Ext(localFile)
localFileName := stdpath.Base(localFile)
localFileBaseName := strings.TrimSuffix(localFile, utils.SourceExt(localFileName))
_, nameExists := objsBaseNameSet[localFileBaseName[:len(localFileBaseName)-1]]
_, downloadFile := driver.downloadSuffix[ext]
if driver.KeepLocalDownloadFile && nameExists && downloadFile {
continue
}

err := os.Remove(localFile)
if err != nil {
log.Errorf("Failed to delete file: %s, error: %v\n", localFile, err)
Expand Down
7 changes: 7 additions & 0 deletions drivers/strm/meta.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ import (
"github.com/OpenListTeam/OpenList/v4/internal/op"
)

const (
SaveLocalInsertMode = "insert"
SaveLocalUpdateMode = "update"
SaveLocalSyncMode = "sync"
)

type Addition struct {
Paths string `json:"paths" required:"true" type:"text"`
SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of the strm file"`
Expand All @@ -17,6 +23,7 @@ type Addition struct {
SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"save strm file locally"`
SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"save strm file local path"`
KeepLocalDownloadFile bool `json:"KeepLocalDownloadFile" default:"false" help:"keep local download files"`
SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"save strm file locally mode" options:"insert,update,sync" default:"insert"`
Version int
}

Expand Down
17 changes: 16 additions & 1 deletion drivers/weiyun/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/avast/retry-go"
weiyunsdkgo "github.com/foxxorcat/weiyun-sdk-go"
"github.com/go-resty/resty/v2"
)

type WeiYun struct {
Expand Down Expand Up @@ -67,7 +68,7 @@ func (d *WeiYun) Init(ctx context.Context) error {
})

// qqCookie保活
if d.client.LoginType() == 1 {
if d.client.LoginType() == weiyunsdkgo.AccountTypeQQ || d.client.LoginType() == weiyunsdkgo.AccountTypeQQOpenID {
d.cron = cron.NewCron(time.Minute * 5)
d.cron.Do(func() {
_ = d.client.KeepAlive()
Expand Down Expand Up @@ -391,6 +392,19 @@ func (d *WeiYun) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr
}, nil
}

func (d *WeiYun) GetDetails(ctx context.Context) (*model.StorageDetails, error) {
info, err := d.client.DiskUserInfoGet(func(request *resty.Request) {
request.SetContext(ctx)
})
if err != nil {
return nil, err
}

return &model.StorageDetails{
DiskUsage: driver.DiskUsageFromUsedAndTotal(uint64(info.UsedSpace), uint64(info.TotalSpace)),
}, nil
}

// func (d *WeiYun) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) {
// return nil, errs.NotSupport
// }
Expand All @@ -405,3 +419,4 @@ var _ driver.Remove = (*WeiYun)(nil)

var _ driver.PutResult = (*WeiYun)(nil)
var _ driver.RenameResult = (*WeiYun)(nil)
var _ driver.WithDetails = (*WeiYun)(nil)
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ require (
github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564
github.com/fclairamb/ftpserverlib v0.26.1-0.20250709223522-4a925d79caf6
github.com/foxxorcat/mopan-sdk-go v0.1.6
github.com/foxxorcat/weiyun-sdk-go v0.1.3
github.com/foxxorcat/weiyun-sdk-go v0.1.4
github.com/gin-contrib/cors v1.7.6
github.com/gin-gonic/gin v1.10.1
github.com/go-resty/resty/v2 v2.16.5
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -285,8 +285,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y=
github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs=
github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY=
github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps=
github.com/foxxorcat/weiyun-sdk-go v0.1.4 h1:X2tFvdqikkJ7awCBbMH7XXk7+uQoJlQksJz9CUU6ZgA=
github.com/foxxorcat/weiyun-sdk-go v0.1.4/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
Expand Down
Loading