Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions internal/fs/copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ func (t *CopyTask) Run() error {
t.ClearEndTime()
t.SetStartTime(time.Now())
defer func() { t.SetEndTime(time.Now()) }()

var err error
if t.srcStorage == nil {
t.srcStorage, err = op.GetStorageByMountPath(t.SrcStorageMp)
Expand All @@ -55,11 +56,27 @@ func (t *CopyTask) Run() error {
if err != nil {
return errors.WithMessage(err, "failed get storage")
}
return copyBetween2Storages(t, t.srcStorage, t.dstStorage, t.SrcObjPath, t.DstDirPath)

// Use the task object's memory address as a unique identifier
taskID := fmt.Sprintf("%p", t)

// Register task to batch tracker
copyBatchTracker.RegisterTask(taskID, t.dstStorage, t.DstDirPath)

// Execute copy operation
err = copyBetween2Storages(t, t.srcStorage, t.dstStorage, t.SrcObjPath, t.DstDirPath)

// Mark task completed and automatically refresh cache if needed
copyBatchTracker.MarkTaskCompletedWithRefresh(taskID)

return err
}

var CopyTaskManager *tache.Manager[*CopyTask]

// Batch tracker for copy operations
var copyBatchTracker = NewBatchTracker("copy")

// Copy if in the same storage, call move method
// if not, add copy task
func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) {
Expand All @@ -75,6 +92,10 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool
if srcStorage.GetStorage() == dstStorage.GetStorage() {
err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...)
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) {
if err == nil {
// Refresh target directory cache after successful same-storage copy
op.ClearCache(dstStorage, dstDirActualPath)
}
return nil, err
}
}
Expand All @@ -98,7 +119,12 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool
_ = link.Close()
return nil, errors.WithMessagef(err, "failed get [%s] stream", srcObjPath)
}
return nil, op.Put(ctx, dstStorage, dstDirActualPath, ss, nil, false)
err = op.Put(ctx, dstStorage, dstDirActualPath, ss, nil, false)
if err == nil {
// Refresh target directory cache after successful direct file copy
op.ClearCache(dstStorage, dstDirActualPath)
}
return nil, err
}
}
// not in the same storage
Expand Down Expand Up @@ -131,6 +157,7 @@ func copyBetween2Storages(t *CopyTask, srcStorage, dstStorage driver.Driver, src
if err != nil {
return errors.WithMessagef(err, "failed list src [%s] objs", srcObjPath)
}

for _, obj := range objs {
if utils.IsCanceled(t.Ctx()) {
return nil
Expand Down
37 changes: 32 additions & 5 deletions internal/fs/move.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,12 @@ func (t *MoveTask) Run() error {
return errors.WithMessage(err, "failed get storage")
}

// Use the task object's memory address as a unique identifier
taskID := fmt.Sprintf("%p", t)

// Register task to batch tracker
moveBatchTracker.RegisterTask(taskID, t.dstStorage, t.DstDirPath)

// Phase 1: Async validation (all validation happens in background)
t.mu.Lock()
t.Status = "validating source and destination"
Expand All @@ -138,13 +144,17 @@ func (t *MoveTask) Run() error {
// Check if source exists
srcObj, err := op.Get(t.Ctx(), t.srcStorage, t.SrcObjPath)
if err != nil {
// Clean up tracker records if task failed
moveBatchTracker.MarkTaskCompletedWithRefresh(taskID)
return errors.WithMessagef(err, "source file [%s] not found", stdpath.Base(t.SrcObjPath))
}

// Check if destination already exists (if validation is required)
if t.ValidateExistence {
dstFilePath := stdpath.Join(t.DstDirPath, srcObj.GetName())
if res, _ := op.Get(t.Ctx(), t.dstStorage, dstFilePath); res != nil {
// Clean up tracker records if task failed
moveBatchTracker.MarkTaskCompletedWithRefresh(taskID)
return errors.Errorf("destination file [%s] already exists", srcObj.GetName())
}
}
Expand All @@ -156,11 +166,16 @@ func (t *MoveTask) Run() error {
t.IsRootTask = true
t.RootTaskID = t.GetID()
t.mu.Unlock()
return t.runRootMoveTask()
err = t.runRootMoveTask()
} else {
// Use safe move logic for files
err = t.safeMoveOperation(srcObj)
}

// Use safe move logic for files
return t.safeMoveOperation(srcObj)
// Mark task completed and automatically refresh cache if needed
moveBatchTracker.MarkTaskCompletedWithRefresh(taskID)

return err
}

func (t *MoveTask) runRootMoveTask() error {
Expand Down Expand Up @@ -236,11 +251,15 @@ func (t *MoveTask) runRootMoveTask() error {
t.mu.Unlock()
t.updateProgress()


return nil
}

var MoveTaskManager *tache.Manager[*MoveTask]

// Batch tracker for move operations
var moveBatchTracker = NewBatchTracker("move")

// GetMoveProgress returns the progress of a move task by task ID
func GetMoveProgress(taskID string) (*MoveProgress, bool) {
if progress, ok := moveProgressMap.Load(taskID); ok {
Expand Down Expand Up @@ -487,7 +506,7 @@ func moveBetween2Storages(t *MoveTask, srcStorage, dstStorage driver.Driver, src
return nil
}
srcSubObjPath := stdpath.Join(srcObjPath, obj.GetName())
MoveTaskManager.Add(&MoveTask{
subTask := &MoveTask{
TaskExtension: task.TaskExtension{
Creator: t.GetCreator(),
ApiUrl: t.ApiUrl,
Expand All @@ -498,7 +517,8 @@ func moveBetween2Storages(t *MoveTask, srcStorage, dstStorage driver.Driver, src
DstDirPath: dstObjPath,
SrcStorageMp: srcStorage.GetStorage().MountPath,
DstStorageMp: dstStorage.GetStorage().MountPath,
})
}
MoveTaskManager.Add(subTask)
}

t.Status = "cleaning up source directory"
Expand All @@ -508,6 +528,8 @@ func moveBetween2Storages(t *MoveTask, srcStorage, dstStorage driver.Driver, src
} else {
t.Status = "completed"
}


return nil
} else {
return moveFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath)
Expand Down Expand Up @@ -549,6 +571,7 @@ func moveFileBetween2Storages(tsk *MoveTask, srcStorage, dstStorage driver.Drive
return errors.WithMessagef(err, "failed to delete src [%s] file from storage [%s] after successful copy", srcFilePath, srcStorage.GetStorage().MountPath)
}


tsk.SetProgress(100)
tsk.Status = "completed"
return nil
Expand Down Expand Up @@ -583,6 +606,10 @@ func _moveWithValidation(ctx context.Context, srcObjPath, dstDirPath string, val
if srcStorage.GetStorage() == dstStorage.GetStorage() {
err = op.Move(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...)
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) {
if err == nil {
// For same-storage moves, refresh cache immediately since no batch tracking is used
op.ClearCache(dstStorage, dstDirActualPath)
}
return nil, err
}
}
Expand Down
153 changes: 153 additions & 0 deletions internal/fs/task.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
package fs

import (
"sync"
"time"

"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/op"
)

// BatchTracker manages batch operations for cache refresh optimization
// It aggregates multiple file operations by target directory and only refreshes
// the cache once when all operations in a directory are completed
type BatchTracker struct {
mu sync.Mutex
dirTasks map[string]*dirTaskInfo // dstStoragePath+dstDirPath -> dirTaskInfo
pendingTasks map[string]string // taskID -> dstStoragePath+dstDirPath
lastCleanup time.Time // last cleanup time
name string // tracker name for debugging
}

type dirTaskInfo struct {
dstStorage driver.Driver
dstDirPath string
pendingTasks map[string]bool // taskID -> true
lastActivity time.Time // last activity time (used for detecting abnormal situations)
}

// NewBatchTracker creates a new batch tracker instance
func NewBatchTracker(name string) *BatchTracker {
return &BatchTracker{
dirTasks: make(map[string]*dirTaskInfo),
pendingTasks: make(map[string]string),
lastCleanup: time.Now(),
name: name,
}
}

// getDirKey generates unique key for target directory
func (bt *BatchTracker) getDirKey(dstStorage driver.Driver, dstDirPath string) string {
return dstStorage.GetStorage().MountPath + ":" + dstDirPath
}

// RegisterTask registers a task to target directory for batch tracking
func (bt *BatchTracker) RegisterTask(taskID string, dstStorage driver.Driver, dstDirPath string) {
bt.mu.Lock()
defer bt.mu.Unlock()

// Periodically clean up expired entries
bt.cleanupIfNeeded()

dirKey := bt.getDirKey(dstStorage, dstDirPath)

// Record task to directory mapping
bt.pendingTasks[taskID] = dirKey

// Initialize or update directory task information
if info, exists := bt.dirTasks[dirKey]; exists {
info.pendingTasks[taskID] = true
info.lastActivity = time.Now()
} else {
bt.dirTasks[dirKey] = &dirTaskInfo{
dstStorage: dstStorage,
dstDirPath: dstDirPath,
pendingTasks: map[string]bool{taskID: true},
lastActivity: time.Now(),
}
}
}

// MarkTaskCompleted marks a task as completed and returns whether cache refresh is needed
// Returns (shouldRefresh, dstStorage, dstDirPath)
func (bt *BatchTracker) MarkTaskCompleted(taskID string) (bool, driver.Driver, string) {
bt.mu.Lock()
defer bt.mu.Unlock()

dirKey, exists := bt.pendingTasks[taskID]
if !exists {
return false, nil, ""
}

// Remove from pending tasks
delete(bt.pendingTasks, taskID)

info, exists := bt.dirTasks[dirKey]
if !exists {
return false, nil, ""
}

// Remove from directory tasks
delete(info.pendingTasks, taskID)

// If no pending tasks left in this directory, trigger cache refresh
if len(info.pendingTasks) == 0 {
dstStorage := info.dstStorage
dstDirPath := info.dstDirPath
delete(bt.dirTasks, dirKey) // Delete directly, no need to update lastActivity
return true, dstStorage, dstDirPath
}

// Only update lastActivity when there are other tasks (indicating the directory still has active tasks)
info.lastActivity = time.Now()
return false, nil, ""
}

// MarkTaskCompletedWithRefresh marks a task as completed and automatically refreshes cache if needed
func (bt *BatchTracker) MarkTaskCompletedWithRefresh(taskID string) {
shouldRefresh, dstStorage, dstDirPath := bt.MarkTaskCompleted(taskID)
if shouldRefresh {
op.ClearCache(dstStorage, dstDirPath)
}
}

// cleanupIfNeeded checks if cleanup is needed and executes cleanup if necessary
func (bt *BatchTracker) cleanupIfNeeded() {
now := time.Now()
// Clean up every 10 minutes
if now.Sub(bt.lastCleanup) > 10*time.Minute {
bt.cleanupStaleEntries()
bt.lastCleanup = now
}
}

// cleanupStaleEntries cleans up timed-out tasks to prevent memory leaks
// Mainly used to clean up residual entries caused by abnormal situations (such as task crashes, process restarts, etc.)
func (bt *BatchTracker) cleanupStaleEntries() {
now := time.Now()
for dirKey, info := range bt.dirTasks {
// If no activity for more than 1 hour, it may indicate an abnormal situation, clean up this entry
// Under normal circumstances, MarkTaskCompleted will be called when the task is completed and the entire entry will be deleted
if now.Sub(info.lastActivity) > time.Hour {
// Clean up related pending tasks
for taskID := range info.pendingTasks {
delete(bt.pendingTasks, taskID)
}
delete(bt.dirTasks, dirKey)
}
}
}

// GetPendingTaskCount returns the number of pending tasks for debugging
func (bt *BatchTracker) GetPendingTaskCount() int {
bt.mu.Lock()
defer bt.mu.Unlock()
return len(bt.pendingTasks)
}

// GetDirTaskCount returns the number of directories being tracked for debugging
func (bt *BatchTracker) GetDirTaskCount() int {
bt.mu.Lock()
defer bt.mu.Unlock()
return len(bt.dirTasks)
}