Skip to content

Commit 72e2ae1

Browse files
KirCuteKirCute
authored
feat(fs): support manually trigger objs update hook (#1620)
* feat(fs): support manually trigger objs update hook * fix: support driver internal copy & move case * fix * fix: apply suggestions of Copilot
1 parent 3e37f57 commit 72e2ae1

File tree

10 files changed

+304
-14
lines changed

10 files changed

+304
-14
lines changed

‎internal/bootstrap/data/setting.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,8 @@ func InitialSettings() []model.SettingItem {
177177
{Key: conf.ShareArchivePreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},
178178
{Key: conf.ShareForceProxy, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
179179
{Key: conf.ShareSummaryContent, Value: "@{{creator}} shared {{#each files}}{{#if @first}}\"{{filename this}}\"{{/if}}{{#if @last}}{{#unless (eq @index 0)}} and {{@index}} more files{{/unless}}{{/if}}{{/each}} from {{site_title}}: {{base_url}}/@s/{{id}}{{#if pwd}} , the share code is {{pwd}}{{/if}}{{#if expires}}, please access before {{dateLocaleString expires}}.{{/if}}", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PUBLIC},
180+
{Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
181+
{Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE},
180182
{Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`},
181183

182184
// single settings

‎internal/conf/const.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ const (
5656
ShareArchivePreview = "share_archive_preview"
5757
ShareForceProxy = "share_force_proxy"
5858
ShareSummaryContent = "share_summary_content"
59+
HandleHookAfterWriting = "handle_hook_after_writing"
60+
HandleHookRateLimit = "handle_hook_rate_limit"
5961
IgnoreSystemFiles = "ignore_system_files"
6062

6163
// index

‎internal/op/archive.go‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/OpenListTeam/OpenList/v4/internal/archive/tool"
1313
"github.com/OpenListTeam/OpenList/v4/internal/cache"
14+
"github.com/OpenListTeam/OpenList/v4/internal/conf"
1415
"github.com/OpenListTeam/OpenList/v4/internal/driver"
1516
"github.com/OpenListTeam/OpenList/v4/internal/errs"
1617
"github.com/OpenListTeam/OpenList/v4/internal/model"
@@ -20,10 +21,13 @@ import (
2021
gocache "github.com/OpenListTeam/go-cache"
2122
"github.com/pkg/errors"
2223
log "github.com/sirupsen/logrus"
24+
"golang.org/x/time/rate"
2325
)
2426

25-
var archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64))
26-
var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
27+
var (
28+
archiveMetaCache = gocache.NewMemCache(gocache.WithShards[*model.ArchiveMetaProvider](64))
29+
archiveMetaG singleflight.Group[*model.ArchiveMetaProvider]
30+
)
2731

2832
func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) {
2933
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
@@ -196,8 +200,10 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg
196200
return obj, archiveMetaProvider, err
197201
}
198202

199-
var archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64))
200-
var archiveListG singleflight.Group[[]model.Obj]
203+
var (
204+
archiveListCache = gocache.NewMemCache(gocache.WithShards[[]model.Obj](64))
205+
archiveListG singleflight.Group[[]model.Obj]
206+
)
201207

202208
func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) {
203209
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
@@ -397,8 +403,10 @@ type objWithLink struct {
397403
obj model.Obj
398404
}
399405

400-
var extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute)
401-
var extractG = singleflight.Group[*objWithLink]{}
406+
var (
407+
extractCache = cache.NewKeyedCache[*objWithLink](5 * time.Minute)
408+
extractG = singleflight.Group[*objWithLink]{}
409+
)
402410

403411
func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) {
404412
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
@@ -506,9 +514,9 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD
506514
return errors.WithMessage(err, "failed to get dst dir")
507515
}
508516

517+
var newObjs []model.Obj
509518
switch s := storage.(type) {
510519
case driver.ArchiveDecompressResult:
511-
var newObjs []model.Obj
512520
newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args)
513521
if err == nil {
514522
if len(newObjs) > 0 {
@@ -527,5 +535,31 @@ func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstD
527535
default:
528536
return errs.NotImplement
529537
}
538+
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
539+
onlyList := false
540+
targetPath := dstDirPath
541+
if newObjs != nil && len(newObjs) == 1 && newObjs[0].IsDir() {
542+
targetPath = stdpath.Join(dstDirPath, newObjs[0].GetName())
543+
} else if newObjs != nil && len(newObjs) == 1 && !newObjs[0].IsDir() {
544+
onlyList = true
545+
} else if args.PutIntoNewDir {
546+
targetPath = stdpath.Join(dstDirPath, strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName())))
547+
} else if innerBase := stdpath.Base(args.InnerPath); innerBase != "." && innerBase != "/" {
548+
targetPath = stdpath.Join(dstDirPath, innerBase)
549+
dstObj, e := GetUnwrap(ctx, storage, targetPath)
550+
onlyList = e != nil || !dstObj.IsDir()
551+
}
552+
if onlyList {
553+
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
554+
} else {
555+
var limiter *rate.Limiter
556+
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
557+
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
558+
limiter = rate.NewLimiter(rate.Limit(f), 1)
559+
}
560+
}
561+
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
562+
}
563+
}
530564
return errors.WithStack(err)
531565
}

‎internal/op/fs.go‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ package op
22

33
import (
44
"context"
5-
stderrors "errors"
65
stdpath "path"
6+
"strconv"
77
"time"
88

9+
"github.com/OpenListTeam/OpenList/v4/internal/conf"
910
"github.com/OpenListTeam/OpenList/v4/internal/driver"
1011
"github.com/OpenListTeam/OpenList/v4/internal/errs"
1112
"github.com/OpenListTeam/OpenList/v4/internal/model"
@@ -14,6 +15,7 @@ import (
1415
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
1516
"github.com/pkg/errors"
1617
log "github.com/sirupsen/logrus"
18+
"golang.org/x/time/rate"
1719
)
1820

1921
var listG singleflight.Group[[]model.Obj]
@@ -310,7 +312,7 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
310312
srcDirPath := stdpath.Dir(srcPath)
311313
dstDirPath = utils.FixAndCleanPath(dstDirPath)
312314
if dstDirPath == srcDirPath {
313-
return stderrors.New("move in place")
315+
return errors.New("move in place")
314316
}
315317
srcRawObj, err := Get(ctx, storage, srcPath)
316318
if err != nil {
@@ -343,8 +345,24 @@ func Move(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
343345
}
344346
}
345347
default:
346-
return errs.NotImplement
348+
err = errs.NotImplement
349+
}
350+
351+
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
352+
if !srcObj.IsDir() {
353+
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
354+
} else {
355+
targetPath := stdpath.Join(dstDirPath, srcObj.GetName())
356+
var limiter *rate.Limiter
357+
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
358+
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
359+
limiter = rate.NewLimiter(rate.Limit(f), 1)
360+
}
361+
}
362+
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
363+
}
347364
}
365+
348366
return errors.WithStack(err)
349367
}
350368

@@ -397,7 +415,7 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
397415
srcPath = utils.FixAndCleanPath(srcPath)
398416
dstDirPath = utils.FixAndCleanPath(dstDirPath)
399417
if dstDirPath == stdpath.Dir(srcPath) {
400-
return stderrors.New("copy in place")
418+
return errors.New("copy in place")
401419
}
402420
srcRawObj, err := Get(ctx, storage, srcPath)
403421
if err != nil {
@@ -428,8 +446,24 @@ func Copy(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string
428446
}
429447
}
430448
default:
431-
return errs.NotImplement
449+
err = errs.NotImplement
450+
}
451+
452+
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
453+
if !srcObj.IsDir() {
454+
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
455+
} else {
456+
targetPath := stdpath.Join(dstDirPath, srcObj.GetName())
457+
var limiter *rate.Limiter
458+
if l, _ := GetSettingItemByKey(conf.HandleHookRateLimit); l != nil {
459+
if f, e := strconv.ParseFloat(l.Value, 64); e == nil && f > .0 {
460+
limiter = rate.NewLimiter(rate.Limit(f), 1)
461+
}
462+
}
463+
go RecursivelyListStorage(context.Background(), storage, targetPath, limiter, nil)
464+
}
432465
}
466+
433467
return errors.WithStack(err)
434468
}
435469

@@ -557,6 +591,9 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod
557591
err = Remove(ctx, storage, tempPath)
558592
}
559593
}
594+
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
595+
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
596+
}
560597
return errors.WithStack(err)
561598
}
562599

@@ -601,6 +638,9 @@ func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url
601638
default:
602639
return errors.WithStack(errs.NotImplement)
603640
}
641+
if !utils.IsBool(lazyCache...) && err == nil && needHandleObjsUpdateHook() {
642+
go List(context.Background(), storage, dstDirPath, model.ListArgs{Refresh: true})
643+
}
604644
log.Debugf("put url [%s](%s) done", dstName, url)
605645
return errors.WithStack(err)
606646
}
@@ -644,3 +684,8 @@ func GetDirectUploadInfo(ctx context.Context, tool string, storage driver.Driver
644684
}
645685
return info, nil
646686
}
687+
688+
func needHandleObjsUpdateHook() bool {
689+
needHandle, _ := GetSettingItemByKey(conf.HandleHookAfterWriting)
690+
return needHandle != nil && (needHandle.Value == "true" || needHandle.Value == "1")
691+
}

‎internal/op/recursive_list.go‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
package op
2+
3+
import (
4+
"context"
5+
stdpath "path"
6+
"sync"
7+
"sync/atomic"
8+
9+
"github.com/OpenListTeam/OpenList/v4/internal/driver"
10+
"github.com/OpenListTeam/OpenList/v4/internal/errs"
11+
"github.com/OpenListTeam/OpenList/v4/internal/model"
12+
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
13+
"github.com/pkg/errors"
14+
log "github.com/sirupsen/logrus"
15+
"golang.org/x/time/rate"
16+
)
17+
18+
var (
19+
ManualScanCancel = atomic.Pointer[context.CancelFunc]{}
20+
ScannedCount = atomic.Uint64{}
21+
)
22+
23+
func ManualScanRunning() bool {
24+
return ManualScanCancel.Load() != nil
25+
}
26+
27+
func BeginManualScan(rawPath string, limit float64) error {
28+
rawPath = utils.FixAndCleanPath(rawPath)
29+
ctx, cancel := context.WithCancel(context.Background())
30+
if !ManualScanCancel.CompareAndSwap(nil, &cancel) {
31+
cancel()
32+
return errors.New("manual scan is running, please try later")
33+
}
34+
ScannedCount.Store(0)
35+
go func() {
36+
defer func() { (*ManualScanCancel.Swap(nil))() }()
37+
err := RecursivelyList(ctx, rawPath, rate.Limit(limit), &ScannedCount)
38+
if err != nil {
39+
log.Errorf("failed recursively list: %v", err)
40+
}
41+
}()
42+
return nil
43+
}
44+
45+
func StopManualScan() {
46+
c := ManualScanCancel.Load()
47+
if c != nil {
48+
(*c)()
49+
}
50+
}
51+
52+
func RecursivelyList(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64) error {
53+
storage, actualPath, err := GetStorageAndActualPath(rawPath)
54+
if err != nil && !errors.Is(err, errs.StorageNotFound) {
55+
return err
56+
} else if err == nil {
57+
var limiter *rate.Limiter
58+
if limit > .0 {
59+
limiter = rate.NewLimiter(limit, 1)
60+
}
61+
RecursivelyListStorage(ctx, storage, actualPath, limiter, counter)
62+
} else {
63+
var wg sync.WaitGroup
64+
recursivelyListVirtual(ctx, rawPath, limit, counter, &wg)
65+
wg.Wait()
66+
}
67+
return nil
68+
}
69+
70+
func recursivelyListVirtual(ctx context.Context, rawPath string, limit rate.Limit, counter *atomic.Uint64, wg *sync.WaitGroup) {
71+
objs := GetStorageVirtualFilesByPath(rawPath)
72+
if counter != nil {
73+
counter.Add(uint64(len(objs)))
74+
}
75+
for _, obj := range objs {
76+
if utils.IsCanceled(ctx) {
77+
return
78+
}
79+
nextPath := stdpath.Join(rawPath, obj.GetName())
80+
storage, actualPath, err := GetStorageAndActualPath(nextPath)
81+
if err != nil && !errors.Is(err, errs.StorageNotFound) {
82+
log.Errorf("error recursively list: failed get storage [%s]: %v", nextPath, err)
83+
} else if err == nil {
84+
var limiter *rate.Limiter
85+
if limit > .0 {
86+
limiter = rate.NewLimiter(limit, 1)
87+
}
88+
wg.Add(1)
89+
go func() {
90+
defer wg.Done()
91+
RecursivelyListStorage(ctx, storage, actualPath, limiter, counter)
92+
}()
93+
} else {
94+
recursivelyListVirtual(ctx, nextPath, limit, counter, wg)
95+
}
96+
}
97+
}
98+
99+
func RecursivelyListStorage(ctx context.Context, storage driver.Driver, actualPath string, limiter *rate.Limiter, counter *atomic.Uint64) {
100+
objs, err := List(ctx, storage, actualPath, model.ListArgs{Refresh: true})
101+
if err != nil {
102+
if !errors.Is(err, context.Canceled) {
103+
log.Errorf("error recursively list: failed list (%s)[%s]: %v", storage.GetStorage().MountPath, actualPath, err)
104+
}
105+
return
106+
}
107+
if counter != nil {
108+
counter.Add(uint64(len(objs)))
109+
}
110+
for _, obj := range objs {
111+
if utils.IsCanceled(ctx) {
112+
return
113+
}
114+
if !obj.IsDir() {
115+
continue
116+
}
117+
if limiter != nil {
118+
if err = limiter.Wait(ctx); err != nil {
119+
return
120+
}
121+
}
122+
nextPath := stdpath.Join(actualPath, obj.GetName())
123+
RecursivelyListStorage(ctx, storage, nextPath, limiter, counter)
124+
}
125+
}

‎internal/setting/setting.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,11 @@ func GetInt(key string, defaultVal int) int {
2828
func GetBool(key string) bool {
2929
return GetStr(key) == "true" || GetStr(key) == "1"
3030
}
31+
32+
func GetFloat(key string, defaultVal float64) float64 {
33+
f, err := strconv.ParseFloat(GetStr(key), 64)
34+
if err != nil {
35+
return defaultVal
36+
}
37+
return f
38+
}

0 commit comments

Comments
 (0)