feat(model): add object mask support and enhance cache/task handling (#1743)

Author: j2rong4cn

drivers/alias/driver.go

@@ -85,6 +85,7 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) {
 	}
 	var ret *model.Object
 	provider := ""
+	var mask model.ObjMask
 	for _, dst := range dsts {
 		rawPath := stdpath.Join(dst, sub)
 		obj, err := fs.Get(ctx, rawPath, &fs.GetArgs{NoLog: true})
@@ -93,6 +94,8 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) {
 		}
 		storage, err := fs.GetStorage(rawPath, &fs.GetStoragesArgs{})
 		if ret == nil {
+			mask = model.GetObjMask(obj)
+			mask &^= model.Temp
 			ret = &model.Object{
 				Path:     path,
 				Name:     obj.GetName(),
@@ -114,14 +117,14 @@ func (d *Alias) Get(ctx context.Context, path string) (model.Obj, error) {
 		return nil, errs.ObjectNotFound
 	}
 	if provider != "" {
-		return &model.ObjectProvider{
+		return model.ObjAddMask(&model.ObjectProvider{
 			Object: *ret,
 			Provider: model.Provider{
 				Provider: provider,
 			},
-		}, nil
+		}, mask), nil
 	}
-	return ret, nil
+	return model.ObjAddMask(ret, mask), nil
 }
 
 func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
@@ -150,21 +153,23 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([
 					Modified: obj.ModTime(),
 					IsFolder: obj.IsDir(),
 				}
+				mask := model.GetObjMask(obj)
+				mask &^= model.Temp
 				if thumb, ok := model.GetThumb(obj); ok {
-					return &model.ObjThumb{
+					return model.ObjAddMask(&model.ObjThumb{
 						Object: objRes,
 						Thumbnail: model.Thumbnail{
 							Thumbnail: thumb,
 						},
-					}, nil
+					}, mask), nil
 				}
 				if details, ok := model.GetStorageDetails(obj); ok {
-					return &model.ObjStorageDetails{
+					return model.ObjAddMask(&model.ObjStorageDetails{
 						Obj:                    &objRes,
 						StorageDetailsWithName: *details,
-					}, nil
+					}, mask), nil
 				}
-				return &objRes, nil
+				return model.ObjAddMask(&objRes, mask), nil
 			})
 		}
 		if err == nil {

drivers/alias/util.go

@@ -33,7 +33,7 @@ func (d *Alias) listRoot(ctx context.Context, withDetails, refresh bool) []model
 			Modified: d.Modified,
 		}
 		idx := len(objs)
-		objs = append(objs, &obj)
+		objs = append(objs, model.ObjAddMask(&obj, model.Virtual))
 		v := d.pathMap[k]
 		if !withDetails || len(v) != 1 {
 			continue

drivers/chunk/driver.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/OpenListTeam/OpenList/v4/internal/conf"
 	"github.com/OpenListTeam/OpenList/v4/internal/driver"
 	"github.com/OpenListTeam/OpenList/v4/internal/errs"
 	"github.com/OpenListTeam/OpenList/v4/internal/fs"
@@ -426,17 +427,18 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream
 		UpdateProgress: up,
 	}
 	dst := stdpath.Join(remoteActualPath, dstDir.GetPath(), d.ChunkPrefix+file.GetName())
+	skipHookCtx := context.WithValue(ctx, conf.SkipHookKey, struct{}{})
 	if d.StoreHash {
 		for ht, value := range file.GetHash().All() {
-			_ = op.Put(ctx, remoteStorage, dst, &stream.FileStream{
+			_ = op.Put(skipHookCtx, remoteStorage, dst, &stream.FileStream{
 				Obj: &model.Object{
 					Name:     fmt.Sprintf("hash_%s_%s%s", ht.Name, value, d.CustomExt),
 					Size:     1,
 					Modified: file.ModTime(),
 				},
 				Mimetype: "application/octet-stream",
 				Reader:   bytes.NewReader([]byte{0}), // 兼容不支持空文件的驱动
-			}, nil, true)
+			}, nil)
 		}
 	}
 	fullPartCount := int(file.GetSize() / d.PartSize)
@@ -447,15 +449,15 @@ func (d *Chunk) Put(ctx context.Context, dstDir model.Obj, file model.FileStream
 	}
 	partIndex := 0
 	for partIndex < fullPartCount {
-		err = op.Put(ctx, remoteStorage, dst, &stream.FileStream{
+		err = op.Put(skipHookCtx, remoteStorage, dst, &stream.FileStream{
 			Obj: &model.Object{
 				Name:     d.getPartName(partIndex),
 				Size:     d.PartSize,
 				Modified: file.ModTime(),
 			},
 			Mimetype: file.GetMimetype(),
 			Reader:   io.LimitReader(upReader, d.PartSize),
-		}, nil, true)
+		}, nil)
 		if err != nil {
 			_ = op.Remove(ctx, remoteStorage, dst)
 			return err

drivers/crypt/driver.go

@@ -3,6 +3,7 @@ package crypt
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	stdpath "path"
@@ -109,43 +110,35 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([
 
 	result := make([]model.Obj, 0, len(objs))
 	for _, obj := range objs {
-		rawName := model.UnwrapObj(obj).GetName()
-		if obj.IsDir() {
-			name, err := d.cipher.DecryptDirName(rawName)
-			if err != nil {
-				// filter illegal files
-				continue
-			}
-			if !d.ShowHidden && strings.HasPrefix(name, ".") {
-				continue
+		size := obj.GetSize()
+		mask := model.GetObjMask(obj)
+		name := obj.GetName()
+		if mask&model.Virtual == 0 {
+			if obj.IsDir() {
+				name, err = d.cipher.DecryptDirName(model.UnwrapObjName(obj).GetName())
+				if err != nil {
+					// filter illegal files
+					continue
+				}
+			} else {
+				size, err = d.cipher.DecryptedSize(size)
+				if err != nil {
+					// filter illegal files
+					continue
+				}
+				name, err = d.cipher.DecryptFileName(model.UnwrapObjName(obj).GetName())
+				if err != nil {
+					// filter illegal files
+					continue
+				}
 			}
-			result = append(result, &model.Object{
-				Path:     stdpath.Join(remoteFullPath, rawName),
-				Name:     name,
-				Size:     0,
-				Modified: obj.ModTime(),
-				IsFolder: obj.IsDir(),
-				Ctime:    obj.CreateTime(),
-				// discarding hash as it's encrypted
-			})
-			continue
-		}
-
-		size, err := d.cipher.DecryptedSize(obj.GetSize())
-		if err != nil {
-			// filter illegal files
-			continue
-		}
-		name, err := d.cipher.DecryptFileName(rawName)
-		if err != nil {
-			// filter illegal files
-			continue
 		}
 		if !d.ShowHidden && strings.HasPrefix(name, ".") {
 			continue
 		}
+		mask &^= model.Temp
 		objRes := &model.Object{
-			Path:     stdpath.Join(remoteFullPath, rawName),
+			Path:     stdpath.Join(remoteFullPath, obj.GetName()),
 			Name:     name,
 			Size:     size,
 			Modified: obj.ModTime(),
@@ -154,20 +147,20 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([
 			// discarding hash as it's encrypted
 		}
 		if !d.Thumbnail || !strings.HasPrefix(args.ReqPath, "/") {
-			result = append(result, objRes)
+			result = append(result, model.ObjAddMask(objRes, mask))
 			continue
 		}
 		thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp")
 		thumb := fmt.Sprintf("%s/d%s?sign=%s",
 			common.GetApiUrl(ctx),
 			utils.EncodePath(thumbPath, true),
 			sign.Sign(thumbPath))
-		result = append(result, &model.ObjThumb{
+		result = append(result, model.ObjAddMask(&model.ObjThumb{
 			Object: *objRes,
 			Thumbnail: model.Thumbnail{
 				Thumbnail: thumb,
 			},
-		})
+		}, mask))
 	}
 
 	return result, nil
@@ -182,7 +175,14 @@ func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) {
 	remoteFullPath := stdpath.Join(d.RemotePath, d.encryptPath(path, firstTryIsFolder))
 	remoteObj, err := fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true})
 	if err != nil {
-		if secondTry && errs.IsObjectNotFound(err) {
+		if errors.Is(err, errs.StorageNotFound) {
+			remoteFullPath = stdpath.Join(d.RemotePath, path)
+			remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true})
+			if err != nil {
+				// 可能是 虚拟路径+开启文件夹加密：返回NotSupport让op.Get去尝试op.List查找
+				return nil, errs.NotSupport
+			}
+		} else if secondTry && errs.IsObjectNotFound(err) {
 			// try the opposite
 			remoteFullPath = stdpath.Join(d.RemotePath, d.encryptPath(path, !firstTryIsFolder))
 			remoteObj, err = fs.Get(ctx, remoteFullPath, &fs.GetArgs{NoLog: true})
@@ -195,20 +195,30 @@ func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) {
 	}
 
 	size := remoteObj.GetSize()
-	name := model.UnwrapObj(remoteObj).GetName()
-	if !remoteObj.IsDir() {
-		size, err = d.cipher.DecryptedSize(size)
-		if err != nil {
-			log.Warnf("DecryptedSize failed for %s ,will use original size, err:%s", path, err)
-		}
-		name, err = d.cipher.DecryptFileName(name)
-		if err != nil {
-			log.Warnf("DecryptFileName failed for %s ,will use original name, err:%s", path, err)
-		}
-	} else {
-		name, err = d.cipher.DecryptDirName(name)
-		if err != nil {
-			log.Warnf("DecryptDirName failed for %s ,will use original name, err:%s", path, err)
+	name := remoteObj.GetName()
+	mask := model.GetObjMask(remoteObj)
+	mask &^= model.Temp
+	if mask&model.Virtual == 0 {
+		if !remoteObj.IsDir() {
+			decryptedSize, err := d.cipher.DecryptedSize(size)
+			if err != nil {
+				log.Warnf("DecryptedSize failed for %s ,will use original size, err:%s", path, err)
+			} else {
+				size = decryptedSize
+			}
+			decryptedName, err := d.cipher.DecryptFileName(model.UnwrapObjName(remoteObj).GetName())
+			if err != nil {
+				log.Warnf("DecryptFileName failed for %s ,will use original name, err:%s", path, err)
+			} else {
+				name = decryptedName
+			}
+		} else {
+			decryptedName, err := d.cipher.DecryptDirName(model.UnwrapObjName(remoteObj).GetName())
+			if err != nil {
+				log.Warnf("DecryptDirName failed for %s ,will use original name, err:%s", path, err)
+			} else {
+				name = decryptedName
+			}
 		}
 	}
 	obj := &model.Object{
@@ -217,8 +227,9 @@ func (d *Crypt) Get(ctx context.Context, path string) (model.Obj, error) {
 		Size:     size,
 		Modified: remoteObj.ModTime(),
 		IsFolder: remoteObj.IsDir(),
+		Ctime:    remoteObj.CreateTime(),
 	}
-	return obj, nil
+	return model.ObjAddMask(obj, mask), nil
 }
 
 // https://github.com/rclone/rclone/blob/v1.67.0/backend/crypt/cipher.go#L37

internal/cache/keyed_cache.go

@@ -70,7 +70,7 @@ func (c *KeyedCache[T]) Delete(key string) {
 	delete(c.entries, key)
 }
 
-func (c *KeyedCache[T]) Take(key string) (T, bool) {
+func (c *KeyedCache[T]) Pop(key string) (T, bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	if entry, exists := c.entries[key]; exists {

internal/conf/const.go

@@ -170,7 +170,7 @@ const (
 )
 
 // ContextKey is the type of context keys.
-type ContextKey int
+type ContextKey int8
 
 const (
 	_ ContextKey = iota
@@ -186,4 +186,5 @@ const (
 	UserAgentKey
 	PathKey
 	SharingIDKey
+	SkipHookKey
 )

internal/driver/driver.go

@@ -110,15 +110,6 @@ type PutURL interface {
 	PutURL(ctx context.Context, dstDir model.Obj, name, url string) error
 }
 
-//type WriteResult interface {
-//	MkdirResult
-//	MoveResult
-//	RenameResult
-//	CopyResult
-//	PutResult
-//	Remove
-//}
-
 type MkdirResult interface {
 	MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error)
 }