From c1d870198a82b35b66c31651173b12215a41033b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 18 Mar 2024 08:37:58 +0100 Subject: [PATCH 1/7] Refactor decomposedfs to be more extensible, e.g. for the posix driver --- go.mod | 2 +- go.sum | 4 +- pkg/storage/fs/posix/lookup/lookup.go | 11 ++ pkg/storage/fs/posix/tree/tree.go | 10 ++ .../utils/decomposedfs/aspects/aspects.go | 9 +- .../utils/decomposedfs/decomposedfs.go | 2 +- pkg/storage/utils/decomposedfs/grants.go | 2 +- .../utils/decomposedfs/lookup/lookup.go | 32 ++++ .../metadata/messagepack_backend.go | 16 ++ .../utils/decomposedfs/metadata/metadata.go | 8 + .../metadata/prefixes/prefixes.go | 2 + .../decomposedfs/metadata/xattrs_backend.go | 20 ++- .../utils/decomposedfs/node/mocks/Tree.go | 111 ++++++++++++++ pkg/storage/utils/decomposedfs/node/node.go | 92 ++++++----- .../utils/decomposedfs/options/options.go | 20 ++- pkg/storage/utils/decomposedfs/revisions.go | 5 +- pkg/storage/utils/decomposedfs/spaces.go | 114 +++++++------- pkg/storage/utils/decomposedfs/spaces_test.go | 42 ----- .../decomposedfs/tree/propagator/sync.go | 1 + pkg/storage/utils/decomposedfs/tree/tree.go | 24 ++- .../utils/decomposedfs/tree/tree_test.go | 24 +++ .../utils/decomposedfs/upload/store.go | 143 +++++++++--------- .../utils/decomposedfs/upload/store_test.go | 10 +- .../utils/decomposedfs/upload/upload.go | 33 +--- pkg/storage/utils/templates/templates.go | 8 +- 25 files changed, 481 insertions(+), 264 deletions(-) diff --git a/go.mod b/go.mod index 7a465ac5e5..c533a17d01 100644 --- a/go.mod +++ b/go.mod @@ -144,7 +144,7 @@ require ( github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect github.com/hashicorp/consul/api v1.15.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/go.sum b/go.sum index 18ee449008..33802cd93b 100644 --- a/go.sum +++ b/go.sum @@ -1164,8 +1164,9 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0 h1:zHs+jv3LO743/zFGcByu2KmpbliCU2AhjcGgrdTwSG4= +github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= @@ -1276,6 +1277,7 @@ github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSAS github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= diff --git a/pkg/storage/fs/posix/lookup/lookup.go b/pkg/storage/fs/posix/lookup/lookup.go index d08d94fd5f..111177d975 100644 --- a/pkg/storage/fs/posix/lookup/lookup.go +++ b/pkg/storage/fs/posix/lookup/lookup.go @@ -118,6 +118,17 @@ func (lu *Lookup) TypeFromPath(ctx context.Context, path string) provider.Resour return t } +func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) { + id, err := lu.metadataBackend.Get(ctx, filepath.Join(parent.InternalPath(), name), prefixes.IDAttr) + if err != nil { + if metadata.IsNotExist(err) { + return "", errtypes.NotFound(name) + } + return "", err + } + return string(id), nil +} + // NodeFromResource takes in a request path or request id and converts it to a Node func (lu *Lookup) NodeFromResource(ctx context.Context, ref *provider.Reference) (*node.Node, error) { ctx, span := tracer.Start(ctx, "NodeFromResource") diff --git a/pkg/storage/fs/posix/tree/tree.go b/pkg/storage/fs/posix/tree/tree.go index b157133cac..d9f693259b 100644 --- a/pkg/storage/fs/posix/tree/tree.go +++ b/pkg/storage/fs/posix/tree/tree.go @@ -687,6 +687,16 @@ func (t *Tree) DeleteBlob(node *node.Node) error { return t.blobstore.Delete(node) } +// BuildSpaceIDIndexEntry returns the entry for the space id index +func (t *Tree) BuildSpaceIDIndexEntry(spaceID, nodeID string) string { + return nodeID +} + +// ResolveSpaceIDIndexEntry returns the node id for the space id index entry +func (t *Tree) ResolveSpaceIDIndexEntry(spaceid, entry string) (string, string, error) { + return spaceid, entry, nil +} + // TODO check if node exists? func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) { ctx, span := tracer.Start(ctx, "createDirNode") diff --git a/pkg/storage/utils/decomposedfs/aspects/aspects.go b/pkg/storage/utils/decomposedfs/aspects/aspects.go index 28ee4495ad..c920c88b8f 100644 --- a/pkg/storage/utils/decomposedfs/aspects/aspects.go +++ b/pkg/storage/utils/decomposedfs/aspects/aspects.go @@ -26,8 +26,9 @@ import ( // Aspects holds dependencies for handling aspects of the decomposedfs type Aspects struct { - Lookup node.PathLookup - Tree node.Tree - Permissions permissions.Permissions - EventStream events.Stream + Lookup node.PathLookup + Tree node.Tree + Permissions permissions.Permissions + EventStream events.Stream + DisableVersioning bool } diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 9af86b4c43..95c4905c07 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -211,7 +211,7 @@ func New(o *options.Options, aspects aspects.Aspects) (storage.FS, error) { userSpaceIndex: userSpaceIndex, groupSpaceIndex: groupSpaceIndex, spaceTypeIndex: spaceTypeIndex, - sessionStore: upload.NewSessionStore(aspects.Lookup, aspects.Tree, o.Root, aspects.EventStream, o.AsyncFileUploads, o.Tokens), + sessionStore: upload.NewSessionStore(aspects.Lookup, aspects.Tree, o.Root, aspects.EventStream, o.AsyncFileUploads, o.Tokens, aspects.DisableVersioning), } if o.AsyncFileUploads { diff --git a/pkg/storage/utils/decomposedfs/grants.go b/pkg/storage/utils/decomposedfs/grants.go index 9dda19fa63..5990a5146c 100644 --- a/pkg/storage/utils/decomposedfs/grants.go +++ b/pkg/storage/utils/decomposedfs/grants.go @@ -327,7 +327,7 @@ func (fs *Decomposedfs) storeGrant(ctx context.Context, n *node.Node, g *provide } // update the indexes only after successfully setting the grant - err := fs.updateIndexes(ctx, g.GetGrantee(), spaceType, n.ID) + err := fs.updateIndexes(ctx, g.GetGrantee(), spaceType, n.SpaceID, n.ID) if err != nil { return err } diff --git a/pkg/storage/utils/decomposedfs/lookup/lookup.go b/pkg/storage/utils/decomposedfs/lookup/lookup.go index e7a8139839..c019f1c9e1 100644 --- a/pkg/storage/utils/decomposedfs/lookup/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup/lookup.go @@ -21,9 +21,11 @@ package lookup import ( "context" "fmt" + "io/fs" "os" "path/filepath" "strings" + "syscall" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -87,6 +89,36 @@ func (lu *Lookup) ReadBlobIDAttr(ctx context.Context, path string) (string, erro } return string(attr), nil } +func readChildNodeFromLink(path string) (string, error) { + link, err := os.Readlink(path) + if err != nil { + return "", err + } + nodeID := strings.TrimLeft(link, "/.") + nodeID = strings.ReplaceAll(nodeID, "/", "") + return nodeID, nil +} + +// The os error is buried inside the fs.PathError error +func isNotDir(err error) bool { + if perr, ok := err.(*fs.PathError); ok { + if serr, ok2 := perr.Err.(syscall.Errno); ok2 { + return serr == syscall.ENOTDIR + } + } + return false +} + +func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) { + nodeID, err := readChildNodeFromLink(filepath.Join(parent.InternalPath(), name)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) || isNotDir(err) { + return nodeID, nil // if the file does not exist we return a node that has Exists = false + } + return "", errors.Wrap(err, "decomposedfs: Wrap: readlink error") + } + return nodeID, nil +} // TypeFromPath returns the type of the node at the given path func (lu *Lookup) TypeFromPath(ctx context.Context, path string) provider.ResourceType { diff --git a/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go b/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go index ab7c7e8fdb..5c19821b48 100644 --- a/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go @@ -304,6 +304,22 @@ func (MessagePackBackend) MetadataPath(path string) string { return path + ".mpk // LockfilePath returns the path of the lock file func (MessagePackBackend) LockfilePath(path string) string { return path + ".mlock" } +// Lock locks the metadata for the given path +func (b MessagePackBackend) Lock(path string) (UnlockFunc, error) { + metaLockPath := b.LockfilePath(path) + mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return nil, err + } + return func() error { + err := mlock.Close() + if err != nil { + return err + } + return os.Remove(metaLockPath) + }, nil +} + func (b MessagePackBackend) cacheKey(path string) string { // rootPath is guaranteed to have no trailing slash // the cache key shouldn't begin with a slash as some stores drop it which can cause diff --git a/pkg/storage/utils/decomposedfs/metadata/metadata.go b/pkg/storage/utils/decomposedfs/metadata/metadata.go index 57557b7078..1b376c2dc2 100644 --- a/pkg/storage/utils/decomposedfs/metadata/metadata.go +++ b/pkg/storage/utils/decomposedfs/metadata/metadata.go @@ -35,6 +35,8 @@ func init() { var errUnconfiguredError = errors.New("no metadata backend configured. Bailing out") +type UnlockFunc func() error + // Backend defines the interface for file attribute backends type Backend interface { Name() string @@ -48,6 +50,7 @@ type Backend interface { SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) error Remove(ctx context.Context, path, key string, acquireLock bool) error + Lock(path string) (UnlockFunc, error) Purge(path string) error Rename(oldPath, newPath string) error IsMetaFile(path string) bool @@ -99,6 +102,11 @@ func (NullBackend) Remove(ctx context.Context, path string, key string, acquireL return errUnconfiguredError } +// Lock locks the metadata for the given path +func (NullBackend) Lock(path string) (UnlockFunc, error) { + return nil, nil +} + // IsMetaFile returns whether the given path represents a meta file func (NullBackend) IsMetaFile(path string) bool { return false } diff --git a/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go b/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go index 020e837fdf..fb0cbef28b 100644 --- a/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go +++ b/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go @@ -28,6 +28,7 @@ package prefixes // "user.ocis." in the xattrs_prefix*.go files. const ( TypeAttr string = OcisPrefix + "type" + IDAttr string = OcisPrefix + "id" ParentidAttr string = OcisPrefix + "parentid" OwnerIDAttr string = OcisPrefix + "owner.id" OwnerIDPAttr string = OcisPrefix + "owner.idp" @@ -89,6 +90,7 @@ const ( QuotaAttr string = OcisPrefix + "quota" // the name given to a storage space. It should not contain any semantics as its only purpose is to be read. + SpaceIDAttr string = OcisPrefix + "space.id" SpaceNameAttr string = OcisPrefix + "space.name" SpaceTypeAttr string = OcisPrefix + "space.type" SpaceDescriptionAttr string = OcisPrefix + "space.description" diff --git a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go index 7c60043590..eb173803d5 100644 --- a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go @@ -113,13 +113,13 @@ func (b XattrsBackend) Set(ctx context.Context, path string, key string, val []b } // SetMultiple sets a set of attribute for the given path -func (XattrsBackend) SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) (err error) { +func (b XattrsBackend) SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) (err error) { if acquireLock { err := os.MkdirAll(filepath.Dir(path), 0600) if err != nil { return err } - lockedFile, err := lockedfile.OpenFile(path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600) + lockedFile, err := lockedfile.OpenFile(b.LockfilePath(path), os.O_CREATE|os.O_WRONLY, 0600) if err != nil { return err } @@ -173,6 +173,22 @@ func (XattrsBackend) MetadataPath(path string) string { return path } // LockfilePath returns the path of the lock file func (XattrsBackend) LockfilePath(path string) string { return path + ".mlock" } +// Lock locks the metadata for the given path +func (b XattrsBackend) Lock(path string) (UnlockFunc, error) { + metaLockPath := b.LockfilePath(path) + mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return nil, err + } + return func() error { + err := mlock.Close() + if err != nil { + return err + } + return os.Remove(metaLockPath) + }, nil +} + func cleanupLockfile(f *lockedfile.File) { _ = f.Close() _ = os.Remove(f.Name()) diff --git a/pkg/storage/utils/decomposedfs/node/mocks/Tree.go b/pkg/storage/utils/decomposedfs/node/mocks/Tree.go index d6a07c8137..21f488e349 100644 --- a/pkg/storage/utils/decomposedfs/node/mocks/Tree.go +++ b/pkg/storage/utils/decomposedfs/node/mocks/Tree.go @@ -44,6 +44,53 @@ func (_m *Tree) EXPECT() *Tree_Expecter { return &Tree_Expecter{mock: &_m.Mock} } +// BuildSpaceIDIndexEntry provides a mock function with given fields: spaceID, nodeID +func (_m *Tree) BuildSpaceIDIndexEntry(spaceID string, nodeID string) string { + ret := _m.Called(spaceID, nodeID) + + if len(ret) == 0 { + panic("no return value specified for BuildSpaceIDIndexEntry") + } + + var r0 string + if rf, ok := ret.Get(0).(func(string, string) string); ok { + r0 = rf(spaceID, nodeID) + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// Tree_BuildSpaceIDIndexEntry_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BuildSpaceIDIndexEntry' +type Tree_BuildSpaceIDIndexEntry_Call struct { + *mock.Call +} + +// BuildSpaceIDIndexEntry is a helper method to define mock.On call +// - spaceID string +// - nodeID string +func (_e *Tree_Expecter) BuildSpaceIDIndexEntry(spaceID interface{}, nodeID interface{}) *Tree_BuildSpaceIDIndexEntry_Call { + return &Tree_BuildSpaceIDIndexEntry_Call{Call: _e.mock.On("BuildSpaceIDIndexEntry", spaceID, nodeID)} +} + +func (_c *Tree_BuildSpaceIDIndexEntry_Call) Run(run func(spaceID string, nodeID string)) *Tree_BuildSpaceIDIndexEntry_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string)) + }) + return _c +} + +func (_c *Tree_BuildSpaceIDIndexEntry_Call) Return(_a0 string) *Tree_BuildSpaceIDIndexEntry_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *Tree_BuildSpaceIDIndexEntry_Call) RunAndReturn(run func(string, string) string) *Tree_BuildSpaceIDIndexEntry_Call { + _c.Call.Return(run) + return _c +} + // CreateDir provides a mock function with given fields: ctx, _a1 func (_m *Tree) CreateDir(ctx context.Context, _a1 *node.Node) error { ret := _m.Called(ctx, _a1) @@ -526,6 +573,70 @@ func (_c *Tree_ReadBlob_Call) RunAndReturn(run func(*node.Node) (io.ReadCloser, return _c } +// ResolveSpaceIDIndexEntry provides a mock function with given fields: spaceID, entry +func (_m *Tree) ResolveSpaceIDIndexEntry(spaceID string, entry string) (string, string, error) { + ret := _m.Called(spaceID, entry) + + if len(ret) == 0 { + panic("no return value specified for ResolveSpaceIDIndexEntry") + } + + var r0 string + var r1 string + var r2 error + if rf, ok := ret.Get(0).(func(string, string) (string, string, error)); ok { + return rf(spaceID, entry) + } + if rf, ok := ret.Get(0).(func(string, string) string); ok { + r0 = rf(spaceID, entry) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string, string) string); ok { + r1 = rf(spaceID, entry) + } else { + r1 = ret.Get(1).(string) + } + + if rf, ok := ret.Get(2).(func(string, string) error); ok { + r2 = rf(spaceID, entry) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// Tree_ResolveSpaceIDIndexEntry_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ResolveSpaceIDIndexEntry' +type Tree_ResolveSpaceIDIndexEntry_Call struct { + *mock.Call +} + +// ResolveSpaceIDIndexEntry is a helper method to define mock.On call +// - spaceID string +// - entry string +func (_e *Tree_Expecter) ResolveSpaceIDIndexEntry(spaceID interface{}, entry interface{}) *Tree_ResolveSpaceIDIndexEntry_Call { + return &Tree_ResolveSpaceIDIndexEntry_Call{Call: _e.mock.On("ResolveSpaceIDIndexEntry", spaceID, entry)} +} + +func (_c *Tree_ResolveSpaceIDIndexEntry_Call) Run(run func(spaceID string, entry string)) *Tree_ResolveSpaceIDIndexEntry_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string)) + }) + return _c +} + +func (_c *Tree_ResolveSpaceIDIndexEntry_Call) Return(_a0 string, _a1 string, _a2 error) *Tree_ResolveSpaceIDIndexEntry_Call { + _c.Call.Return(_a0, _a1, _a2) + return _c +} + +func (_c *Tree_ResolveSpaceIDIndexEntry_Call) RunAndReturn(run func(string, string) (string, string, error)) *Tree_ResolveSpaceIDIndexEntry_Call { + _c.Call.Return(run) + return _c +} + // RestoreRecycleItemFunc provides a mock function with given fields: ctx, spaceid, key, trashPath, target func (_m *Tree) RestoreRecycleItemFunc(ctx context.Context, spaceid string, key string, trashPath string, target *node.Node) (*node.Node, *node.Node, func() error, error) { ret := _m.Called(ctx, spaceid, key, trashPath, target) diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 48432f2241..b63eb243f1 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -21,16 +21,16 @@ package node import ( "context" "crypto/md5" + "crypto/sha1" "encoding/hex" "fmt" "hash" + "hash/adler32" "io" - "io/fs" "os" "path/filepath" "strconv" "strings" - "syscall" "time" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -103,6 +103,9 @@ type Tree interface { ReadBlob(node *Node) (io.ReadCloser, error) DeleteBlob(node *Node) error + BuildSpaceIDIndexEntry(spaceID, nodeID string) string + ResolveSpaceIDIndexEntry(spaceID, entry string) (string, string, error) + Propagate(ctx context.Context, node *Node, sizeDiff int64) (err error) } @@ -112,7 +115,10 @@ type PathLookup interface { NodeFromResource(ctx context.Context, ref *provider.Reference) (*Node, error) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *Node, err error) + NodeIDFromParentAndName(ctx context.Context, n *Node, name string) (string, error) + GenerateSpaceID(spaceType string, owner *userpb.User) (string, error) + InternalRoot() string InternalPath(spaceID, nodeID string) string Path(ctx context.Context, n *Node, hasPermission PermissionFunc) (path string, err error) @@ -124,6 +130,11 @@ type PathLookup interface { CopyMetadata(ctx context.Context, src, target string, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error) } +type IDCacher interface { + CacheID(ctx context.Context, spaceID, nodeID, val string) error + GetCachedID(ctx context.Context, spaceID, nodeID string) (string, bool) +} + // Node represents a node in the tree and provides methods to get a Parent or Child instance type Node struct { SpaceID string @@ -362,26 +373,6 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis return n, nil } -// The os error is buried inside the fs.PathError error -func isNotDir(err error) bool { - if perr, ok := err.(*fs.PathError); ok { - if serr, ok2 := perr.Err.(syscall.Errno); ok2 { - return serr == syscall.ENOTDIR - } - } - return false -} - -func readChildNodeFromLink(path string) (string, error) { - link, err := os.Readlink(path) - if err != nil { - return "", err - } - nodeID := strings.TrimLeft(link, "/.") - nodeID = strings.ReplaceAll(nodeID, "/", "") - return nodeID, nil -} - // Child returns the child node with the given name func (n *Node) Child(ctx context.Context, name string) (*Node, error) { ctx, span := tracer.Start(ctx, "Child") @@ -393,24 +384,24 @@ func (n *Node) Child(ctx context.Context, name string) (*Node, error) { } else if n.SpaceRoot != nil { spaceID = n.SpaceRoot.ID } - nodeID, err := readChildNodeFromLink(filepath.Join(n.InternalPath(), name)) - if err != nil { - if errors.Is(err, fs.ErrNotExist) || isNotDir(err) { - - c := &Node{ - SpaceID: spaceID, - lu: n.lu, - ParentID: n.ID, - Name: name, - SpaceRoot: n.SpaceRoot, - } - return c, nil // if the file does not exist we return a node that has Exists = false - } + c := &Node{ + SpaceID: spaceID, + lu: n.lu, + ParentID: n.ID, + Name: name, + SpaceRoot: n.SpaceRoot, + } - return nil, errors.Wrap(err, "decomposedfs: Wrap: readlink error") + nodeID, err := n.lu.NodeIDFromParentAndName(ctx, n, name) + switch err.(type) { + case nil: + // ok + case errtypes.IsNotFound: + return c, nil // if the file does not exist we return a node that has Exists = false + default: + return nil, err } - var c *Node c, err = ReadNode(ctx, n.lu, spaceID, nodeID, false, n.SpaceRoot, true) if err != nil { return nil, errors.Wrap(err, "could not read child node") @@ -1360,3 +1351,30 @@ func enoughDiskSpace(path string, fileSize uint64) bool { } return avalB > fileSize } + +// CalculateChecksums calculates the sha1, md5 and adler32 checksums of a file +func CalculateChecksums(ctx context.Context, path string) (hash.Hash, hash.Hash, hash.Hash32, error) { + sha1h := sha1.New() + md5h := md5.New() + adler32h := adler32.New() + + _, subspan := tracer.Start(ctx, "os.Open") + f, err := os.Open(path) + subspan.End() + if err != nil { + return nil, nil, nil, err + } + defer f.Close() + + r1 := io.TeeReader(f, sha1h) + r2 := io.TeeReader(r1, md5h) + + _, subspan = tracer.Start(ctx, "io.Copy") + _, err = io.Copy(adler32h, r2) + subspan.End() + if err != nil { + return nil, nil, nil, err + } + + return sha1h, md5h, adler32h, nil +} diff --git a/pkg/storage/utils/decomposedfs/options/options.go b/pkg/storage/utils/decomposedfs/options/options.go index 086490879b..0afd9d309d 100644 --- a/pkg/storage/utils/decomposedfs/options/options.go +++ b/pkg/storage/utils/decomposedfs/options/options.go @@ -50,9 +50,15 @@ type Options struct { // ocis fs works on top of a dir of uuid nodes Root string `mapstructure:"root"` + // the upload directory where uploads in progress are stored + UploadDirectory string `mapstructure:"upload_directory"` + // UserLayout describes the relative path from the storage's root node to the users home node. UserLayout string `mapstructure:"user_layout"` + // ProjectLayout describes the relative path from the storage's root node to the project spaces root directory. + ProjectLayout string `mapstructure:"project_layout"` + // propagate mtime changes as tmtime (tree modification time) to the parent directory when user.ocis.propagation=1 is set on a node TreeTimeAccounting bool `mapstructure:"treetime_accounting"` @@ -73,10 +79,9 @@ type Options struct { Tokens TokenOptions `mapstructure:"tokens"` - // FileMetadataCache for file metadata + StatCache cache.Config `mapstructure:"statcache"` FileMetadataCache cache.Config `mapstructure:"filemetadatacache"` - // IDCache for symlink lookups of direntry to node id - IDCache cache.Config `mapstructure:"idcache"` + IDCache cache.Config `mapstructure:"idcache"` MaxAcquireLockCycles int `mapstructure:"max_acquire_lock_cycles"` LockCycleDurationFactor int `mapstructure:"lock_cycle_duration_factor"` @@ -120,6 +125,11 @@ func New(m map[string]interface{}) (*Options, error) { if o.UserLayout == "" { o.UserLayout = "{{.Id.OpaqueId}}" } + + if o.ProjectLayout == "" { + o.ProjectLayout = "{{.Id.OpaqueId}}" + } + // ensure user layout has no starting or trailing / o.UserLayout = strings.Trim(o.UserLayout, "/") @@ -160,5 +170,9 @@ func New(m map[string]interface{}) (*Options, error) { o.AsyncPropagatorOptions.PropagationDelay = 5 * time.Second } + if o.UploadDirectory == "" { + o.UploadDirectory = filepath.Join(o.Root, "uploads") + } + return o, nil } diff --git a/pkg/storage/utils/decomposedfs/revisions.go b/pkg/storage/utils/decomposedfs/revisions.go index c079c3bddb..93ea47c718 100644 --- a/pkg/storage/utils/decomposedfs/revisions.go +++ b/pkg/storage/utils/decomposedfs/revisions.go @@ -213,7 +213,10 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer if err != nil { return err } - defer f.Close() + defer func() { + _ = f.Close() + _ = os.Remove(fs.lu.MetadataBackend().LockfilePath(n.InternalPath())) + }() // move current version to new revision nodePath := fs.lu.InternalPath(spaceID, kp[0]) diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 6a555ea12f..1f80676017 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -45,7 +45,6 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/permissions" - "github.com/cs3org/reva/v2/pkg/storage/utils/filelocks" "github.com/cs3org/reva/v2/pkg/storage/utils/templates" "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/utils" @@ -73,21 +72,17 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr if err != nil { return nil, err } - // allow sending a space id if reqSpaceID := utils.ReadPlainFromOpaque(req.Opaque, "spaceid"); reqSpaceID != "" { spaceID = reqSpaceID } - // allow sending a space description + description := utils.ReadPlainFromOpaque(req.Opaque, "description") - // allow sending a spaceAlias alias := utils.ReadPlainFromOpaque(req.Opaque, "spaceAlias") if alias == "" { - alias = templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, fs.o.GeneralSpaceAliasTemplate) + alias = templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, spaceID, fs.o.GeneralSpaceAliasTemplate) } - // TODO enforce a uuid? - // TODO clarify if we want to enforce a single personal storage space or if we want to allow sending the spaceid if req.Type == _spaceTypePersonal { - alias = templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, fs.o.PersonalSpaceAliasTemplate) + alias = templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, spaceID, fs.o.PersonalSpaceAliasTemplate) } root, err := node.ReadNode(ctx, fs.lu, spaceID, spaceID, true, nil, false) // will fall into `Exists` case below @@ -103,11 +98,28 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr // create a directory node root.SetType(provider.ResourceType_RESOURCE_TYPE_CONTAINER) rootPath := root.InternalPath() + switch req.Type { + case _spaceTypePersonal: + if fs.o.UserLayout != "" { + rootPath = filepath.Join(fs.o.Root, templates.WithUser(u, fs.o.UserLayout)) + } + case _spaceTypeProject: + if fs.o.ProjectLayout != "" { + rootPath = filepath.Join(fs.o.Root, templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, spaceID, fs.o.ProjectLayout)) + } + } if err := os.MkdirAll(rootPath, 0700); err != nil { return nil, errors.Wrap(err, "Decomposedfs: error creating node") } + // Store id in cache + if c, ok := fs.lu.(node.IDCacher); ok { + if err := c.CacheID(ctx, spaceID, spaceID, rootPath); err != nil { + return nil, err + } + } + if req.GetOwner() != nil && req.GetOwner().GetId() != nil { root.SetOwner(req.GetOwner().GetId()) } else { @@ -115,6 +127,8 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr } metadata := node.Attributes{} + metadata.SetString(prefixes.IDAttr, spaceID) + metadata.SetString(prefixes.SpaceIDAttr, spaceID) metadata.SetString(prefixes.OwnerIDAttr, root.Owner().GetOpaqueId()) metadata.SetString(prefixes.OwnerIDPAttr, root.Owner().GetIdp()) metadata.SetString(prefixes.OwnerTypeAttr, utils.UserTypeToString(root.Owner().GetType())) @@ -159,7 +173,7 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr err = fs.updateIndexes(ctx, &provider.Grantee{ Type: provider.GranteeType_GRANTEE_TYPE_USER, Id: &provider.Grantee_UserId{UserId: req.GetOwner().GetId()}, - }, req.Type, root.ID) + }, req.Type, root.ID, root.ID) if err != nil { return nil, err } @@ -199,19 +213,6 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr return resp, nil } -// ReadSpaceAndNodeFromIndexLink reads a symlink and parses space and node id if the link has the correct format, eg: -// ../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51 -// ../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z -func ReadSpaceAndNodeFromIndexLink(link string) (string, string, error) { - // ../../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid - // 0 1 2 3 4 5 6 7 8 9 10 11 - parts := strings.Split(link, string(filepath.Separator)) - if len(parts) != 12 || parts[0] != ".." || parts[1] != ".." || parts[2] != ".." || parts[3] != "spaces" || parts[6] != "nodes" { - return "", "", errtypes.InternalError("malformed link") - } - return strings.Join(parts[4:6], ""), strings.Join(parts[7:12], ""), nil -} - // ListStorageSpaces returns a list of StorageSpaces. // The list can be filtered by space type or space id. // Spaces are persisted with symlinks in /spaces// pointing to ../../nodes/, the root node of the space @@ -302,7 +303,7 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide return spaces, nil } - matches := map[string]struct{}{} + matches := map[string]string{} var allMatches map[string]string var err error @@ -313,11 +314,11 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide } if nodeID == spaceIDAny { - for _, match := range allMatches { - matches[match] = struct{}{} + for spaceID, nodeID := range allMatches { + matches[spaceID] = nodeID } } else { - matches[allMatches[nodeID]] = struct{}{} + matches[allMatches[nodeID]] = allMatches[nodeID] } // get Groups for userid @@ -340,11 +341,11 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide } if nodeID == spaceIDAny { - for _, match := range allMatches { - matches[match] = struct{}{} + for spaceID, nodeID := range allMatches { + matches[spaceID] = nodeID } } else { - matches[allMatches[nodeID]] = struct{}{} + matches[allMatches[nodeID]] = allMatches[nodeID] } } @@ -370,11 +371,11 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide } if nodeID == spaceIDAny { - for _, match := range allMatches { - matches[match] = struct{}{} + for spaceID, nodeID := range allMatches { + matches[spaceID] = nodeID } } else { - matches[allMatches[nodeID]] = struct{}{} + matches[allMatches[nodeID]] = allMatches[nodeID] } } } @@ -391,15 +392,15 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide // the personal spaces must also use the nodeid and not the name numShares := atomic.Int64{} errg, ctx := errgroup.WithContext(ctx) - work := make(chan string, len(matches)) + work := make(chan []string, len(matches)) results := make(chan *provider.StorageSpace, len(matches)) // Distribute work errg.Go(func() error { defer close(work) - for match := range matches { + for spaceID, nodeID := range matches { select { - case work <- match: + case work <- []string{spaceID, nodeID}: case <-ctx.Done(): return ctx.Err() } @@ -415,26 +416,15 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide for i := 0; i < numWorkers; i++ { errg.Go(func() error { for match := range work { - var err error - // TODO introduce metadata.IsLockFile(path) - // do not investigate flock files any further. They indicate file locks but are not relevant here. - if strings.HasSuffix(match, filelocks.LockFileSuffix) { - continue - } - // skip metadata files - if fs.lu.MetadataBackend().IsMetaFile(match) { - continue - } - // always read link in case storage space id != node id - linkSpaceID, linkNodeID, err := ReadSpaceAndNodeFromIndexLink(match) + spaceID, nodeID, err := fs.tp.ResolveSpaceIDIndexEntry(match[0], match[1]) if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("match", match).Msg("could not read link, skipping") + appctx.GetLogger(ctx).Error().Err(err).Str("id", nodeID).Msg("resolve space id index entry, skipping") continue } - n, err := node.ReadNode(ctx, fs.lu, linkSpaceID, linkNodeID, true, nil, true) + n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID, true, nil, true) if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("id", linkNodeID).Msg("could not read node, skipping") + appctx.GetLogger(ctx).Error().Err(err).Str("id", nodeID).Msg("could not read node, skipping") continue } @@ -450,7 +440,7 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide case errtypes.NotFound: // ok default: - appctx.GetLogger(ctx).Error().Err(err).Str("id", linkNodeID).Msg("could not convert to storage space") + appctx.GetLogger(ctx).Error().Err(err).Str("id", nodeID).Msg("could not convert to storage space") } continue } @@ -504,7 +494,6 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide } return spaces, nil - } // UserIDToUserAndGroups converts a user ID to a user with groups @@ -752,8 +741,12 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De return n.SetDTime(ctx, &dtime) } -func (fs *Decomposedfs) updateIndexes(ctx context.Context, grantee *provider.Grantee, spaceType, spaceID string) error { - err := fs.linkStorageSpaceType(ctx, spaceType, spaceID) +// the value of `target` depends on the implementation: +// - for ocis/s3ng it is the relative link to the space root +// - for the posixfs it is the node id +func (fs *Decomposedfs) updateIndexes(ctx context.Context, grantee *provider.Grantee, spaceType, spaceID, nodeID string) error { + target := fs.tp.BuildSpaceIDIndexEntry(spaceID, nodeID) + err := fs.linkStorageSpaceType(ctx, spaceType, spaceID, target) if err != nil { return err } @@ -766,26 +759,23 @@ func (fs *Decomposedfs) updateIndexes(ctx context.Context, grantee *provider.Gra // create space grant index switch { case grantee.Type == provider.GranteeType_GRANTEE_TYPE_USER: - return fs.linkSpaceByUser(ctx, grantee.GetUserId().GetOpaqueId(), spaceID) + return fs.linkSpaceByUser(ctx, grantee.GetUserId().GetOpaqueId(), spaceID, target) case grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP: - return fs.linkSpaceByGroup(ctx, grantee.GetGroupId().GetOpaqueId(), spaceID) + return fs.linkSpaceByGroup(ctx, grantee.GetGroupId().GetOpaqueId(), spaceID, target) default: return errtypes.BadRequest("invalid grantee type: " + grantee.GetType().String()) } } -func (fs *Decomposedfs) linkSpaceByUser(ctx context.Context, userID, spaceID string) error { - target := "../../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) +func (fs *Decomposedfs) linkSpaceByUser(ctx context.Context, userID, spaceID, target string) error { return fs.userSpaceIndex.Add(userID, spaceID, target) } -func (fs *Decomposedfs) linkSpaceByGroup(ctx context.Context, groupID, spaceID string) error { - target := "../../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) +func (fs *Decomposedfs) linkSpaceByGroup(ctx context.Context, groupID, spaceID, target string) error { return fs.groupSpaceIndex.Add(groupID, spaceID, target) } -func (fs *Decomposedfs) linkStorageSpaceType(ctx context.Context, spaceType string, spaceID string) error { - target := "../../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) +func (fs *Decomposedfs) linkStorageSpaceType(ctx context.Context, spaceType, spaceID, target string) error { return fs.spaceTypeIndex.Add(spaceType, spaceID, target) } diff --git a/pkg/storage/utils/decomposedfs/spaces_test.go b/pkg/storage/utils/decomposedfs/spaces_test.go index 21071750c7..39c4f5d066 100644 --- a/pkg/storage/utils/decomposedfs/spaces_test.go +++ b/pkg/storage/utils/decomposedfs/spaces_test.go @@ -20,7 +20,6 @@ package decomposedfs_test import ( "context" - "os" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" cs3permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" @@ -28,7 +27,6 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" helpers "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/testhelpers" . "github.com/onsi/ginkgo/v2" @@ -254,46 +252,6 @@ var _ = Describe("Spaces", func() { }) }) - Describe("ReadSpaceAndNodeFromSpaceTypeLink", func() { - var ( - tmpdir string - ) - - BeforeEach(func() { - tmpdir, _ = os.MkdirTemp(os.TempDir(), "ReadSpaceAndNodeFromSpaceTypeLink-") - }) - - AfterEach(func() { - if tmpdir != "" { - os.RemoveAll(tmpdir) - } - }) - - DescribeTable("ReadSpaceAndNodeFromSpaceTypeLink", - func(link string, expectSpace string, expectedNode string, shouldErr bool) { - space, node, err := decomposedfs.ReadSpaceAndNodeFromIndexLink(link) - if shouldErr { - Expect(err).To(HaveOccurred()) - } else { - Expect(err).ToNot(HaveOccurred()) - } - Expect(space).To(Equal(expectSpace)) - Expect(node).To(Equal(expectedNode)) - }, - - Entry("invalid number of slashes", "../../../spaces/sp_ace-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("does not contain spaces", "../../../spac_s/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("does not contain nodes", "../../../spaces/sp/ace-id/nod_s/sh/or/tn/od/eid", "", "", true), - Entry("does not start with ..", "_./../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("does not start with ../..", "../_./../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("does not start with ../../..", "../_./../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("invalid", "../../../spaces/space-id/nodes/sh/or/tn/od/eid", "", "", true), - Entry("uuid", "../../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51", false), - Entry("uuid", "../../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z", "4c510ada-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z", false), - Entry("short", "../../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "space-id", "shortnodeid", false), - ) - }) - Describe("Update Space", func() { var ( env *helpers.TestEnv diff --git a/pkg/storage/utils/decomposedfs/tree/propagator/sync.go b/pkg/storage/utils/decomposedfs/tree/propagator/sync.go index f3729cb971..5043d19c90 100644 --- a/pkg/storage/utils/decomposedfs/tree/propagator/sync.go +++ b/pkg/storage/utils/decomposedfs/tree/propagator/sync.go @@ -98,6 +98,7 @@ func (p SyncPropagator) Propagate(ctx context.Context, n *node.Node, sizeDiff in if err == nil && cerr != nil && !errors.Is(cerr, os.ErrClosed) { err = cerr // only overwrite err with en error from close if the former was nil } + _ = os.Remove(f.Name()) }() if n, err = n.Parent(ctx); err != nil { diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index 3d8a057d8b..e5b0f96e13 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -68,7 +68,6 @@ type Tree struct { options *options.Options - // used to cache symlink lookups for child names to node ids idCache store.Store } @@ -766,6 +765,29 @@ func (t *Tree) DeleteBlob(node *node.Node) error { return t.blobstore.Delete(node) } +// BuildSpaceIDIndexEntry returns the entry for the space id index +func (t *Tree) BuildSpaceIDIndexEntry(spaceID, nodeID string) string { + return "../../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) +} + +// ResolveSpaceIDIndexEntry returns the node id for the space id index entry +func (t *Tree) ResolveSpaceIDIndexEntry(_, entry string) (string, string, error) { + return ReadSpaceAndNodeFromIndexLink(entry) +} + +// ReadSpaceAndNodeFromIndexLink reads a symlink and parses space and node id if the link has the correct format, eg: +// ../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51 +// ../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z +func ReadSpaceAndNodeFromIndexLink(link string) (string, string, error) { + // ../../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid + // 0 1 2 3 4 5 6 7 8 9 10 11 + parts := strings.Split(link, string(filepath.Separator)) + if len(parts) != 12 || parts[0] != ".." || parts[1] != ".." || parts[2] != ".." || parts[3] != "spaces" || parts[6] != "nodes" { + return "", "", errtypes.InternalError("malformed link") + } + return strings.Join(parts[4:6], ""), strings.Join(parts[7:12], ""), nil +} + // TODO check if node exists? func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) { ctx, span := tracer.Start(ctx, "createDirNode") diff --git a/pkg/storage/utils/decomposedfs/tree/tree_test.go b/pkg/storage/utils/decomposedfs/tree/tree_test.go index 74acdf7473..5701158c05 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree_test.go +++ b/pkg/storage/utils/decomposedfs/tree/tree_test.go @@ -420,4 +420,28 @@ var _ = Describe("Tree", func() { }) }) }) + + DescribeTable("ReadSpaceAndNodeFromIndexLink", + func(link string, expectSpace string, expectedNode string, shouldErr bool) { + space, node, err := tree.ReadSpaceAndNodeFromIndexLink(link) + if shouldErr { + Expect(err).To(HaveOccurred()) + } else { + Expect(err).ToNot(HaveOccurred()) + } + Expect(space).To(Equal(expectSpace)) + Expect(node).To(Equal(expectedNode)) + }, + + Entry("invalid number of slashes", "../../../spaces/sp_ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("does not contain spaces", "../../../spac_s/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("does not contain nodes", "../../../spaces/sp/ace-id/nod_s/sh/or/tn/od/eid", "", "", true), + Entry("does not start with ..", "_./../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("does not start with ../..", "../_./../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("does not start with ../../..", "../_./../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("invalid", "../../../spaces/space-id/nodes/sh/or/tn/od/eid", "", "", true), + Entry("uuid", "../../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51", false), + Entry("uuid", "../../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z", "4c510ada-c86b-4815-8820-42cdf82c3d51", "4c510ada-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z", false), + Entry("short", "../../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid", "space-id", "shortnodeid", false), + ) }) diff --git a/pkg/storage/utils/decomposedfs/upload/store.go b/pkg/storage/utils/decomposedfs/upload/store.go index b880b39baa..f5841fcc5c 100644 --- a/pkg/storage/utils/decomposedfs/upload/store.go +++ b/pkg/storage/utils/decomposedfs/upload/store.go @@ -34,7 +34,7 @@ import ( "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/events" - "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" @@ -53,23 +53,25 @@ type PermissionsChecker interface { // OcisStore manages upload sessions type OcisStore struct { - lu node.PathLookup - tp Tree - root string - pub events.Publisher - async bool - tknopts options.TokenOptions + lu node.PathLookup + tp Tree + root string + pub events.Publisher + async bool + tknopts options.TokenOptions + disableVersioning bool } // NewSessionStore returns a new OcisStore -func NewSessionStore(lu node.PathLookup, tp Tree, root string, pub events.Publisher, async bool, tknopts options.TokenOptions) *OcisStore { +func NewSessionStore(lu node.PathLookup, tp Tree, root string, pub events.Publisher, async bool, tknopts options.TokenOptions, DisableVersioning bool) *OcisStore { return &OcisStore{ - lu: lu, - tp: tp, - root: root, - pub: pub, - async: async, - tknopts: tknopts, + lu: lu, + tp: tp, + root: root, + pub: pub, + async: async, + tknopts: tknopts, + disableVersioning: DisableVersioning, } } @@ -204,7 +206,7 @@ func (store OcisStore) CreateNodeForUpload(session *OcisSession, initAttrs node. return nil, err } - var f *lockedfile.File + var unlock metadata.UnlockFunc if session.NodeExists() { // TODO this is wrong. The node should be created when the upload starts, the revisions should be created independently of the node // we do not need to propagate a change when a node is created, only when the upload is ready. // that still creates problems for desktop clients because if another change causes propagation it will detects an empty file @@ -214,21 +216,25 @@ func (store OcisStore) CreateNodeForUpload(session *OcisSession, initAttrs node. // so we have to check if the node has been created meanwhile (well, only in case the upload does not know the nodeid ... or the NodeExists array that is checked by session.NodeExists()) // FIXME look at the disk again to see if the file has been created in between, or just try initializing a new node and do the update existing node as a fallback. <- the latter! - f, err = store.updateExistingNode(ctx, session, n, session.SpaceID(), uint64(session.Size())) - if f != nil { - appctx.GetLogger(ctx).Debug().Str("lockfile", f.Name()).Interface("err", err).Msg("got lock file from updateExistingNode") + unlock, err = store.updateExistingNode(ctx, session, n, session.SpaceID(), uint64(session.Size())) + if unlock != nil { + appctx.GetLogger(ctx).Info().Interface("err", err).Msg("got lock file from updateExistingNode") } } else { - f, err = store.initNewNode(ctx, session, n, uint64(session.Size())) - if f != nil { - appctx.GetLogger(ctx).Debug().Str("lockfile", f.Name()).Interface("err", err).Msg("got lock file from initNewNode") + if c, ok := store.lu.(node.IDCacher); ok { + c.CacheID(ctx, n.SpaceID, n.ID, filepath.Join(n.ParentPath(), n.Name)) + } + unlock, err = store.initNewNode(ctx, session, n, uint64(session.Size())) + if unlock != nil { + appctx.GetLogger(ctx).Info().Interface("err", err).Msg("got lock file from initNewNode") } } defer func() { - if f == nil { + if unlock == nil { return } - if err := f.Close(); err != nil { + + if err := unlock(); err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Str("parentid", n.ParentID).Msg("could not close lock") } }() @@ -243,6 +249,7 @@ func (store OcisStore) CreateNodeForUpload(session *OcisSession, initAttrs node. } // overwrite technical information + initAttrs.SetString(prefixes.IDAttr, n.ID) initAttrs.SetString(prefixes.MTimeAttr, mtime.UTC().Format(time.RFC3339Nano)) initAttrs.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE)) initAttrs.SetString(prefixes.ParentidAttr, n.ParentID) @@ -264,14 +271,14 @@ func (store OcisStore) CreateNodeForUpload(session *OcisSession, initAttrs node. return n, nil } -func (store OcisStore) initNewNode(ctx context.Context, session *OcisSession, n *node.Node, fsize uint64) (*lockedfile.File, error) { +func (store OcisStore) initNewNode(ctx context.Context, session *OcisSession, n *node.Node, fsize uint64) (metadata.UnlockFunc, error) { // create folder structure (if needed) if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil { return nil, err } // create and write lock new node metadata - f, err := lockedfile.OpenFile(store.lu.MetadataBackend().LockfilePath(n.InternalPath()), os.O_RDWR|os.O_CREATE, 0600) + unlock, err := store.lu.MetadataBackend().Lock(n.InternalPath()) if err != nil { return nil, err } @@ -279,36 +286,20 @@ func (store OcisStore) initNewNode(ctx context.Context, session *OcisSession, n // we also need to touch the actual node file here it stores the mtime of the resource h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600) if err != nil { - return f, err + return unlock, err } h.Close() if _, err := node.CheckQuota(ctx, n.SpaceRoot, false, 0, fsize); err != nil { - return f, err + return unlock, err } - // link child name to parent if it is new - childNameLink := filepath.Join(n.ParentPath(), n.Name) - relativeNodePath := filepath.Join("../../../../../", lookup.Pathify(n.ID, 4, 2)) - log := appctx.GetLogger(ctx).With().Str("childNameLink", childNameLink).Str("relativeNodePath", relativeNodePath).Logger() - log.Info().Msg("initNewNode: creating symlink") - - if err = os.Symlink(relativeNodePath, childNameLink); err != nil { - log.Info().Err(err).Msg("initNewNode: symlink failed") - if errors.Is(err, iofs.ErrExist) { - log.Info().Err(err).Msg("initNewNode: symlink already exists") - return f, errtypes.AlreadyExists(n.Name) - } - return f, errors.Wrap(err, "Decomposedfs: could not symlink child entry") - } - log.Info().Msg("initNewNode: symlink created") - // on a new file the sizeDiff is the fileSize session.info.MetaData["sizeDiff"] = strconv.FormatInt(int64(fsize), 10) - return f, nil + return unlock, nil } -func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSession, n *node.Node, spaceID string, fsize uint64) (*lockedfile.File, error) { +func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSession, n *node.Node, spaceID string, fsize uint64) (metadata.UnlockFunc, error) { targetPath := n.InternalPath() // write lock existing node before reading any metadata @@ -317,35 +308,43 @@ func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSess return nil, err } + unlock := func() error { + err := f.Close() + if err != nil { + return err + } + return os.Remove(store.lu.MetadataBackend().LockfilePath(targetPath)) + } + old, _ := node.ReadNode(ctx, store.lu, spaceID, n.ID, false, nil, false) if _, err := node.CheckQuota(ctx, n.SpaceRoot, true, uint64(old.Blobsize), fsize); err != nil { - return f, err + return unlock, err } oldNodeMtime, err := old.GetMTime(ctx) if err != nil { - return f, err + return unlock, err } oldNodeEtag, err := node.CalculateEtag(old.ID, oldNodeMtime) if err != nil { - return f, err + return unlock, err } // When the if-match header was set we need to check if the // etag still matches before finishing the upload. if session.HeaderIfMatch() != "" && session.HeaderIfMatch() != oldNodeEtag { - return f, errtypes.Aborted("etag mismatch") + return unlock, errtypes.Aborted("etag mismatch") } // When the if-none-match header was set we need to check if any of the // etags matches before finishing the upload. if session.HeaderIfNoneMatch() != "" { if session.HeaderIfNoneMatch() == "*" { - return f, errtypes.Aborted("etag mismatch, resource exists") + return unlock, errtypes.Aborted("etag mismatch, resource exists") } for _, ifNoneMatchTag := range strings.Split(session.HeaderIfNoneMatch(), ",") { if ifNoneMatchTag == oldNodeEtag { - return f, errtypes.Aborted("etag mismatch") + return unlock, errtypes.Aborted("etag mismatch") } } } @@ -355,37 +354,41 @@ func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSess if session.HeaderIfUnmodifiedSince() != "" { ifUnmodifiedSince, err := time.Parse(time.RFC3339Nano, session.HeaderIfUnmodifiedSince()) if err != nil { - return f, errtypes.InternalError(fmt.Sprintf("failed to parse if-unmodified-since time: %s", err)) + return unlock, errtypes.InternalError(fmt.Sprintf("failed to parse if-unmodified-since time: %s", err)) } if oldNodeMtime.After(ifUnmodifiedSince) { - return f, errtypes.Aborted("if-unmodified-since mismatch") + return unlock, errtypes.Aborted("if-unmodified-since mismatch") } } - session.info.MetaData["versionsPath"] = session.store.lu.InternalPath(spaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano)) - session.info.MetaData["sizeDiff"] = strconv.FormatInt((int64(fsize) - old.Blobsize), 10) + versionPath := n.InternalPath() + if !store.disableVersioning { + versionPath = session.store.lu.InternalPath(spaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano)) - // create version node - if _, err := os.Create(session.info.MetaData["versionsPath"]); err != nil { - return f, err - } + // create version node + if _, err := os.Create(session.info.MetaData["versionsPath"]); err != nil { + return unlock, err + } - // copy blob metadata to version node - if err := store.lu.CopyMetadataWithSourceLock(ctx, targetPath, session.info.MetaData["versionsPath"], func(attributeName string, value []byte) (newValue []byte, copy bool) { - return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || - attributeName == prefixes.TypeAttr || - attributeName == prefixes.BlobIDAttr || - attributeName == prefixes.BlobsizeAttr || - attributeName == prefixes.MTimeAttr - }, f, true); err != nil { - return f, err + // copy blob metadata to version node + if err := store.lu.CopyMetadataWithSourceLock(ctx, targetPath, session.info.MetaData["versionsPath"], func(attributeName string, value []byte) (newValue []byte, copy bool) { + return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || + attributeName == prefixes.TypeAttr || + attributeName == prefixes.BlobIDAttr || + attributeName == prefixes.BlobsizeAttr || + attributeName == prefixes.MTimeAttr + }, f, true); err != nil { + return unlock, err + } } + session.info.MetaData["sizeDiff"] = strconv.FormatInt((int64(fsize) - old.Blobsize), 10) + session.info.MetaData["versionsPath"] = versionPath // keep mtime from previous version if err := os.Chtimes(session.info.MetaData["versionsPath"], oldNodeMtime, oldNodeMtime); err != nil { - return f, errtypes.InternalError(fmt.Sprintf("failed to change mtime of version node: %s", err)) + return unlock, errtypes.InternalError(fmt.Sprintf("failed to change mtime of version node: %s", err)) } - return f, nil + return unlock, nil } diff --git a/pkg/storage/utils/decomposedfs/upload/store_test.go b/pkg/storage/utils/decomposedfs/upload/store_test.go index c0863628f8..f287f62bca 100644 --- a/pkg/storage/utils/decomposedfs/upload/store_test.go +++ b/pkg/storage/utils/decomposedfs/upload/store_test.go @@ -22,7 +22,7 @@ func TestInitNewNode(t *testing.T) { lookup := lookup.New(metadata.NewMessagePackBackend(root, cache.Config{}), &options.Options{Root: root}) - store := NewSessionStore(lookup, nil, root, nil, false, options.TokenOptions{}) + store := NewSessionStore(lookup, nil, root, nil, false, options.TokenOptions{}, false) rootNode := node.New("e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "", "", 0, "", providerv1beta1.ResourceType_RESOURCE_TYPE_CONTAINER, &userv1beta1.UserId{}, lookup) rootNode.Exists = true @@ -34,18 +34,18 @@ func TestInitNewNode(t *testing.T) { } n := node.New("e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "930b7a2e-b745-41e1-8a9b-712582021842", "e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "newchild", 10, "26493c53-2634-45f8-949f-dc07b88df9b0", providerv1beta1.ResourceType_RESOURCE_TYPE_FILE, &userv1beta1.UserId{}, lookup) n.SpaceRoot = rootNode - f, err := store.initNewNode(context.Background(), store.New(context.Background()), n, 10) + unlock, err := store.initNewNode(context.Background(), store.New(context.Background()), n, 10) if err != nil { t.Fatalf(err.Error()) } - defer f.Close() + defer unlock() // try initializing the same new node again in case a concurrent requests tries to create a file with the same name n = node.New("e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "a6ede986-cfcd-41c5-a820-6eee955a1c2b", "e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "newchild", 10, "26493c53-2634-45f8-949f-dc07b88df9b0", providerv1beta1.ResourceType_RESOURCE_TYPE_FILE, &userv1beta1.UserId{}, lookup) n.SpaceRoot = rootNode - f2, err := store.initNewNode(context.Background(), store.New(context.Background()), n, 10) + unlock2, err := store.initNewNode(context.Background(), store.New(context.Background()), n, 10) if _, ok := err.(errtypes.IsAlreadyExists); !ok { t.Fatalf(`initNewNode(with same 'newchild' name), %v, want %v`, err, errtypes.AlreadyExists("newchild")) } - f2.Close() + unlock2() } diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index fe2861e556..cd2ccd5821 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -20,12 +20,9 @@ package upload import ( "context" - "crypto/md5" - "crypto/sha1" "encoding/hex" "fmt" "hash" - "hash/adler32" "io" "io/fs" "os" @@ -131,37 +128,13 @@ func (session *OcisSession) FinishUpload(ctx context.Context) error { ctx = ctxpkg.ContextSetInitiator(ctx, session.InitiatorID()) - // calculate the checksum of the written bytes - // they will all be written to the metadata later, so we cannot omit any of them - // TODO only calculate the checksum in sync that was requested to match, the rest could be async ... but the tests currently expect all to be present - // TODO the hashes all implement BinaryMarshaler so we could try to persist the state for resumable upload. we would neet do keep track of the copied bytes ... - sha1h := sha1.New() - md5h := md5.New() - adler32h := adler32.New() - { - _, subspan := tracer.Start(ctx, "os.Open") - f, err := os.Open(session.binPath()) - subspan.End() - if err != nil { - // we can continue if no oc checksum header is set - log.Info().Err(err).Str("binPath", session.binPath()).Msg("error opening binPath") - } - defer f.Close() - - r1 := io.TeeReader(f, sha1h) - r2 := io.TeeReader(r1, md5h) - - _, subspan = tracer.Start(ctx, "io.Copy") - _, err = io.Copy(adler32h, r2) - subspan.End() - if err != nil { - log.Info().Err(err).Msg("error copying checksums") - } + sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath()) + if err != nil { + log.Info().Err(err).Msg("error copying checksums") } // compare if they match the sent checksum // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads - var err error if session.info.MetaData["checksum"] != "" { var err error parts := strings.SplitN(session.info.MetaData["checksum"], " ", 2) diff --git a/pkg/storage/utils/templates/templates.go b/pkg/storage/utils/templates/templates.go index 0ad48074e5..bd9052773a 100644 --- a/pkg/storage/utils/templates/templates.go +++ b/pkg/storage/utils/templates/templates.go @@ -52,6 +52,7 @@ type ( *UserData SpaceType string SpaceName string + SpaceId string } // EmailData contains mail data @@ -89,9 +90,9 @@ func WithUser(u *userpb.User, tpl string) string { } // WithSpacePropertiesAndUser generates a layout based on user data and a space type. -func WithSpacePropertiesAndUser(u *userpb.User, spaceType string, spaceName string, tpl string) string { +func WithSpacePropertiesAndUser(u *userpb.User, spaceType string, spaceName string, spaceID string, tpl string) string { tpl = clean(tpl) - sd := newSpaceData(u, spaceType, spaceName) + sd := newSpaceData(u, spaceType, spaceName, spaceID) // compile given template tpl t, err := template.New("tpl").Funcs(sprig.TxtFuncMap()).Parse(tpl) if err != nil { @@ -147,12 +148,13 @@ func newUserData(u *userpb.User) *UserData { return ut } -func newSpaceData(u *userpb.User, st string, n string) *SpaceData { +func newSpaceData(u *userpb.User, st, n, id string) *SpaceData { userData := newUserData(u) sd := &SpaceData{ userData, st, n, + id, } return sd } From 590f245ef800d431fab04d408c6c38c930867a72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 18 Mar 2024 09:20:14 +0100 Subject: [PATCH 2/7] Improve logging --- pkg/storage/utils/decomposedfs/upload/store.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/upload/store.go b/pkg/storage/utils/decomposedfs/upload/store.go index f5841fcc5c..d741fdeb0b 100644 --- a/pkg/storage/utils/decomposedfs/upload/store.go +++ b/pkg/storage/utils/decomposedfs/upload/store.go @@ -217,20 +217,24 @@ func (store OcisStore) CreateNodeForUpload(session *OcisSession, initAttrs node. // FIXME look at the disk again to see if the file has been created in between, or just try initializing a new node and do the update existing node as a fallback. <- the latter! unlock, err = store.updateExistingNode(ctx, session, n, session.SpaceID(), uint64(session.Size())) - if unlock != nil { - appctx.GetLogger(ctx).Info().Interface("err", err).Msg("got lock file from updateExistingNode") + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Msg("failed to update existing node") } } else { if c, ok := store.lu.(node.IDCacher); ok { - c.CacheID(ctx, n.SpaceID, n.ID, filepath.Join(n.ParentPath(), n.Name)) + err := c.CacheID(ctx, n.SpaceID, n.ID, filepath.Join(n.ParentPath(), n.Name)) + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Msg("failed to cache id") + } } unlock, err = store.initNewNode(ctx, session, n, uint64(session.Size())) - if unlock != nil { - appctx.GetLogger(ctx).Info().Interface("err", err).Msg("got lock file from initNewNode") + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Msg("failed to init new node") } } defer func() { if unlock == nil { + appctx.GetLogger(ctx).Info().Msg("did not get a unlockfunc, not unlocking") return } From 7d34249edfbf8c80eab438deebf6ee62f1efd9ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 18 Mar 2024 10:01:30 +0100 Subject: [PATCH 3/7] Do not use a UserLayout by default --- pkg/storage/utils/decomposedfs/options/options.go | 8 -------- pkg/storage/utils/decomposedfs/testhelpers/helpers.go | 1 - 2 files changed, 9 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/options/options.go b/pkg/storage/utils/decomposedfs/options/options.go index 0afd9d309d..df731be573 100644 --- a/pkg/storage/utils/decomposedfs/options/options.go +++ b/pkg/storage/utils/decomposedfs/options/options.go @@ -122,14 +122,6 @@ func New(m map[string]interface{}) (*Options, error) { o.MetadataBackend = "xattrs" } - if o.UserLayout == "" { - o.UserLayout = "{{.Id.OpaqueId}}" - } - - if o.ProjectLayout == "" { - o.ProjectLayout = "{{.Id.OpaqueId}}" - } - // ensure user layout has no starting or trailing / o.UserLayout = strings.Trim(o.UserLayout, "/") diff --git a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go index 66f5726954..75de32f52d 100644 --- a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go +++ b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go @@ -96,7 +96,6 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { "treetime_accounting": true, "treesize_accounting": true, "share_folder": "/Shares", - "user_layout": "{{.Id.OpaqueId}}", } // make it possible to override single config values for k, v := range config { From b843f23aeb7577422cb196f08335f2e6819b9ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 18 Mar 2024 10:01:54 +0100 Subject: [PATCH 4/7] Fix creating new nodes --- pkg/storage/utils/decomposedfs/lookup/lookup.go | 15 --------------- pkg/storage/utils/decomposedfs/metadata/errors.go | 11 +++++++++++ pkg/storage/utils/decomposedfs/node/node.go | 8 +++----- pkg/storage/utils/decomposedfs/node/node_test.go | 2 +- .../utils/decomposedfs/options/options_test.go | 2 +- pkg/storage/utils/decomposedfs/upload/store.go | 4 ++-- .../utils/decomposedfs/upload/store_test.go | 6 ++++-- 7 files changed, 22 insertions(+), 26 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/lookup/lookup.go b/pkg/storage/utils/decomposedfs/lookup/lookup.go index c019f1c9e1..3337969bdd 100644 --- a/pkg/storage/utils/decomposedfs/lookup/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup/lookup.go @@ -21,11 +21,9 @@ package lookup import ( "context" "fmt" - "io/fs" "os" "path/filepath" "strings" - "syscall" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -99,22 +97,9 @@ func readChildNodeFromLink(path string) (string, error) { return nodeID, nil } -// The os error is buried inside the fs.PathError error -func isNotDir(err error) bool { - if perr, ok := err.(*fs.PathError); ok { - if serr, ok2 := perr.Err.(syscall.Errno); ok2 { - return serr == syscall.ENOTDIR - } - } - return false -} - func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) { nodeID, err := readChildNodeFromLink(filepath.Join(parent.InternalPath(), name)) if err != nil { - if errors.Is(err, fs.ErrNotExist) || isNotDir(err) { - return nodeID, nil // if the file does not exist we return a node that has Exists = false - } return "", errors.Wrap(err, "decomposedfs: Wrap: readlink error") } return nodeID, nil diff --git a/pkg/storage/utils/decomposedfs/metadata/errors.go b/pkg/storage/utils/decomposedfs/metadata/errors.go index f0d0ab14df..628910c4ed 100644 --- a/pkg/storage/utils/decomposedfs/metadata/errors.go +++ b/pkg/storage/utils/decomposedfs/metadata/errors.go @@ -19,6 +19,7 @@ package metadata import ( + "io/fs" "os" "syscall" @@ -50,3 +51,13 @@ func IsAttrUnset(err error) bool { } return false } + +// The os error is buried inside the fs.PathError error +func IsNotDir(err error) bool { + if perr, ok := errors.Cause(err).(*fs.PathError); ok { + if serr, ok2 := perr.Err.(syscall.Errno); ok2 { + return serr == syscall.ENOTDIR + } + } + return false +} diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index b63eb243f1..6fc1184cbd 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -393,12 +393,10 @@ func (n *Node) Child(ctx context.Context, name string) (*Node, error) { } nodeID, err := n.lu.NodeIDFromParentAndName(ctx, n, name) - switch err.(type) { - case nil: - // ok - case errtypes.IsNotFound: + switch { + case metadata.IsNotExist(err) || metadata.IsNotDir(err): return c, nil // if the file does not exist we return a node that has Exists = false - default: + case err != nil: return nil, err } diff --git a/pkg/storage/utils/decomposedfs/node/node_test.go b/pkg/storage/utils/decomposedfs/node/node_test.go index 06e2c7243e..136ced1ac6 100644 --- a/pkg/storage/utils/decomposedfs/node/node_test.go +++ b/pkg/storage/utils/decomposedfs/node/node_test.go @@ -161,7 +161,7 @@ var _ = Describe("Node", func() { Expect(child.Blobsize).To(Equal(int64(1234))) }) - It("handles (broken) links including file segments by returning an non-existent node", func() { + It("handles broken links including file segments by returning an non-existent node", func() { child, err := parent.Child(env.Ctx, "file1/broken") Expect(err).ToNot(HaveOccurred()) Expect(child).ToNot(BeNil()) diff --git a/pkg/storage/utils/decomposedfs/options/options_test.go b/pkg/storage/utils/decomposedfs/options/options_test.go index dca13b570b..11a83ebac7 100644 --- a/pkg/storage/utils/decomposedfs/options/options_test.go +++ b/pkg/storage/utils/decomposedfs/options/options_test.go @@ -43,7 +43,7 @@ var _ = Describe("Options", func() { }) It("sets defaults", func() { - Expect(len(o.UserLayout) > 0).To(BeTrue()) + Expect(o.MetadataBackend).ToNot(BeEmpty()) }) Context("with unclean root path configuration", func() { diff --git a/pkg/storage/utils/decomposedfs/upload/store.go b/pkg/storage/utils/decomposedfs/upload/store.go index d741fdeb0b..63d85e3500 100644 --- a/pkg/storage/utils/decomposedfs/upload/store.go +++ b/pkg/storage/utils/decomposedfs/upload/store.go @@ -63,7 +63,7 @@ type OcisStore struct { } // NewSessionStore returns a new OcisStore -func NewSessionStore(lu node.PathLookup, tp Tree, root string, pub events.Publisher, async bool, tknopts options.TokenOptions, DisableVersioning bool) *OcisStore { +func NewSessionStore(lu node.PathLookup, tp Tree, root string, pub events.Publisher, async bool, tknopts options.TokenOptions, disableVersioning bool) *OcisStore { return &OcisStore{ lu: lu, tp: tp, @@ -71,7 +71,7 @@ func NewSessionStore(lu node.PathLookup, tp Tree, root string, pub events.Publis pub: pub, async: async, tknopts: tknopts, - disableVersioning: DisableVersioning, + disableVersioning: disableVersioning, } } diff --git a/pkg/storage/utils/decomposedfs/upload/store_test.go b/pkg/storage/utils/decomposedfs/upload/store_test.go index f287f62bca..bdf2f89c4d 100644 --- a/pkg/storage/utils/decomposedfs/upload/store_test.go +++ b/pkg/storage/utils/decomposedfs/upload/store_test.go @@ -38,7 +38,9 @@ func TestInitNewNode(t *testing.T) { if err != nil { t.Fatalf(err.Error()) } - defer unlock() + defer func() { + _ = unlock() + }() // try initializing the same new node again in case a concurrent requests tries to create a file with the same name n = node.New("e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "a6ede986-cfcd-41c5-a820-6eee955a1c2b", "e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "newchild", 10, "26493c53-2634-45f8-949f-dc07b88df9b0", providerv1beta1.ResourceType_RESOURCE_TYPE_FILE, &userv1beta1.UserId{}, lookup) @@ -47,5 +49,5 @@ func TestInitNewNode(t *testing.T) { if _, ok := err.(errtypes.IsAlreadyExists); !ok { t.Fatalf(`initNewNode(with same 'newchild' name), %v, want %v`, err, errtypes.AlreadyExists("newchild")) } - unlock2() + _ = unlock2() } From 5892b45c735f10d0f82c1e7a47a5cf5436c42583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 18 Mar 2024 10:58:02 +0100 Subject: [PATCH 5/7] Fix uploads --- pkg/storage/fs/posix/tree/tree.go | 28 +++++++++ .../utils/decomposedfs/node/mocks/Tree.go | 61 +++++++++++++++++++ pkg/storage/utils/decomposedfs/node/node.go | 2 + pkg/storage/utils/decomposedfs/tree/tree.go | 44 +++++++++++++ .../utils/decomposedfs/upload/store.go | 40 +++--------- .../utils/decomposedfs/upload/store_test.go | 8 ++- .../utils/decomposedfs/upload/upload.go | 21 ------- 7 files changed, 147 insertions(+), 57 deletions(-) diff --git a/pkg/storage/fs/posix/tree/tree.go b/pkg/storage/fs/posix/tree/tree.go index d9f693259b..88d24bc9a2 100644 --- a/pkg/storage/fs/posix/tree/tree.go +++ b/pkg/storage/fs/posix/tree/tree.go @@ -34,6 +34,7 @@ import ( "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" @@ -697,6 +698,33 @@ func (t *Tree) ResolveSpaceIDIndexEntry(spaceid, entry string) (string, string, return spaceid, entry, nil } +// InitNewNode initializes a new node +func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (metadata.UnlockFunc, error) { + // create folder structure (if needed) + if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil { + return nil, err + } + + // create and write lock new node metadata + unlock, err := t.lookup.MetadataBackend().Lock(n.InternalPath()) + if err != nil { + return nil, err + } + + // we also need to touch the actual node file here it stores the mtime of the resource + h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return unlock, err + } + h.Close() + + if _, err := node.CheckQuota(ctx, n.SpaceRoot, false, 0, fsize); err != nil { + return unlock, err + } + + return unlock, nil +} + // TODO check if node exists? func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) { ctx, span := tracer.Start(ctx, "createDirNode") diff --git a/pkg/storage/utils/decomposedfs/node/mocks/Tree.go b/pkg/storage/utils/decomposedfs/node/mocks/Tree.go index 21f488e349..a8e72bdb62 100644 --- a/pkg/storage/utils/decomposedfs/node/mocks/Tree.go +++ b/pkg/storage/utils/decomposedfs/node/mocks/Tree.go @@ -26,6 +26,7 @@ import ( io "io" + metadata "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" mock "github.com/stretchr/testify/mock" node "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" @@ -290,6 +291,66 @@ func (_c *Tree_GetMD_Call) RunAndReturn(run func(context.Context, *node.Node) (f return _c } +// InitNewNode provides a mock function with given fields: ctx, n, fsize +func (_m *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (metadata.UnlockFunc, error) { + ret := _m.Called(ctx, n, fsize) + + if len(ret) == 0 { + panic("no return value specified for InitNewNode") + } + + var r0 metadata.UnlockFunc + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *node.Node, uint64) (metadata.UnlockFunc, error)); ok { + return rf(ctx, n, fsize) + } + if rf, ok := ret.Get(0).(func(context.Context, *node.Node, uint64) metadata.UnlockFunc); ok { + r0 = rf(ctx, n, fsize) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(metadata.UnlockFunc) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *node.Node, uint64) error); ok { + r1 = rf(ctx, n, fsize) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Tree_InitNewNode_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'InitNewNode' +type Tree_InitNewNode_Call struct { + *mock.Call +} + +// InitNewNode is a helper method to define mock.On call +// - ctx context.Context +// - n *node.Node +// - fsize uint64 +func (_e *Tree_Expecter) InitNewNode(ctx interface{}, n interface{}, fsize interface{}) *Tree_InitNewNode_Call { + return &Tree_InitNewNode_Call{Call: _e.mock.On("InitNewNode", ctx, n, fsize)} +} + +func (_c *Tree_InitNewNode_Call) Run(run func(ctx context.Context, n *node.Node, fsize uint64)) *Tree_InitNewNode_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*node.Node), args[2].(uint64)) + }) + return _c +} + +func (_c *Tree_InitNewNode_Call) Return(_a0 metadata.UnlockFunc, _a1 error) *Tree_InitNewNode_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Tree_InitNewNode_Call) RunAndReturn(run func(context.Context, *node.Node, uint64) (metadata.UnlockFunc, error)) *Tree_InitNewNode_Call { + _c.Call.Return(run) + return _c +} + // ListFolder provides a mock function with given fields: ctx, _a1 func (_m *Tree) ListFolder(ctx context.Context, _a1 *node.Node) ([]*node.Node, error) { ret := _m.Called(ctx, _a1) diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 6fc1184cbd..f7437d1723 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -99,6 +99,8 @@ type Tree interface { RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPath string, target *Node) (*Node, *Node, func() error, error) PurgeRecycleItemFunc(ctx context.Context, spaceid, key, purgePath string) (*Node, func() error, error) + InitNewNode(ctx context.Context, n *Node, fsize uint64) (metadata.UnlockFunc, error) + WriteBlob(node *Node, source string) error ReadBlob(node *Node) (io.ReadCloser, error) DeleteBlob(node *Node) error diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index e5b0f96e13..dbb44a1edd 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -34,6 +34,7 @@ import ( "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" @@ -661,6 +662,49 @@ func (t *Tree) PurgeRecycleItemFunc(ctx context.Context, spaceid, key string, pa return rn, fn, nil } +// InitNewNode initializes a new node +func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (metadata.UnlockFunc, error) { + // create folder structure (if needed) + if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil { + return nil, err + } + + // create and write lock new node metadata + unlock, err := t.lookup.MetadataBackend().Lock(n.InternalPath()) + if err != nil { + return nil, err + } + + // we also need to touch the actual node file here it stores the mtime of the resource + h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return unlock, err + } + h.Close() + + if _, err := node.CheckQuota(ctx, n.SpaceRoot, false, 0, fsize); err != nil { + return unlock, err + } + + // link child name to parent if it is new + childNameLink := filepath.Join(n.ParentPath(), n.Name) + relativeNodePath := filepath.Join("../../../../../", lookup.Pathify(n.ID, 4, 2)) + log := appctx.GetLogger(ctx).With().Str("childNameLink", childNameLink).Str("relativeNodePath", relativeNodePath).Logger() + log.Info().Msg("initNewNode: creating symlink") + + if err = os.Symlink(relativeNodePath, childNameLink); err != nil { + log.Info().Err(err).Msg("initNewNode: symlink failed") + if errors.Is(err, fs.ErrExist) { + log.Info().Err(err).Msg("initNewNode: symlink already exists") + return unlock, errtypes.AlreadyExists(n.Name) + } + return unlock, errors.Wrap(err, "Decomposedfs: could not symlink child entry") + } + log.Info().Msg("initNewNode: symlink created") + + return unlock, nil +} + func (t *Tree) removeNode(ctx context.Context, path, timeSuffix string, n *node.Node) error { logger := appctx.GetLogger(ctx) diff --git a/pkg/storage/utils/decomposedfs/upload/store.go b/pkg/storage/utils/decomposedfs/upload/store.go index 63d85e3500..b5b1aa9fdb 100644 --- a/pkg/storage/utils/decomposedfs/upload/store.go +++ b/pkg/storage/utils/decomposedfs/upload/store.go @@ -54,7 +54,7 @@ type PermissionsChecker interface { // OcisStore manages upload sessions type OcisStore struct { lu node.PathLookup - tp Tree + tp node.Tree root string pub events.Publisher async bool @@ -63,7 +63,7 @@ type OcisStore struct { } // NewSessionStore returns a new OcisStore -func NewSessionStore(lu node.PathLookup, tp Tree, root string, pub events.Publisher, async bool, tknopts options.TokenOptions, disableVersioning bool) *OcisStore { +func NewSessionStore(lu node.PathLookup, tp node.Tree, root string, pub events.Publisher, async bool, tknopts options.TokenOptions, disableVersioning bool) *OcisStore { return &OcisStore{ lu: lu, tp: tp, @@ -227,10 +227,12 @@ func (store OcisStore) CreateNodeForUpload(session *OcisSession, initAttrs node. appctx.GetLogger(ctx).Error().Err(err).Msg("failed to cache id") } } - unlock, err = store.initNewNode(ctx, session, n, uint64(session.Size())) + + unlock, err = store.tp.InitNewNode(ctx, n, uint64(session.Size())) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Msg("failed to init new node") } + session.info.MetaData["sizeDiff"] = strconv.FormatInt(session.Size(), 10) } defer func() { if unlock == nil { @@ -275,34 +277,6 @@ func (store OcisStore) CreateNodeForUpload(session *OcisSession, initAttrs node. return n, nil } -func (store OcisStore) initNewNode(ctx context.Context, session *OcisSession, n *node.Node, fsize uint64) (metadata.UnlockFunc, error) { - // create folder structure (if needed) - if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil { - return nil, err - } - - // create and write lock new node metadata - unlock, err := store.lu.MetadataBackend().Lock(n.InternalPath()) - if err != nil { - return nil, err - } - - // we also need to touch the actual node file here it stores the mtime of the resource - h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - return unlock, err - } - h.Close() - - if _, err := node.CheckQuota(ctx, n.SpaceRoot, false, 0, fsize); err != nil { - return unlock, err - } - - // on a new file the sizeDiff is the fileSize - session.info.MetaData["sizeDiff"] = strconv.FormatInt(int64(fsize), 10) - return unlock, nil -} - func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSession, n *node.Node, spaceID string, fsize uint64) (metadata.UnlockFunc, error) { targetPath := n.InternalPath() @@ -371,12 +345,12 @@ func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSess versionPath = session.store.lu.InternalPath(spaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano)) // create version node - if _, err := os.Create(session.info.MetaData["versionsPath"]); err != nil { + if _, err := os.Create(versionPath); err != nil { return unlock, err } // copy blob metadata to version node - if err := store.lu.CopyMetadataWithSourceLock(ctx, targetPath, session.info.MetaData["versionsPath"], func(attributeName string, value []byte) (newValue []byte, copy bool) { + if err := store.lu.CopyMetadataWithSourceLock(ctx, targetPath, versionPath, func(attributeName string, value []byte) (newValue []byte, copy bool) { return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || attributeName == prefixes.TypeAttr || attributeName == prefixes.BlobIDAttr || diff --git a/pkg/storage/utils/decomposedfs/upload/store_test.go b/pkg/storage/utils/decomposedfs/upload/store_test.go index bdf2f89c4d..58dce7ce9d 100644 --- a/pkg/storage/utils/decomposedfs/upload/store_test.go +++ b/pkg/storage/utils/decomposedfs/upload/store_test.go @@ -13,6 +13,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" ) // TestInitNewNode calls greetings.initNewNode @@ -21,8 +22,9 @@ func TestInitNewNode(t *testing.T) { root := t.TempDir() lookup := lookup.New(metadata.NewMessagePackBackend(root, cache.Config{}), &options.Options{Root: root}) + tp := tree.New(lookup, nil, &options.Options{}, nil) - store := NewSessionStore(lookup, nil, root, nil, false, options.TokenOptions{}, false) + store := NewSessionStore(lookup, tp, root, nil, false, options.TokenOptions{}, false) rootNode := node.New("e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "", "", 0, "", providerv1beta1.ResourceType_RESOURCE_TYPE_CONTAINER, &userv1beta1.UserId{}, lookup) rootNode.Exists = true @@ -34,7 +36,7 @@ func TestInitNewNode(t *testing.T) { } n := node.New("e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "930b7a2e-b745-41e1-8a9b-712582021842", "e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "newchild", 10, "26493c53-2634-45f8-949f-dc07b88df9b0", providerv1beta1.ResourceType_RESOURCE_TYPE_FILE, &userv1beta1.UserId{}, lookup) n.SpaceRoot = rootNode - unlock, err := store.initNewNode(context.Background(), store.New(context.Background()), n, 10) + unlock, err := store.tp.InitNewNode(context.Background(), n, 10) if err != nil { t.Fatalf(err.Error()) } @@ -45,7 +47,7 @@ func TestInitNewNode(t *testing.T) { // try initializing the same new node again in case a concurrent requests tries to create a file with the same name n = node.New("e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "a6ede986-cfcd-41c5-a820-6eee955a1c2b", "e48c4e7a-beac-4b82-b991-a5cff7b8c39c", "newchild", 10, "26493c53-2634-45f8-949f-dc07b88df9b0", providerv1beta1.ResourceType_RESOURCE_TYPE_FILE, &userv1beta1.UserId{}, lookup) n.SpaceRoot = rootNode - unlock2, err := store.initNewNode(context.Background(), store.New(context.Background()), n, 10) + unlock2, err := store.tp.InitNewNode(context.Background(), n, 10) if _, ok := err.(errtypes.IsAlreadyExists); !ok { t.Fatalf(`initNewNode(with same 'newchild' name), %v, want %v`, err, errtypes.AlreadyExists("newchild")) } diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index cd2ccd5821..9d93b112a9 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -50,27 +50,6 @@ func init() { tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/upload") } -// Tree is used to manage a tree hierarchy -type Tree interface { - Setup() error - - GetMD(ctx context.Context, node *node.Node) (os.FileInfo, error) - ListFolder(ctx context.Context, node *node.Node) ([]*node.Node, error) - // CreateHome(owner *userpb.UserId) (n *node.Node, err error) - CreateDir(ctx context.Context, node *node.Node) (err error) - // CreateReference(ctx context.Context, node *node.Node, targetURI *url.URL) error - Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) (err error) - Delete(ctx context.Context, node *node.Node) (err error) - RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPath string, target *node.Node) (*node.Node, *node.Node, func() error, error) - PurgeRecycleItemFunc(ctx context.Context, spaceid, key, purgePath string) (*node.Node, func() error, error) - - WriteBlob(node *node.Node, binPath string) error - ReadBlob(node *node.Node) (io.ReadCloser, error) - DeleteBlob(node *node.Node) error - - Propagate(ctx context.Context, node *node.Node, sizeDiff int64) (err error) -} - var defaultFilePerm = os.FileMode(0664) // WriteChunk writes the stream from the reader to the given offset of the upload From b2470187abc91ed8f3ac20f2751a22b37d8c80d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 22 Apr 2024 11:28:15 +0200 Subject: [PATCH 6/7] Add changelog --- changelog/unreleased/extensible-decomposedfs.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/extensible-decomposedfs.md diff --git a/changelog/unreleased/extensible-decomposedfs.md b/changelog/unreleased/extensible-decomposedfs.md new file mode 100644 index 0000000000..efb2f36c80 --- /dev/null +++ b/changelog/unreleased/extensible-decomposedfs.md @@ -0,0 +1,5 @@ +Enhancement: Make decomposedfs more extensible + +We refactored decomposedfs to make it more extensible, e.g. for the posixfs storage driver. + +https://github.com/cs3org/reva/pull/4581 From 034a8cf4a5c9a3186a8bba7366934e619be8390a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Tue, 23 Apr 2024 08:31:25 +0200 Subject: [PATCH 7/7] Introduce separate variables for the space path templates --- pkg/storage/utils/decomposedfs/options/options.go | 2 ++ pkg/storage/utils/decomposedfs/spaces.go | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/options/options.go b/pkg/storage/utils/decomposedfs/options/options.go index df731be573..e68bd73d06 100644 --- a/pkg/storage/utils/decomposedfs/options/options.go +++ b/pkg/storage/utils/decomposedfs/options/options.go @@ -71,7 +71,9 @@ type Options struct { PermTLSMode pool.TLSMode PersonalSpaceAliasTemplate string `mapstructure:"personalspacealias_template"` + PersonalSpacePathTemplate string `mapstructure:"personalspacepath_template"` GeneralSpaceAliasTemplate string `mapstructure:"generalspacealias_template"` + GeneralSpacePathTemplate string `mapstructure:"generalspacepath_template"` AsyncFileUploads bool `mapstructure:"asyncfileuploads"` diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 1f80676017..77702ec57d 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -100,12 +100,12 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr rootPath := root.InternalPath() switch req.Type { case _spaceTypePersonal: - if fs.o.UserLayout != "" { - rootPath = filepath.Join(fs.o.Root, templates.WithUser(u, fs.o.UserLayout)) + if fs.o.PersonalSpacePathTemplate != "" { + rootPath = filepath.Join(fs.o.Root, templates.WithUser(u, fs.o.PersonalSpacePathTemplate)) } - case _spaceTypeProject: - if fs.o.ProjectLayout != "" { - rootPath = filepath.Join(fs.o.Root, templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, spaceID, fs.o.ProjectLayout)) + default: + if fs.o.GeneralSpacePathTemplate != "" { + rootPath = filepath.Join(fs.o.Root, templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, spaceID, fs.o.GeneralSpacePathTemplate)) } }