diff --git a/changelog/unreleased/improve-posixfs.md b/changelog/unreleased/improve-posixfs.md new file mode 100644 index 0000000000..1e788f1732 --- /dev/null +++ b/changelog/unreleased/improve-posixfs.md @@ -0,0 +1,5 @@ +Enhancement: Add trashbin support to posixfs alongside other improvements + +We added support for trashbins to posixfs. Posixfs also saw a number of other improvement, bugfixes and optimizations. + +https://github.com/cs3org/reva/pull/4779 diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 4a86ba41eb..278ce61dfc 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -178,7 +178,7 @@ func (s *svc) Handler() http.Handler { var head string head, r.URL.Path = router.ShiftPath(r.URL.Path) - log.Debug().Str("head", head).Str("tail", r.URL.Path).Msg("http routing") + log.Debug().Str("method", r.Method).Str("head", head).Str("tail", r.URL.Path).Msg("http routing") switch head { case "status.php", "status": s.doStatus(w, r) diff --git a/pkg/storage/fs/posix/blobstore/blobstore.go b/pkg/storage/fs/posix/blobstore/blobstore.go index bdbeb4536d..cf01a9e138 100644 --- a/pkg/storage/fs/posix/blobstore/blobstore.go +++ b/pkg/storage/fs/posix/blobstore/blobstore.go @@ -41,6 +41,11 @@ func New(root string) (*Blobstore, error) { // Upload stores some data in the blobstore under the given key func (bs *Blobstore) Upload(node *node.Node, source string) error { + path := node.InternalPath() + + // preserve the mtime of the file + fi, _ := os.Stat(path) + file, err := os.Open(source) if err != nil { return errors.Wrap(err, "Decomposedfs: oCIS blobstore: Can not open source file to upload") @@ -59,7 +64,15 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error { return errors.Wrapf(err, "could not write blob '%s'", node.InternalPath()) } - return w.Flush() + err = w.Flush() + if err != nil { + return err + } + + if fi != nil { + return os.Chtimes(path, fi.ModTime(), fi.ModTime()) + } + return nil } // Download retrieves a blob from the blobstore for reading diff --git a/pkg/storage/fs/posix/lookup/lookup.go b/pkg/storage/fs/posix/lookup/lookup.go index 824466887a..d188831cb6 100644 --- a/pkg/storage/fs/posix/lookup/lookup.go +++ b/pkg/storage/fs/posix/lookup/lookup.go @@ -72,15 +72,17 @@ type Lookup struct { IDCache IDCache metadataBackend metadata.Backend userMapper usermapper.Mapper + tm node.TimeManager } // New returns a new Lookup instance -func New(b metadata.Backend, um usermapper.Mapper, o *options.Options) *Lookup { +func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.TimeManager) *Lookup { lu := &Lookup{ Options: o, metadataBackend: b, IDCache: NewStoreIDCache(&o.Options), userMapper: um, + tm: tm, } return lu @@ -122,22 +124,12 @@ func (lu *Lookup) MetadataBackend() metadata.Backend { return lu.metadataBackend } -// ReadBlobSizeAttr reads the blobsize from the xattrs -func (lu *Lookup) ReadBlobSizeAttr(ctx context.Context, path string) (int64, error) { - blobSize, err := lu.metadataBackend.GetInt64(ctx, path, prefixes.BlobsizeAttr) +func (lu *Lookup) ReadBlobIDAndSizeAttr(ctx context.Context, path string, _ node.Attributes) (string, int64, error) { + fi, err := os.Stat(path) if err != nil { - return 0, errors.Wrapf(err, "error reading blobsize xattr") + return "", 0, errors.Wrap(err, "error stating file") } - return blobSize, nil -} - -// ReadBlobIDAttr reads the blobsize from the xattrs -func (lu *Lookup) ReadBlobIDAttr(ctx context.Context, path string) (string, error) { - attr, err := lu.metadataBackend.Get(ctx, path, prefixes.BlobIDAttr) - if err != nil { - return "", errors.Wrapf(err, "error reading blobid xattr") - } - return string(attr), nil + return "", fi.Size(), nil } // TypeFromPath returns the type of the node at the given path @@ -421,3 +413,8 @@ func (lu *Lookup) GenerateSpaceID(spaceType string, owner *user.User) (string, e return "", fmt.Errorf("unsupported space type: %s", spaceType) } } + +// TimeManager returns the time manager +func (lu *Lookup) TimeManager() node.TimeManager { + return lu.tm +} diff --git a/pkg/storage/fs/posix/options/options.go b/pkg/storage/fs/posix/options/options.go index 314e019b8d..c323258d38 100644 --- a/pkg/storage/fs/posix/options/options.go +++ b/pkg/storage/fs/posix/options/options.go @@ -19,6 +19,8 @@ package options import ( + "time" + decomposedoptions "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -29,6 +31,8 @@ type Options struct { UseSpaceGroups bool `mapstructure:"use_space_groups"` + ScanDebounceDelay time.Duration `mapstructure:"scan_debounce_delay"` + WatchFS bool `mapstructure:"watch_fs"` WatchType string `mapstructure:"watch_type"` WatchPath string `mapstructure:"watch_path"` diff --git a/pkg/storage/fs/posix/posix.go b/pkg/storage/fs/posix/posix.go index dfffdf04b9..537689d459 100644 --- a/pkg/storage/fs/posix/posix.go +++ b/pkg/storage/fs/posix/posix.go @@ -36,6 +36,8 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/fs/posix/blobstore" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/options" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/trashbin" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree" "github.com/cs3org/reva/v2/pkg/storage/fs/registry" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs" @@ -68,24 +70,39 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) { return nil, err } - bs, err := blobstore.New(o.Root) - if err != nil { - return nil, err - } - + fs := &posixFS{} um := usermapper.NewUnixMapper() var lu *lookup.Lookup switch o.MetadataBackend { case "xattrs": - lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o) + lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{}) case "messagepack": - lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o) + lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{}) default: return nil, fmt.Errorf("unknown metadata backend %s, only 'messagepack' or 'xattrs' (default) supported", o.MetadataBackend) } - tp, err := tree.New(lu, bs, um, o, stream, store.Create( + trashbin, err := trashbin.New(o, lu) + if err != nil { + return nil, err + } + err = trashbin.Setup(fs) + if err != nil { + return nil, err + } + + bs, err := blobstore.New(o.Root) + if err != nil { + return nil, err + } + + switch o.IDCache.Store { + case "", "memory", "noop": + return nil, fmt.Errorf("the posix driver requires a shared id cache, e.g. nats-js-kv or redis") + } + + tp, err := tree.New(lu, bs, um, trashbin, o, stream, store.Create( store.Store(o.IDCache.Store), store.TTL(o.IDCache.TTL), store.Size(o.IDCache.Size), @@ -113,6 +130,7 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) { EventStream: stream, UserMapper: um, DisableVersioning: true, + Trashbin: trashbin, } dfs, err := decomposedfs.New(&o.Options, aspects) @@ -154,10 +172,8 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) { } mw := middleware.NewFS(dfs, hooks...) - fs := &posixFS{ - FS: mw, - um: um, - } + fs.FS = mw + fs.um = um return fs, nil } diff --git a/pkg/storage/fs/posix/posix_test.go b/pkg/storage/fs/posix/posix_test.go index 49b670f7df..67f138ba68 100644 --- a/pkg/storage/fs/posix/posix_test.go +++ b/pkg/storage/fs/posix/posix_test.go @@ -42,6 +42,9 @@ var _ = Describe("Posix", func() { "root": tmpRoot, "share_folder": "/Shares", "permissionssvc": "any", + "idcache": map[string]interface{}{ + "cache_store": "nats-js-kv", + }, } }) diff --git a/pkg/storage/fs/posix/testhelpers/helpers.go b/pkg/storage/fs/posix/testhelpers/helpers.go index 60d3a5761d..1ed248ecc0 100644 --- a/pkg/storage/fs/posix/testhelpers/helpers.go +++ b/pkg/storage/fs/posix/testhelpers/helpers.go @@ -36,6 +36,8 @@ import ( "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/options" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/trashbin" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/aspects" @@ -152,12 +154,13 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { }, }, } + var lu *lookup.Lookup switch o.MetadataBackend { case "xattrs": - lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o) + lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{}) case "messagepack": - lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o) + lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{}) default: return nil, fmt.Errorf("unknown metadata backend %s", o.MetadataBackend) } @@ -175,7 +178,11 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { ) bs := &treemocks.Blobstore{} - tree, err := tree.New(lu, bs, um, o, nil, store.Create()) + tree, err := tree.New(lu, bs, um, &trashbin.Trashbin{}, o, nil, store.Create()) + if err != nil { + return nil, err + } + tb, err := trashbin.New(o, lu) if err != nil { return nil, err } @@ -183,6 +190,7 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { Lookup: lu, Tree: tree, Permissions: permissions.NewPermissions(pmock, permissionsSelector), + Trashbin: tb, } fs, err := decomposedfs.New(&o.Options, aspects) if err != nil { diff --git a/pkg/storage/fs/posix/timemanager/timemanager.go b/pkg/storage/fs/posix/timemanager/timemanager.go new file mode 100644 index 0000000000..ee5407d9f9 --- /dev/null +++ b/pkg/storage/fs/posix/timemanager/timemanager.go @@ -0,0 +1,117 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package timemanager + +import ( + "context" + "os" + "syscall" + "time" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" +) + +// Manager is responsible for managing time-related operations on files and directories. +type Manager struct { +} + +// OverrideMtime overrides the modification time (mtime) of a node with the specified time. +func (m *Manager) OverrideMtime(ctx context.Context, n *node.Node, _ *node.Attributes, mtime time.Time) error { + return os.Chtimes(n.InternalPath(), mtime, mtime) +} + +// MTime returns the modification time (mtime) of a node. +func (m *Manager) MTime(ctx context.Context, n *node.Node) (time.Time, error) { + fi, err := os.Stat(n.InternalPath()) + if err != nil { + return time.Time{}, err + } + return fi.ModTime(), nil +} + +// SetMTime sets the modification time (mtime) of a node to the specified time. +func (m *Manager) SetMTime(ctx context.Context, n *node.Node, mtime *time.Time) error { + return os.Chtimes(n.InternalPath(), *mtime, *mtime) +} + +// TMTime returns the tree modification time (tmtime) of a node. +// If the tmtime is not set, it falls back to the modification time (mtime). +func (m *Manager) TMTime(ctx context.Context, n *node.Node) (time.Time, error) { + b, err := n.XattrString(ctx, prefixes.TreeMTimeAttr) + if err == nil { + return time.Parse(time.RFC3339Nano, b) + } + + // no tmtime, use mtime + return m.MTime(ctx, n) +} + +// SetTMTime sets the tree modification time (tmtime) of a node to the specified time. +// If tmtime is nil, the tmtime attribute is removed. +func (m *Manager) SetTMTime(ctx context.Context, n *node.Node, tmtime *time.Time) error { + if tmtime == nil { + return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true) + } + return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, tmtime.UTC().Format(time.RFC3339Nano)) +} + +// CTime returns the creation time (ctime) of a node. +func (m *Manager) CTime(ctx context.Context, n *node.Node) (time.Time, error) { + fi, err := os.Stat(n.InternalPath()) + if err != nil { + return time.Time{}, err + } + + stat := fi.Sys().(*syscall.Stat_t) + //nolint:unconvert + return time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)), nil +} + +// TCTime returns the tree creation time (tctime) of a node. +// Since decomposedfs does not differentiate between ctime and mtime, it falls back to TMTime. +func (m *Manager) TCTime(ctx context.Context, n *node.Node) (time.Time, error) { + // decomposedfs does not differentiate between ctime and mtime + return m.TMTime(ctx, n) +} + +// SetTCTime sets the tree creation time (tctime) of a node to the specified time. +// Since decomposedfs does not differentiate between ctime and mtime, it falls back to SetTMTime. +func (m *Manager) SetTCTime(ctx context.Context, n *node.Node, tmtime *time.Time) error { + // decomposedfs does not differentiate between ctime and mtime + return m.SetTMTime(ctx, n, tmtime) +} + +// DTime returns the deletion time (dtime) of a node. +func (m *Manager) DTime(ctx context.Context, n *node.Node) (tmTime time.Time, err error) { + b, err := n.XattrString(ctx, prefixes.DTimeAttr) + if err != nil { + return time.Time{}, err + } + return time.Parse(time.RFC3339Nano, b) +} + +// SetDTime sets the deletion time (dtime) of a node to the specified time. +// If t is nil, the dtime attribute is removed. +func (m *Manager) SetDTime(ctx context.Context, n *node.Node, t *time.Time) (err error) { + if t == nil { + return n.RemoveXattr(ctx, prefixes.DTimeAttr, true) + } + return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano)) +} diff --git a/pkg/storage/fs/posix/trashbin/trashbin.go b/pkg/storage/fs/posix/trashbin/trashbin.go new file mode 100644 index 0000000000..f0fed08e16 --- /dev/null +++ b/pkg/storage/fs/posix/trashbin/trashbin.go @@ -0,0 +1,307 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package trashbin + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/v2/pkg/storage" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/options" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" + "github.com/cs3org/reva/v2/pkg/utils" + "github.com/google/uuid" +) + +type Trashbin struct { + fs storage.FS + o *options.Options + lu *lookup.Lookup +} + +const ( + trashHeader = `[Trash Info]` + timeFormat = "2006-01-02T15:04:05" +) + +// New returns a new Trashbin +func New(o *options.Options, lu *lookup.Lookup) (*Trashbin, error) { + return &Trashbin{ + o: o, + lu: lu, + }, nil +} + +func (tb *Trashbin) writeInfoFile(trashPath, id, path string) error { + c := trashHeader + c += "\nPath=" + path + c += "\nDeletionDate=" + time.Now().Format(timeFormat) + + return os.WriteFile(filepath.Join(trashPath, "info", id+".trashinfo"), []byte(c), 0644) +} + +func (tb *Trashbin) readInfoFile(trashPath, id string) (string, *typesv1beta1.Timestamp, error) { + c, err := os.ReadFile(filepath.Join(trashPath, "info", id+".trashinfo")) + if err != nil { + return "", nil, err + } + + var ( + path string + ts *typesv1beta1.Timestamp + ) + + for _, line := range strings.Split(string(c), "\n") { + if strings.HasPrefix(line, "DeletionDate=") { + t, err := time.ParseInLocation(timeFormat, strings.TrimSpace(strings.TrimPrefix(line, "DeletionDate=")), time.Local) + if err != nil { + return "", nil, err + } + ts = utils.TimeToTS(t) + } + if strings.HasPrefix(line, "Path=") { + path = strings.TrimPrefix(line, "Path=") + } + } + + return path, ts, nil +} + +// Setup the trashbin +func (tb *Trashbin) Setup(fs storage.FS) error { + if tb.fs != nil { + return nil + } + + tb.fs = fs + return nil +} + +func trashRootForNode(n *node.Node) string { + return filepath.Join(n.SpaceRoot.InternalPath(), ".Trash") +} + +func (tb *Trashbin) MoveToTrash(ctx context.Context, n *node.Node, path string) error { + key := uuid.New().String() + trashPath := trashRootForNode(n) + + err := os.MkdirAll(filepath.Join(trashPath, "info"), 0755) + if err != nil { + return err + } + err = os.MkdirAll(filepath.Join(trashPath, "files"), 0755) + if err != nil { + return err + } + + relPath := strings.TrimPrefix(path, n.SpaceRoot.InternalPath()) + relPath = strings.TrimPrefix(relPath, "/") + err = tb.writeInfoFile(trashPath, key, relPath) + if err != nil { + return err + } + + // purge metadata + if err = tb.lu.IDCache.DeleteByPath(ctx, path); err != nil { + return err + } + + itemTrashPath := filepath.Join(trashPath, "files", key+".trashitem") + err = tb.lu.MetadataBackend().Rename(path, itemTrashPath) + if err != nil { + return err + } + + return os.Rename(path, itemTrashPath) +} + +// ListRecycle returns the list of available recycle items +// ref -> the space (= resourceid), key -> deleted node id, relativePath = relative to key +func (tb *Trashbin) ListRecycle(ctx context.Context, ref *provider.Reference, key, relativePath string) ([]*provider.RecycleItem, error) { + n, err := tb.lu.NodeFromResource(ctx, ref) + if err != nil { + return nil, err + } + + trashRoot := trashRootForNode(n) + base := filepath.Join(trashRoot, "files") + + var originalPath string + var ts *typesv1beta1.Timestamp + if key != "" { + // this is listing a specific item/folder + base = filepath.Join(base, key+".trashitem", relativePath) + originalPath, ts, err = tb.readInfoFile(trashRoot, key) + originalPath = filepath.Join(originalPath, relativePath) + if err != nil { + return nil, err + } + } + + items := []*provider.RecycleItem{} + entries, err := os.ReadDir(filepath.Clean(base)) + if err != nil { + switch err.(type) { + case *os.PathError: + return items, nil + default: + return nil, err + } + } + + for _, entry := range entries { + var fi os.FileInfo + var entryOriginalPath string + var entryKey string + if strings.HasSuffix(entry.Name(), ".trashitem") { + entryKey = strings.TrimSuffix(entry.Name(), ".trashitem") + entryOriginalPath, ts, err = tb.readInfoFile(trashRoot, entryKey) + if err != nil { + continue + } + + fi, err = entry.Info() + if err != nil { + continue + } + } else { + fi, err = os.Stat(filepath.Join(base, entry.Name())) + entryKey = entry.Name() + entryOriginalPath = filepath.Join(originalPath, entry.Name()) + if err != nil { + continue + } + } + + item := &provider.RecycleItem{ + Key: filepath.Join(key, relativePath, entryKey), + Size: uint64(fi.Size()), + Ref: &provider.Reference{ + ResourceId: &provider.ResourceId{ + SpaceId: ref.GetResourceId().GetSpaceId(), + OpaqueId: ref.GetResourceId().GetSpaceId(), + }, + Path: entryOriginalPath, + }, + DeletionTime: ts, + } + if entry.IsDir() { + item.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER + } else { + item.Type = provider.ResourceType_RESOURCE_TYPE_FILE + } + + items = append(items, item) + } + + return items, nil +} + +// RestoreRecycleItem restores the specified item +func (tb *Trashbin) RestoreRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string, restoreRef *provider.Reference) error { + n, err := tb.lu.NodeFromResource(ctx, ref) + if err != nil { + return err + } + + trashRoot := trashRootForNode(n) + trashPath := filepath.Clean(filepath.Join(trashRoot, "files", key+".trashitem", relativePath)) + + restoreBaseNode, err := tb.lu.NodeFromID(ctx, restoreRef.GetResourceId()) + if err != nil { + return err + } + restorePath := filepath.Join(restoreBaseNode.InternalPath(), restoreRef.GetPath()) + + id, err := tb.lu.MetadataBackend().Get(ctx, trashPath, prefixes.IDAttr) + if err != nil { + return err + } + + // update parent id in case it was restored to a different location + parentID, err := tb.lu.MetadataBackend().Get(ctx, filepath.Dir(restorePath), prefixes.IDAttr) + if err != nil { + return err + } + if len(parentID) == 0 { + return fmt.Errorf("trashbin: parent id not found for %s", restorePath) + } + + err = tb.lu.MetadataBackend().Set(ctx, trashPath, prefixes.ParentidAttr, parentID) + if err != nil { + return err + } + + // restore the item + err = os.Rename(trashPath, restorePath) + if err != nil { + return err + } + _ = tb.lu.CacheID(ctx, n.SpaceID, string(id), restorePath) + + // cleanup trash info + if relativePath == "." || relativePath == "/" { + return os.Remove(filepath.Join(trashRoot, "info", key+".trashinfo")) + } else { + return nil + } +} + +// PurgeRecycleItem purges the specified item, all its children and all their revisions +func (tb *Trashbin) PurgeRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string) error { + n, err := tb.lu.NodeFromResource(ctx, ref) + if err != nil { + return err + } + + trashRoot := trashRootForNode(n) + err = os.RemoveAll(filepath.Clean(filepath.Join(trashRoot, "files", key+".trashitem", relativePath))) + if err != nil { + return err + } + + cleanPath := filepath.Clean(relativePath) + if cleanPath == "." || cleanPath == "/" { + return os.Remove(filepath.Join(trashRoot, "info", key+".trashinfo")) + } + return nil +} + +// EmptyRecycle empties the trash +func (tb *Trashbin) EmptyRecycle(ctx context.Context, ref *provider.Reference) error { + n, err := tb.lu.NodeFromResource(ctx, ref) + if err != nil { + return err + } + + trashRoot := trashRootForNode(n) + err = os.RemoveAll(filepath.Clean(filepath.Join(trashRoot, "files"))) + if err != nil { + return err + } + return os.RemoveAll(filepath.Clean(filepath.Join(trashRoot, "info"))) +} diff --git a/pkg/storage/fs/posix/tree/assimilation.go b/pkg/storage/fs/posix/tree/assimilation.go index a350175807..496c841cb8 100644 --- a/pkg/storage/fs/posix/tree/assimilation.go +++ b/pkg/storage/fs/posix/tree/assimilation.go @@ -78,6 +78,11 @@ func NewScanDebouncer(d time.Duration, f func(item scanItem)) *ScanDebouncer { // Debounce restarts the debounce timer for the given space func (d *ScanDebouncer) Debounce(item scanItem) { + if d.after == 0 { + d.f(item) + return + } + d.mutex.Lock() defer d.mutex.Unlock() @@ -213,7 +218,7 @@ func (t *Tree) Scan(path string, action EventAction, isDir bool, recurse bool) e func (t *Tree) HandleFileDelete(path string) error { // purge metadata _ = t.lookup.(*lookup.Lookup).IDCache.DeleteByPath(context.Background(), path) - _ = t.lookup.MetadataBackend().Purge(path) + _ = t.lookup.MetadataBackend().Purge(context.Background(), path) // send event owner, spaceID, nodeID, parentID, err := t.getOwnerAndIDs(filepath.Dir(path)) @@ -333,56 +338,75 @@ func (t *Tree) assimilate(item scanItem) error { id, err = t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.IDAttr) if err == nil { previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), spaceID, string(id)) - - // This item had already been assimilated in the past. Update the path - _ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), spaceID, string(id), item.Path) - previousParentID, _ := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr) - fi, err := t.updateFile(item.Path, string(id), spaceID) - if err != nil { - return err - } - - // was it moved? + // was it moved or copied/restored with a clashing id? if ok && len(previousParentID) > 0 && previousPath != item.Path { - // purge original metadata. Only delete the path entry using DeletePath(reverse lookup), not the whole entry pair. - _ = t.lookup.(*lookup.Lookup).IDCache.DeletePath(context.Background(), previousPath) - _ = t.lookup.MetadataBackend().Purge(previousPath) + _, err := os.Stat(previousPath) + if err == nil { + // this id clashes with an existing id -> clear metadata and re-assimilate - if fi.IsDir() { - // if it was moved and it is a directory we need to propagate the move - go func() { _ = t.WarmupIDCache(item.Path, false) }() - } + _ = t.lookup.MetadataBackend().Purge(context.Background(), item.Path) + go func() { + _ = t.assimilate(scanItem{Path: item.Path, ForceRescan: true}) + }() + } else { + // this is a move + _ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), spaceID, string(id), item.Path) + _, err := t.updateFile(item.Path, string(id), spaceID) + if err != nil { + return err + } - parentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr) - if err == nil && len(parentID) > 0 { - ref := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: t.options.MountID, - SpaceId: spaceID, - OpaqueId: string(parentID), - }, - Path: filepath.Base(item.Path), + // purge original metadata. Only delete the path entry using DeletePath(reverse lookup), not the whole entry pair. + _ = t.lookup.(*lookup.Lookup).IDCache.DeletePath(context.Background(), previousPath) + _ = t.lookup.MetadataBackend().Purge(context.Background(), previousPath) + + fi, err := os.Stat(item.Path) + if err != nil { + return err } - oldRef := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: t.options.MountID, - SpaceId: spaceID, - OpaqueId: string(previousParentID), - }, - Path: filepath.Base(previousPath), + if fi.IsDir() { + // if it was moved and it is a directory we need to propagate the move + go func() { _ = t.WarmupIDCache(item.Path, false) }() + } + + parentID, err := t.lookup.MetadataBackend().Get(context.Background(), item.Path, prefixes.ParentidAttr) + if err == nil && len(parentID) > 0 { + ref := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: t.options.MountID, + SpaceId: spaceID, + OpaqueId: string(parentID), + }, + Path: filepath.Base(item.Path), + } + oldRef := &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: t.options.MountID, + SpaceId: spaceID, + OpaqueId: string(previousParentID), + }, + Path: filepath.Base(previousPath), + } + t.PublishEvent(events.ItemMoved{ + SpaceOwner: user, + Executant: user, + Owner: user, + Ref: ref, + OldReference: oldRef, + Timestamp: utils.TSNow(), + }) } - t.PublishEvent(events.ItemMoved{ - SpaceOwner: user, - Executant: user, - Owner: user, - Ref: ref, - OldReference: oldRef, - Timestamp: utils.TSNow(), - }) } - // } + } else { + // This item had already been assimilated in the past. Update the path + _ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), spaceID, string(id), item.Path) + + _, err := t.updateFile(item.Path, string(id), spaceID) + if err != nil { + return err + } } } else { // assimilate new file @@ -472,10 +496,6 @@ assimilate: prefixes.IDAttr: []byte(id), prefixes.NameAttr: []byte(filepath.Base(path)), } - prevMtime, err := previousAttribs.Time(prefixes.MTimeAttr) - if err != nil || prevMtime.Before(fi.ModTime()) { - attributes[prefixes.MTimeAttr] = []byte(fi.ModTime().Format(time.RFC3339Nano)) - } if len(parentID) > 0 { attributes[prefixes.ParentidAttr] = []byte(parentID) } @@ -496,25 +516,15 @@ assimilate: attributes[prefixes.PropagationAttr] = []byte("1") } else { attributes.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE)) - attributes.SetString(prefixes.BlobIDAttr, id) - attributes.SetInt64(prefixes.BlobsizeAttr, fi.Size()) - - // propagate the change - sizeDiff := fi.Size() - if previousAttribs != nil && previousAttribs[prefixes.BlobsizeAttr] != nil { - oldSize, err := attributes.Int64(prefixes.BlobsizeAttr) - if err == nil { - sizeDiff -= oldSize - } - } + } - n := node.New(spaceID, id, parentID, filepath.Base(path), fi.Size(), "", provider.ResourceType_RESOURCE_TYPE_FILE, nil, t.lookup) - n.SpaceRoot = &node.Node{SpaceID: spaceID, ID: spaceID} - err = t.Propagate(context.Background(), n, sizeDiff) - if err != nil { - return nil, errors.Wrap(err, "failed to propagate") - } + n := node.New(spaceID, id, parentID, filepath.Base(path), fi.Size(), "", provider.ResourceType_RESOURCE_TYPE_FILE, nil, t.lookup) + n.SpaceRoot = &node.Node{SpaceID: spaceID, ID: spaceID} + err = t.Propagate(context.Background(), n, 0) + if err != nil { + return nil, errors.Wrap(err, "failed to propagate") } + err = t.lookup.MetadataBackend().SetMultiple(context.Background(), path, attributes, false) if err != nil { return nil, errors.Wrap(err, "failed to set attributes") @@ -547,15 +557,15 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error { sizes := make(map[string]int64) err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - // skip lock files if isLockFile(path) { return nil } + if err != nil { + return err + } + // calculate tree sizes if !info.IsDir() { dir := filepath.Dir(path) @@ -593,6 +603,16 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error { id, ok := attribs[prefixes.IDAttr] if ok { + // Check if the item on the previous still exists. In this case it might have been a copy with extended attributes -> set new ID + previousPath, ok := t.lookup.(*lookup.Lookup).GetCachedID(context.Background(), string(spaceID), string(id)) + if ok && previousPath != path { + // this id clashes with an existing id -> clear metadata and re-assimilate + _, err := os.Stat(previousPath) + if err == nil { + _ = t.lookup.MetadataBackend().Purge(context.Background(), path) + _ = t.assimilate(scanItem{Path: path, ForceRescan: true}) + } + } _ = t.lookup.(*lookup.Lookup).CacheID(context.Background(), string(spaceID), string(id), path) } } else if assimilate { @@ -600,12 +620,14 @@ func (t *Tree) WarmupIDCache(root string, assimilate bool) error { } return nil }) - if err != nil { - return err - } for dir, size := range sizes { _ = t.lookup.MetadataBackend().Set(context.Background(), dir, prefixes.TreesizeAttr, []byte(fmt.Sprintf("%d", size))) } + + if err != nil { + return err + } + return nil } diff --git a/pkg/storage/fs/posix/tree/gpfsfilauditloggingwatcher.go b/pkg/storage/fs/posix/tree/gpfsfilauditloggingwatcher.go index a61c172802..12d6fd5890 100644 --- a/pkg/storage/fs/posix/tree/gpfsfilauditloggingwatcher.go +++ b/pkg/storage/fs/posix/tree/gpfsfilauditloggingwatcher.go @@ -59,6 +59,9 @@ start: if err != nil { continue } + if isLockFile(ev.Path) || isTrash(ev.Path) { + continue + } switch ev.Event { case "CREATE": go func() { _ = w.tree.Scan(ev.Path, ActionCreate, false, false) }() diff --git a/pkg/storage/fs/posix/tree/gpfswatchfolderwatcher.go b/pkg/storage/fs/posix/tree/gpfswatchfolderwatcher.go index 6d1e295269..19446158ef 100644 --- a/pkg/storage/fs/posix/tree/gpfswatchfolderwatcher.go +++ b/pkg/storage/fs/posix/tree/gpfswatchfolderwatcher.go @@ -41,7 +41,7 @@ func (w *GpfsWatchFolderWatcher) Watch(topic string) { continue } - if isLockFile(lwev.Path) { + if isLockFile(lwev.Path) || isTrash(lwev.Path) { continue } diff --git a/pkg/storage/fs/posix/tree/inotifywatcher.go b/pkg/storage/fs/posix/tree/inotifywatcher.go index c50ddc1d46..aa729fe615 100644 --- a/pkg/storage/fs/posix/tree/inotifywatcher.go +++ b/pkg/storage/fs/posix/tree/inotifywatcher.go @@ -42,7 +42,7 @@ func (iw *InotifyWatcher) Watch(path string) { select { case event := <-events: for _, e := range event.Events { - if isLockFile(event.Filename) { + if isLockFile(event.Filename) || isTrash(event.Filename) { continue } switch e { diff --git a/pkg/storage/fs/posix/tree/tree.go b/pkg/storage/fs/posix/tree/tree.go index bacceefe1c..3aeadb477e 100644 --- a/pkg/storage/fs/posix/tree/tree.go +++ b/pkg/storage/fs/posix/tree/tree.go @@ -19,7 +19,6 @@ package tree import ( - "bytes" "context" "fmt" "io" @@ -28,7 +27,6 @@ import ( "path/filepath" "regexp" "strings" - "time" "github.com/google/uuid" "github.com/pkg/errors" @@ -46,6 +44,7 @@ import ( "github.com/cs3org/reva/v2/pkg/logger" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup" "github.com/cs3org/reva/v2/pkg/storage/fs/posix/options" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/trashbin" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" @@ -82,6 +81,7 @@ type scanItem struct { type Tree struct { lookup node.PathLookup blobstore Blobstore + trashbin *trashbin.Trashbin propagator propagator.Propagator options *options.Options @@ -100,18 +100,19 @@ type Tree struct { type PermissionCheckFunc func(rp *provider.ResourcePermissions) bool // New returns a new instance of Tree -func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, o *options.Options, es events.Stream, cache store.Store) (*Tree, error) { +func New(lu node.PathLookup, bs Blobstore, um usermapper.Mapper, trashbin *trashbin.Trashbin, o *options.Options, es events.Stream, cache store.Store) (*Tree, error) { log := logger.New() scanQueue := make(chan scanItem) t := &Tree{ lookup: lu, blobstore: bs, userMapper: um, + trashbin: trashbin, options: o, idCache: cache, propagator: propagator.New(lu, &o.Options), scanQueue: scanQueue, - scanDebouncer: NewScanDebouncer(1000*time.Millisecond, func(item scanItem) { + scanDebouncer: NewScanDebouncer(o.ScanDebounceDelay, func(item scanItem) { scanQueue <- item }), es: es, @@ -229,14 +230,17 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool, if markprocessing { attributes[prefixes.StatusPrefix] = []byte(node.ProcessingStatus) } - nodeMTime := time.Now() if mtime != "" { - nodeMTime, err = utils.MTimeToTime(mtime) + nodeMTime, err := utils.MTimeToTime(mtime) + if err != nil { + return err + } + err = os.Chtimes(nodePath, nodeMTime, nodeMTime) if err != nil { return err } } - attributes[prefixes.MTimeAttr] = []byte(nodeMTime.UTC().Format(time.RFC3339Nano)) + err = n.SetXattrsWithContext(ctx, attributes, false) if err != nil { return err @@ -297,18 +301,6 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) return errors.Wrap(err, "Decomposedfs: could not update old node attributes") } - // the size diff is the current treesize or blobsize of the old/source node - var sizeDiff int64 - if oldNode.IsDir(ctx) { - treeSize, err := oldNode.GetTreeSize(ctx) - if err != nil { - return err - } - sizeDiff = int64(treeSize) - } else { - sizeDiff = oldNode.Blobsize - } - // rename node err = os.Rename( filepath.Join(oldNode.ParentPath(), oldNode.Name), @@ -334,7 +326,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) newNode.ID = oldNode.ID } _ = t.lookup.(*lookup.Lookup).CacheID(ctx, newNode.SpaceID, newNode.ID, filepath.Join(newNode.ParentPath(), newNode.Name)) - // update id cache for the moved subtree + // update id cache for the moved subtree. if oldNode.IsDir(ctx) { err = t.WarmupIDCache(filepath.Join(newNode.ParentPath(), newNode.Name), false) if err != nil { @@ -342,15 +334,11 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) } } - // TODO inefficient because we might update several nodes twice, only propagate unchanged nodes? - // collect in a list, then only stat each node once - // also do this in a go routine ... webdav should check the etag async - - err = t.Propagate(ctx, oldNode, -sizeDiff) + err = t.Propagate(ctx, oldNode, 0) if err != nil { return errors.Wrap(err, "Decomposedfs: Move: could not propagate old node") } - err = t.Propagate(ctx, newNode, sizeDiff) + err = t.Propagate(ctx, newNode, 0) if err != nil { return errors.Wrap(err, "Decomposedfs: Move: could not propagate new node") } @@ -394,7 +382,7 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro g.Go(func() error { defer close(work) for _, name := range names { - if isLockFile(name) { + if isLockFile(name) || isTrash(name) { continue } @@ -469,7 +457,7 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro } // Delete deletes a node in the tree by moving it to the trash -func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { +func (t *Tree) Delete(ctx context.Context, n *node.Node) error { path := n.InternalPath() if !strings.HasPrefix(path, t.options.Root) { @@ -500,28 +488,11 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { // Remove lock file if it exists _ = os.Remove(n.LockFilePath()) - // purge metadata - err = filepath.WalkDir(path, func(path string, _ fs.DirEntry, err error) error { - if err != nil { - return err - } - - if err = t.lookup.(*lookup.Lookup).IDCache.DeleteByPath(ctx, path); err != nil { - return err - } - if err = t.lookup.MetadataBackend().Purge(path); err != nil { - return err - } - return nil - }) + err := t.trashbin.MoveToTrash(ctx, n, path) if err != nil { return err } - if err = os.RemoveAll(path); err != nil { - return - } - return t.Propagate(ctx, n, sizeDiff) } @@ -659,7 +630,7 @@ func (t *Tree) removeNode(ctx context.Context, path string, n *node.Node) error return err } - if err := t.lookup.MetadataBackend().Purge(path); err != nil { + if err := t.lookup.MetadataBackend().Purge(ctx, path); err != nil { log.Error().Err(err).Str("path", t.lookup.MetadataBackend().MetadataPath(path)).Msg("error purging node metadata") return err } @@ -673,42 +644,15 @@ func (t *Tree) removeNode(ctx context.Context, path string, n *node.Node) error } // delete revisions - revs, err := filepath.Glob(n.InternalPath() + node.RevisionIDDelimiter + "*") - if err != nil { - log.Error().Err(err).Str("path", n.InternalPath()+node.RevisionIDDelimiter+"*").Msg("glob failed badly") - return err - } - for _, rev := range revs { - if t.lookup.MetadataBackend().IsMetaFile(rev) { - continue - } - - bID, err := t.lookup.ReadBlobIDAttr(ctx, rev) - if err != nil { - log.Error().Err(err).Str("revision", rev).Msg("error reading blobid attribute") - return err - } - - if err := utils.RemoveItem(rev); err != nil { - log.Error().Err(err).Str("revision", rev).Msg("error removing revision node") - return err - } - - if bID != "" { - if err := t.DeleteBlob(&node.Node{SpaceID: n.SpaceID, BlobID: bID}); err != nil { - log.Error().Err(err).Str("revision", rev).Str("blobID", bID).Msg("error removing revision node blob") - return err - } - } - - } + // posixfs doesn't do revisions yet return nil } // Propagate propagates changes to the root of the tree -func (t *Tree) Propagate(ctx context.Context, n *node.Node, sizeDiff int64) (err error) { - return t.propagator.Propagate(ctx, n, sizeDiff) +func (t *Tree) Propagate(ctx context.Context, n *node.Node, _ int64) (err error) { + // We do not propagate size diffs here but rely on the assimilation to take care of the tree sizes instead + return t.propagator.Propagate(ctx, n, 0) } // WriteBlob writes a blob to the blobstore @@ -718,10 +662,6 @@ func (t *Tree) WriteBlob(node *node.Node, source string) error { // ReadBlob reads a blob from the blobstore func (t *Tree) ReadBlob(node *node.Node) (io.ReadCloser, error) { - if node.BlobID == "" { - // there is no blob yet - we are dealing with a 0 byte file - return io.NopCloser(bytes.NewReader([]byte{})), nil - } return t.blobstore.Download(node) } @@ -730,10 +670,6 @@ func (t *Tree) DeleteBlob(node *node.Node) error { if node == nil { return fmt.Errorf("could not delete blob, nil node was given") } - if node.BlobID == "" { - return fmt.Errorf("could not delete blob, node with empty blob id was given") - } - return t.blobstore.Delete(node) } @@ -886,3 +822,7 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( func isLockFile(path string) bool { return strings.HasSuffix(path, ".lock") || strings.HasSuffix(path, ".flock") || strings.HasSuffix(path, ".mlock") } + +func isTrash(path string) bool { + return strings.HasSuffix(path, ".trashinfo") || strings.HasSuffix(path, ".trashitem") +} diff --git a/pkg/storage/fs/posix/tree/tree_test.go b/pkg/storage/fs/posix/tree/tree_test.go index 1f7c31086c..b660fb9a22 100644 --- a/pkg/storage/fs/posix/tree/tree_test.go +++ b/pkg/storage/fs/posix/tree/tree_test.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "log" "os" + "os/exec" "strings" "time" @@ -222,6 +223,45 @@ var _ = Describe("Tree", func() { g.Expect(n.Blobsize).To(Equal(int64(0))) }).Should(Succeed()) }) + + It("handles id clashes", func() { + // Create empty file + _, err := os.Create(root + "/original.txt") + Expect(err).ToNot(HaveOccurred()) + + fileID := "" + // Wait for the file to be indexed + Eventually(func(g Gomega) { + n, err := env.Lookup.NodeFromResource(env.Ctx, &provider.Reference{ + ResourceId: env.SpaceRootRes, + Path: subtree + "/original.txt", + }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(n).ToNot(BeNil()) + g.Expect(n.Type(env.Ctx)).To(Equal(provider.ResourceType_RESOURCE_TYPE_FILE)) + g.Expect(n.ID).ToNot(BeEmpty()) + fileID = n.ID + g.Expect(n.Blobsize).To(Equal(int64(0))) + }).Should(Succeed()) + + // cp file + cmd := exec.Command("cp", "-a", root+"/original.txt", root+"/moved.txt") + err = cmd.Run() + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + n, err := env.Lookup.NodeFromResource(env.Ctx, &provider.Reference{ + ResourceId: env.SpaceRootRes, + Path: subtree + "/moved.txt", + }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(n).ToNot(BeNil()) + g.Expect(n.Type(env.Ctx)).To(Equal(provider.ResourceType_RESOURCE_TYPE_FILE)) + g.Expect(n.ID).ToNot(BeEmpty()) + g.Expect(n.ID).ToNot(Equal(fileID)) + g.Expect(n.Blobsize).To(Equal(int64(0))) + }).Should(Succeed()) + }) }) Describe("of directories", func() { diff --git a/pkg/storage/utils/decomposedfs/aspects/aspects.go b/pkg/storage/utils/decomposedfs/aspects/aspects.go index 3aeb542c72..65a9e0d983 100644 --- a/pkg/storage/utils/decomposedfs/aspects/aspects.go +++ b/pkg/storage/utils/decomposedfs/aspects/aspects.go @@ -22,6 +22,7 @@ import ( "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/permissions" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/trashbin" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/usermapper" ) @@ -29,6 +30,7 @@ import ( type Aspects struct { Lookup node.PathLookup Tree node.Tree + Trashbin trashbin.Trashbin Permissions permissions.Permissions EventStream events.Stream DisableVersioning bool diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 2bb6b1076c..41dd5705ce 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -49,6 +49,8 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/permissions" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/timemanager" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/trashbin" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/usermapper" @@ -110,6 +112,7 @@ type SessionStore interface { type Decomposedfs struct { lu node.PathLookup tp node.Tree + trashbin trashbin.Trashbin o *options.Options p permissions.Permissions um usermapper.Mapper @@ -133,9 +136,9 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) ( var lu *lookup.Lookup switch o.MetadataBackend { case "xattrs": - lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), o) + lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), o, &timemanager.Manager{}) case "messagepack": - lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), o) + lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), o, &timemanager.Manager{}) default: return nil, fmt.Errorf("unknown metadata backend %s, only 'messagepack' or 'xattrs' (default) supported", o.MetadataBackend) } @@ -162,6 +165,7 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) ( Permissions: permissions.NewPermissions(node.NewPermissions(lu), permissionsSelector), EventStream: es, DisableVersioning: o.DisableVersioning, + Trashbin: &DecomposedfsTrashbin{}, } return New(o, aspects) @@ -209,6 +213,9 @@ func New(o *options.Options, aspects aspects.Aspects) (storage.FS, error) { return nil, err } + if aspects.Trashbin == nil { + return nil, errors.New("need trashbin") + } // set a null usermapper if we don't have one if aspects.UserMapper == nil { aspects.UserMapper = &usermapper.NullMapper{} @@ -217,6 +224,7 @@ func New(o *options.Options, aspects aspects.Aspects) (storage.FS, error) { fs := &Decomposedfs{ tp: aspects.Tree, lu: aspects.Lookup, + trashbin: aspects.Trashbin, o: o, p: aspects.Permissions, um: aspects.UserMapper, @@ -228,6 +236,9 @@ func New(o *options.Options, aspects aspects.Aspects) (storage.FS, error) { spaceTypeIndex: spaceTypeIndex, } fs.sessionStore = upload.NewSessionStore(fs, aspects, o.Root, o.AsyncFileUploads, o.Tokens) + if err = fs.trashbin.Setup(fs); err != nil { + return nil, err + } if o.AsyncFileUploads { if fs.stream == nil { @@ -1201,3 +1212,16 @@ func (fs *Decomposedfs) Unlock(ctx context.Context, ref *provider.Reference, loc return node.Unlock(ctx, lock) } + +func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference, key, relativePath string) ([]*provider.RecycleItem, error) { + return fs.trashbin.ListRecycle(ctx, ref, key, relativePath) +} +func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string, restoreRef *provider.Reference) error { + return fs.trashbin.RestoreRecycleItem(ctx, ref, key, relativePath, restoreRef) +} +func (fs *Decomposedfs) PurgeRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string) error { + return fs.trashbin.PurgeRecycleItem(ctx, ref, key, relativePath) +} +func (fs *Decomposedfs) EmptyRecycle(ctx context.Context, ref *provider.Reference) error { + return fs.trashbin.EmptyRecycle(ctx, ref) +} diff --git a/pkg/storage/utils/decomposedfs/lookup/lookup.go b/pkg/storage/utils/decomposedfs/lookup/lookup.go index 3337969bdd..fc95764aaf 100644 --- a/pkg/storage/utils/decomposedfs/lookup/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup/lookup.go @@ -55,13 +55,15 @@ type Lookup struct { Options *options.Options metadataBackend metadata.Backend + tm node.TimeManager } // New returns a new Lookup instance -func New(b metadata.Backend, o *options.Options) *Lookup { +func New(b metadata.Backend, o *options.Options, tm node.TimeManager) *Lookup { return &Lookup{ Options: o, metadataBackend: b, + tm: tm, } } @@ -70,23 +72,34 @@ func (lu *Lookup) MetadataBackend() metadata.Backend { return lu.metadataBackend } -// ReadBlobSizeAttr reads the blobsize from the xattrs -func (lu *Lookup) ReadBlobSizeAttr(ctx context.Context, path string) (int64, error) { - blobSize, err := lu.metadataBackend.GetInt64(ctx, path, prefixes.BlobsizeAttr) - if err != nil { - return 0, errors.Wrapf(err, "error reading blobsize xattr") - } - return blobSize, nil -} +func (lu *Lookup) ReadBlobIDAndSizeAttr(ctx context.Context, path string, attrs node.Attributes) (string, int64, error) { + blobID := "" + blobSize := int64(0) + var err error -// ReadBlobIDAttr reads the blobsize from the xattrs -func (lu *Lookup) ReadBlobIDAttr(ctx context.Context, path string) (string, error) { - attr, err := lu.metadataBackend.Get(ctx, path, prefixes.BlobIDAttr) - if err != nil { - return "", errors.Wrapf(err, "error reading blobid xattr") + if attrs != nil { + blobID = attrs.String(prefixes.BlobIDAttr) + if blobID != "" { + blobSize, err = attrs.Int64(prefixes.BlobsizeAttr) + if err != nil { + return "", 0, err + } + } + } else { + attrs, err := lu.metadataBackend.All(ctx, path) + if err != nil { + return "", 0, errors.Wrapf(err, "error reading blobid xattr") + } + nodeAttrs := node.Attributes(attrs) + blobID = nodeAttrs.String(prefixes.BlobIDAttr) + blobSize, err = nodeAttrs.Int64(prefixes.BlobsizeAttr) + if err != nil { + return "", 0, errors.Wrapf(err, "error reading blobsize xattr") + } } - return string(attr), nil + return blobID, blobSize, nil } + func readChildNodeFromLink(path string) (string, error) { link, err := os.Readlink(path) if err != nil { @@ -369,6 +382,11 @@ func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, sourcePath, ta return lu.MetadataBackend().SetMultiple(ctx, targetPath, newAttrs, acquireTargetLock) } +// TimeManager returns the time manager +func (lu *Lookup) TimeManager() node.TimeManager { + return lu.tm +} + // DetectBackendOnDisk returns the name of the metadata backend being used on disk func DetectBackendOnDisk(root string) string { matches, _ := filepath.Glob(filepath.Join(root, "spaces", "*", "*")) diff --git a/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go b/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go index 5c19821b48..90c5327a71 100644 --- a/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go @@ -273,7 +273,7 @@ func (MessagePackBackend) IsMetaFile(path string) bool { } // Purge purges the data of a given path -func (b MessagePackBackend) Purge(path string) error { +func (b MessagePackBackend) Purge(_ context.Context, path string) error { if err := b.metaCache.RemoveMetadata(b.cacheKey(path)); err != nil { return err } diff --git a/pkg/storage/utils/decomposedfs/metadata/metadata.go b/pkg/storage/utils/decomposedfs/metadata/metadata.go index 1b376c2dc2..3c95586b22 100644 --- a/pkg/storage/utils/decomposedfs/metadata/metadata.go +++ b/pkg/storage/utils/decomposedfs/metadata/metadata.go @@ -51,7 +51,7 @@ type Backend interface { Remove(ctx context.Context, path, key string, acquireLock bool) error Lock(path string) (UnlockFunc, error) - Purge(path string) error + Purge(ctx context.Context, path string) error Rename(oldPath, newPath string) error IsMetaFile(path string) bool MetadataPath(path string) string @@ -111,7 +111,7 @@ func (NullBackend) Lock(path string) (UnlockFunc, error) { func (NullBackend) IsMetaFile(path string) bool { return false } // Purge purges the data of a given path from any cache that might hold it -func (NullBackend) Purge(purges string) error { return errUnconfiguredError } +func (NullBackend) Purge(_ context.Context, purges string) error { return errUnconfiguredError } // Rename moves the data for a given path to a new path func (NullBackend) Rename(oldPath, newPath string) error { return errUnconfiguredError } diff --git a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go index 82d870c576..2e2a953eee 100644 --- a/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go +++ b/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/cs3org/reva/v2/pkg/storage/cache" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/filelocks" "github.com/pkg/errors" "github.com/pkg/xattr" @@ -213,7 +214,24 @@ func (b XattrsBackend) Remove(ctx context.Context, path string, key string, acqu func (XattrsBackend) IsMetaFile(path string) bool { return strings.HasSuffix(path, ".meta.lock") } // Purge purges the data of a given path -func (b XattrsBackend) Purge(path string) error { +func (b XattrsBackend) Purge(ctx context.Context, path string) error { + _, err := os.Stat(path) + if err == nil { + attribs, err := b.getAll(ctx, path, true) + if err != nil { + return err + } + + for attr := range attribs { + if strings.HasPrefix(attr, prefixes.OcisPrefix) { + err := xattr.Remove(path, attr) + if err != nil { + return err + } + } + } + } + return b.metaCache.RemoveMetadata(b.cacheKey(path)) } diff --git a/pkg/storage/utils/decomposedfs/migrator/migrator_test.go b/pkg/storage/utils/decomposedfs/migrator/migrator_test.go index b072713ff2..bed82a6fc1 100644 --- a/pkg/storage/utils/decomposedfs/migrator/migrator_test.go +++ b/pkg/storage/utils/decomposedfs/migrator/migrator_test.go @@ -11,6 +11,7 @@ import ( "github.com/rs/zerolog" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" @@ -89,7 +90,7 @@ var _ = Describe("Migrator", func() { path = nRef.InternalPath() // Change backend to messagepack - env.Lookup = lookup.New(backend, env.Options) + env.Lookup = lookup.New(backend, env.Options, &timemanager.Manager{}) }) It("migrates", func() { diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 42a507e303..795ce65c0e 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -23,6 +23,7 @@ import ( "crypto/md5" "crypto/sha1" "encoding/hex" + "encoding/json" "fmt" "hash" "hash/adler32" @@ -84,6 +85,29 @@ const ( ProcessingStatus = "processing:" ) +type TimeManager interface { + // OverrideMTime overrides the mtime of the node, either on the node itself or in the given attributes, depending on the implementation + OverrideMtime(ctx context.Context, n *Node, attrs *Attributes, mtime time.Time) error + + // MTime returns the mtime of the node + MTime(ctx context.Context, n *Node) (time.Time, error) + // SetMTime sets the mtime of the node + SetMTime(ctx context.Context, n *Node, mtime *time.Time) error + + // TMTime returns the tmtime of the node + TMTime(ctx context.Context, n *Node) (time.Time, error) + // SetTMTime sets the tmtime of the node + SetTMTime(ctx context.Context, n *Node, tmtime *time.Time) error + + // CTime returns the ctime of the node + CTime(ctx context.Context, n *Node) (time.Time, error) + + // DTime returns the deletion time of the node + DTime(ctx context.Context, n *Node) (time.Time, error) + // SetDTime sets the deletion time of the node + SetDTime(ctx context.Context, n *Node, mtime *time.Time) error +} + // Tree is used to manage a tree hierarchy type Tree interface { Setup() error @@ -125,8 +149,8 @@ type PathLookup interface { InternalPath(spaceID, nodeID string) string Path(ctx context.Context, n *Node, hasPermission PermissionFunc) (path string, err error) MetadataBackend() metadata.Backend - ReadBlobSizeAttr(ctx context.Context, path string) (int64, error) - ReadBlobIDAttr(ctx context.Context, path string) (string, error) + TimeManager() TimeManager + ReadBlobIDAndSizeAttr(ctx context.Context, path string, attrs Attributes) (string, int64, error) TypeFromPath(ctx context.Context, path string) provider.ResourceType CopyMetadataWithSourceLock(ctx context.Context, sourcePath, targetPath string, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error) CopyMetadata(ctx context.Context, src, target string, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error) @@ -172,6 +196,26 @@ func New(spaceID, id, parentID, name string, blobsize int64, blobID string, t pr } } +func (n *Node) MarshalJSON() ([]byte, error) { + return json.Marshal(&struct { + Name string `json:"name"` + ID string `json:"id"` + SpaceID string `json:"spaceID"` + ParentID string `json:"parentID"` + BlobID string `json:"blobID"` + BlobSize int64 `json:"blobSize"` + Exists bool `json:"exists"` + }{ + Name: n.Name, + ID: n.ID, + SpaceID: n.SpaceID, + ParentID: n.ParentID, + BlobID: n.BlobID, + BlobSize: n.Blobsize, + Exists: n.Exists, + }) +} + // Type returns the node's resource type func (n *Node) Type(ctx context.Context) provider.ResourceType { if n.nodeType != nil { @@ -351,22 +395,12 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canLis } if revisionSuffix == "" { - n.BlobID = attrs.String(prefixes.BlobIDAttr) - if n.BlobID != "" { - blobSize, err := attrs.Int64(prefixes.BlobsizeAttr) - if err != nil { - return nil, err - } - n.Blobsize = blobSize - } - } else { - n.BlobID, err = lu.ReadBlobIDAttr(ctx, nodePath+revisionSuffix) + n.BlobID, n.Blobsize, err = lu.ReadBlobIDAndSizeAttr(ctx, nodePath, attrs) if err != nil { return nil, err } - - // Lookup blobsize - n.Blobsize, err = lu.ReadBlobSizeAttr(ctx, nodePath+revisionSuffix) + } else { + n.BlobID, n.Blobsize, err = lu.ReadBlobIDAndSizeAttr(ctx, nodePath+revisionSuffix, nil) if err != nil { return nil, err } @@ -899,55 +933,6 @@ func (n *Node) HasPropagation(ctx context.Context) (propagation bool) { return false } -// GetTMTime reads the tmtime from the extended attributes, falling back to GetMTime() -func (n *Node) GetTMTime(ctx context.Context) (time.Time, error) { - b, err := n.XattrString(ctx, prefixes.TreeMTimeAttr) - if err == nil { - return time.Parse(time.RFC3339Nano, b) - } - - // no tmtime, use mtime - return n.GetMTime(ctx) -} - -// GetMTime reads the mtime from the extended attributes, falling back to disk -func (n *Node) GetMTime(ctx context.Context) (time.Time, error) { - b, err := n.XattrString(ctx, prefixes.MTimeAttr) - if err != nil { - fi, err := os.Lstat(n.InternalPath()) - if err != nil { - return time.Time{}, err - } - return fi.ModTime(), nil - } - return time.Parse(time.RFC3339Nano, b) -} - -// SetTMTime writes the UTC tmtime to the extended attributes or removes the attribute if nil is passed -func (n *Node) SetTMTime(ctx context.Context, t *time.Time) (err error) { - if t == nil { - return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true) - } - return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, t.UTC().Format(time.RFC3339Nano)) -} - -// GetDTime reads the dtime from the extended attributes -func (n *Node) GetDTime(ctx context.Context) (tmTime time.Time, err error) { - b, err := n.XattrString(ctx, prefixes.DTimeAttr) - if err != nil { - return time.Time{}, err - } - return time.Parse(time.RFC3339Nano, b) -} - -// SetDTime writes the UTC dtime to the extended attributes or removes the attribute if nil is passed -func (n *Node) SetDTime(ctx context.Context, t *time.Time) (err error) { - if t == nil { - return n.RemoveXattr(ctx, prefixes.DTimeAttr, true) - } - return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano)) -} - // IsDisabled returns true when the node has a dmtime attribute set // only used to check if a space is disabled // FIXME confusing with the trash logic @@ -1378,3 +1363,28 @@ func CalculateChecksums(ctx context.Context, path string) (hash.Hash, hash.Hash, return sha1h, md5h, adler32h, nil } + +// GetMTime reads the mtime from the extended attributes +func (n *Node) GetMTime(ctx context.Context) (time.Time, error) { + return n.lu.TimeManager().MTime(ctx, n) +} + +// GetTMTime reads the tmtime from the extended attributes +func (n *Node) GetTMTime(ctx context.Context) (time.Time, error) { + return n.lu.TimeManager().TMTime(ctx, n) +} + +// SetTMTime writes the UTC tmtime to the extended attributes or removes the attribute if nil is passed +func (n *Node) SetTMTime(ctx context.Context, t *time.Time) (err error) { + return n.lu.TimeManager().SetTMTime(ctx, n, t) +} + +// GetDTime reads the dmtime from the extended attributes +func (n *Node) GetDTime(ctx context.Context) (time.Time, error) { + return n.lu.TimeManager().DTime(ctx, n) +} + +// SetDTime writes the UTC dmtime to the extended attributes or removes the attribute if nil is passed +func (n *Node) SetDTime(ctx context.Context, t *time.Time) (err error) { + return n.lu.TimeManager().SetDTime(ctx, n, t) +} diff --git a/pkg/storage/utils/decomposedfs/recycle.go b/pkg/storage/utils/decomposedfs/recycle.go index fdc99b806b..c9be0783f1 100644 --- a/pkg/storage/utils/decomposedfs/recycle.go +++ b/pkg/storage/utils/decomposedfs/recycle.go @@ -33,12 +33,26 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" + "github.com/cs3org/reva/v2/pkg/storage" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storagespace" ) +type DecomposedfsTrashbin struct { + fs *Decomposedfs +} + +// Setup the trashbin +func (tb *DecomposedfsTrashbin) Setup(fs storage.FS) error { + if _, ok := fs.(*Decomposedfs); !ok { + return errors.New("invalid filesystem") + } + tb.fs = fs.(*Decomposedfs) + return nil +} + // Recycle items are stored inside the node folder and start with the uuid of the deleted node. // The `.T.` indicates it is a trash item and what follows is the timestamp of the deletion. // The deleted file is kept in the same location/dir as the original node. This prevents deletes @@ -49,7 +63,7 @@ import ( // ListRecycle returns the list of available recycle items // ref -> the space (= resourceid), key -> deleted node id, relativePath = relative to key -func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference, key, relativePath string) ([]*provider.RecycleItem, error) { +func (tb *DecomposedfsTrashbin) ListRecycle(ctx context.Context, ref *provider.Reference, key, relativePath string) ([]*provider.RecycleItem, error) { if ref == nil || ref.ResourceId == nil || ref.ResourceId.OpaqueId == "" { return nil, errtypes.BadRequest("spaceid required") @@ -62,11 +76,11 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference sublog := appctx.GetLogger(ctx).With().Str("spaceid", spaceID).Str("key", key).Str("relative_path", relativePath).Logger() // check permissions - trashnode, err := fs.lu.NodeFromSpaceID(ctx, spaceID) + trashnode, err := tb.fs.lu.NodeFromSpaceID(ctx, spaceID) if err != nil { return nil, err } - rp, err := fs.p.AssembleTrashPermissions(ctx, trashnode) + rp, err := tb.fs.p.AssembleTrashPermissions(ctx, trashnode) switch { case err != nil: return nil, err @@ -78,13 +92,13 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference } if key == "" && relativePath == "" { - return fs.listTrashRoot(ctx, spaceID) + return tb.listTrashRoot(ctx, spaceID) } // build a list of trash items relative to the given trash root and path items := make([]*provider.RecycleItem, 0) - trashRootPath := filepath.Join(fs.getRecycleRoot(spaceID), lookup.Pathify(key, 4, 2)) + trashRootPath := filepath.Join(tb.getRecycleRoot(spaceID), lookup.Pathify(key, 4, 2)) originalPath, _, timeSuffix, err := readTrashLink(trashRootPath) if err != nil { sublog.Error().Err(err).Str("trashRoot", trashRootPath).Msg("error reading trash link") @@ -92,7 +106,7 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference } origin := "" - attrs, err := fs.lu.MetadataBackend().All(ctx, originalPath) + attrs, err := tb.fs.lu.MetadataBackend().All(ctx, originalPath) if err != nil { return items, err } @@ -118,21 +132,21 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference var size int64 if relativePath == "" { // this is the case when we want to directly list a file in the trashbin - nodeType := fs.lu.TypeFromPath(ctx, originalPath) + nodeType := tb.fs.lu.TypeFromPath(ctx, originalPath) switch nodeType { case provider.ResourceType_RESOURCE_TYPE_FILE: - size, err = fs.lu.ReadBlobSizeAttr(ctx, originalPath) + _, size, err = tb.fs.lu.ReadBlobIDAndSizeAttr(ctx, originalPath, nil) if err != nil { return items, err } case provider.ResourceType_RESOURCE_TYPE_CONTAINER: - size, err = fs.lu.MetadataBackend().GetInt64(ctx, originalPath, prefixes.TreesizeAttr) + size, err = tb.fs.lu.MetadataBackend().GetInt64(ctx, originalPath, prefixes.TreesizeAttr) if err != nil { return items, err } } item := &provider.RecycleItem{ - Type: fs.lu.TypeFromPath(ctx, originalPath), + Type: tb.fs.lu.TypeFromPath(ctx, originalPath), Size: uint64(size), Key: filepath.Join(key, relativePath), DeletionTime: deletionTime, @@ -165,16 +179,16 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference // reset size size = 0 - nodeType := fs.lu.TypeFromPath(ctx, resolvedChildPath) + nodeType := tb.fs.lu.TypeFromPath(ctx, resolvedChildPath) switch nodeType { case provider.ResourceType_RESOURCE_TYPE_FILE: - size, err = fs.lu.ReadBlobSizeAttr(ctx, resolvedChildPath) + _, size, err = tb.fs.lu.ReadBlobIDAndSizeAttr(ctx, resolvedChildPath, nil) if err != nil { sublog.Error().Err(err).Str("name", name).Msg("invalid blob size, skipping") continue } case provider.ResourceType_RESOURCE_TYPE_CONTAINER: - size, err = fs.lu.MetadataBackend().GetInt64(ctx, resolvedChildPath, prefixes.TreesizeAttr) + size, err = tb.fs.lu.MetadataBackend().GetInt64(ctx, resolvedChildPath, prefixes.TreesizeAttr) if err != nil { sublog.Error().Err(err).Str("name", name).Msg("invalid tree size, skipping") continue @@ -218,16 +232,16 @@ func readTrashLink(path string) (string, string, string, error) { return resolved, link[15:51], link[54:], nil } -func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*provider.RecycleItem, error) { +func (tb *DecomposedfsTrashbin) listTrashRoot(ctx context.Context, spaceID string) ([]*provider.RecycleItem, error) { log := appctx.GetLogger(ctx) - trashRoot := fs.getRecycleRoot(spaceID) + trashRoot := tb.getRecycleRoot(spaceID) items := []*provider.RecycleItem{} subTrees, err := filepath.Glob(trashRoot + "/*") if err != nil { return nil, err } - numWorkers := fs.o.MaxConcurrency + numWorkers := tb.fs.o.MaxConcurrency if len(subTrees) < numWorkers { numWorkers = len(subTrees) } @@ -273,13 +287,13 @@ func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*p continue } - attrs, err := fs.lu.MetadataBackend().All(ctx, nodePath) + attrs, err := tb.fs.lu.MetadataBackend().All(ctx, nodePath) if err != nil { log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("could not get extended attributes, skipping") continue } - nodeType := fs.lu.TypeFromPath(ctx, nodePath) + nodeType := tb.fs.lu.TypeFromPath(ctx, nodePath) if nodeType == provider.ResourceType_RESOURCE_TYPE_INVALID { log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("invalid node type, skipping") continue @@ -331,14 +345,14 @@ func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*p } // RestoreRecycleItem restores the specified item -func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string, restoreRef *provider.Reference) error { +func (tb *DecomposedfsTrashbin) RestoreRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string, restoreRef *provider.Reference) error { if ref == nil { return errtypes.BadRequest("missing reference, needs a space id") } var targetNode *node.Node if restoreRef != nil { - tn, err := fs.lu.NodeFromResource(ctx, restoreRef) + tn, err := tb.fs.lu.NodeFromResource(ctx, restoreRef) if err != nil { return err } @@ -346,13 +360,13 @@ func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, ref *provider.Re targetNode = tn } - rn, parent, restoreFunc, err := fs.tp.RestoreRecycleItemFunc(ctx, ref.ResourceId.SpaceId, key, relativePath, targetNode) + rn, parent, restoreFunc, err := tb.fs.tp.RestoreRecycleItemFunc(ctx, ref.ResourceId.SpaceId, key, relativePath, targetNode) if err != nil { return err } // check permissions of deleted node - rp, err := fs.p.AssembleTrashPermissions(ctx, rn) + rp, err := tb.fs.p.AssembleTrashPermissions(ctx, rn) switch { case err != nil: return err @@ -367,7 +381,7 @@ func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, ref *provider.Re storagespace.ContextSendSpaceOwnerID(ctx, rn.SpaceOwnerOrManager(ctx)) // check we can write to the parent of the restore reference - pp, err := fs.p.AssemblePermissions(ctx, parent) + pp, err := tb.fs.p.AssemblePermissions(ctx, parent) switch { case err != nil: return err @@ -384,12 +398,12 @@ func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, ref *provider.Re } // PurgeRecycleItem purges the specified item, all its children and all their revisions -func (fs *Decomposedfs) PurgeRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string) error { +func (tb *DecomposedfsTrashbin) PurgeRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string) error { if ref == nil { return errtypes.BadRequest("missing reference, needs a space id") } - rn, purgeFunc, err := fs.tp.PurgeRecycleItemFunc(ctx, ref.ResourceId.OpaqueId, key, relativePath) + rn, purgeFunc, err := tb.fs.tp.PurgeRecycleItemFunc(ctx, ref.ResourceId.OpaqueId, key, relativePath) if err != nil { if errors.Is(err, iofs.ErrNotExist) { return errtypes.NotFound(key) @@ -398,7 +412,7 @@ func (fs *Decomposedfs) PurgeRecycleItem(ctx context.Context, ref *provider.Refe } // check permissions of deleted node - rp, err := fs.p.AssembleTrashPermissions(ctx, rn) + rp, err := tb.fs.p.AssembleTrashPermissions(ctx, rn) switch { case err != nil: return err @@ -414,26 +428,26 @@ func (fs *Decomposedfs) PurgeRecycleItem(ctx context.Context, ref *provider.Refe } // EmptyRecycle empties the trash -func (fs *Decomposedfs) EmptyRecycle(ctx context.Context, ref *provider.Reference) error { +func (tb *DecomposedfsTrashbin) EmptyRecycle(ctx context.Context, ref *provider.Reference) error { if ref == nil || ref.ResourceId == nil || ref.ResourceId.OpaqueId == "" { return errtypes.BadRequest("spaceid must be set") } - items, err := fs.ListRecycle(ctx, ref, "", "") + items, err := tb.ListRecycle(ctx, ref, "", "") if err != nil { return err } for _, i := range items { - if err := fs.PurgeRecycleItem(ctx, ref, i.Key, ""); err != nil { + if err := tb.PurgeRecycleItem(ctx, ref, i.Key, ""); err != nil { return err } } // TODO what permission should we check? we could check the root node of the user? or the owner permissions on his home root node? // The current impl will wipe your own trash. or when no user provided the trash of 'root' - return os.RemoveAll(fs.getRecycleRoot(ref.ResourceId.SpaceId)) + return os.RemoveAll(tb.getRecycleRoot(ref.ResourceId.SpaceId)) } -func (fs *Decomposedfs) getRecycleRoot(spaceID string) string { - return filepath.Join(fs.getSpaceRoot(spaceID), "trash") +func (tb *DecomposedfsTrashbin) getRecycleRoot(spaceID string) string { + return filepath.Join(tb.fs.getSpaceRoot(spaceID), "trash") } diff --git a/pkg/storage/utils/decomposedfs/revisions.go b/pkg/storage/utils/decomposedfs/revisions.go index 030b66bb50..771d60878a 100644 --- a/pkg/storage/utils/decomposedfs/revisions.go +++ b/pkg/storage/utils/decomposedfs/revisions.go @@ -86,7 +86,7 @@ func (fs *Decomposedfs) ListRevisions(ctx context.Context, ref *provider.Referen Key: n.ID + node.RevisionIDDelimiter + parts[1], Mtime: uint64(mtime.Unix()), } - blobSize, err := fs.lu.ReadBlobSizeAttr(ctx, items[i]) + _, blobSize, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, items[i], nil) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("error reading blobsize xattr, using 0") } @@ -148,13 +148,9 @@ func (fs *Decomposedfs) DownloadRevision(ctx context.Context, ref *provider.Refe contentPath := fs.lu.InternalPath(spaceID, revisionKey) - blobid, err := fs.lu.ReadBlobIDAttr(ctx, contentPath) + blobid, blobsize, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, contentPath, nil) if err != nil { - return nil, errors.Wrapf(err, "Decomposedfs: could not read blob id of revision '%s' for node '%s'", n.ID, revisionKey) - } - blobsize, err := fs.lu.ReadBlobSizeAttr(ctx, contentPath) - if err != nil { - return nil, errors.Wrapf(err, "Decomposedfs: could not read blob size of revision '%s' for node '%s'", n.ID, revisionKey) + return nil, errors.Wrapf(err, "Decomposedfs: could not read blob id and size for revision '%s' of node '%s'", n.ID, revisionKey) } revisionNode := node.Node{SpaceID: spaceID, BlobID: blobid, Blobsize: blobsize} // blobsize is needed for the s3ng blobstore @@ -238,7 +234,7 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer if err := os.Remove(newRevisionPath); err != nil { log.Error().Err(err).Str("revision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node") } - if err := fs.lu.MetadataBackend().Purge(newRevisionPath); err != nil { + if err := fs.lu.MetadataBackend().Purge(ctx, newRevisionPath); err != nil { log.Error().Err(err).Str("revision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node") } } @@ -299,7 +295,7 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer if err := os.Remove(fs.lu.MetadataBackend().LockfilePath(restoredRevisionPath)); err != nil { log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not delete old revision metadata lockfile, continuing") } - if err := fs.lu.MetadataBackend().Purge(restoredRevisionPath); err != nil { + if err := fs.lu.MetadataBackend().Purge(ctx, restoredRevisionPath); err != nil { log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not purge old revision from cache, continuing") } diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 8cedb275ab..31d556a69c 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -739,7 +739,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De } // invalidate cache - if err := fs.lu.MetadataBackend().Purge(n.InternalPath()); err != nil { + if err := fs.lu.MetadataBackend().Purge(ctx, n.InternalPath()); err != nil { return err } diff --git a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go index 1aa49e7a7c..b8b9d5d53b 100644 --- a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go +++ b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go @@ -25,6 +25,7 @@ import ( "path/filepath" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/aspects" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" @@ -150,9 +151,9 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { var lu *lookup.Lookup switch o.MetadataBackend { case "xattrs": - lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), o) + lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), o, &timemanager.Manager{}) case "messagepack": - lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), o) + lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), o, &timemanager.Manager{}) default: return nil, fmt.Errorf("unknown metadata backend %s", o.MetadataBackend) } @@ -175,6 +176,7 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) { Lookup: lu, Tree: tree, Permissions: permissions.NewPermissions(pmock, permissionsSelector), + Trashbin: &decomposedfs.DecomposedfsTrashbin{}, } fs, err := decomposedfs.New(o, aspects) if err != nil { diff --git a/pkg/storage/utils/decomposedfs/timemanager/timemanager.go b/pkg/storage/utils/decomposedfs/timemanager/timemanager.go new file mode 100644 index 0000000000..06be4eb57a --- /dev/null +++ b/pkg/storage/utils/decomposedfs/timemanager/timemanager.go @@ -0,0 +1,127 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +package timemanager + +import ( + "context" + "os" + "time" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" +) + +// Manager is responsible for managing time-related attributes of nodes in a decomposed file system. +type Manager struct { +} + +// OverrideMtime overrides the modification time (mtime) attribute of a node with the given time. +func (m *Manager) OverrideMtime(ctx context.Context, _ *node.Node, attrs *node.Attributes, mtime time.Time) error { + attrs.SetString(prefixes.MTimeAttr, mtime.UTC().Format(time.RFC3339Nano)) + return nil +} + +// MTime retrieves the modification time (mtime) attribute of a node. +// If the attribute is not set, it falls back to the file's last modification time. +func (dtm *Manager) MTime(ctx context.Context, n *node.Node) (time.Time, error) { + b, err := n.XattrString(ctx, prefixes.MTimeAttr) + if err != nil { + fi, err := os.Lstat(n.InternalPath()) + if err != nil { + return time.Time{}, err + } + return fi.ModTime(), nil + } + return time.Parse(time.RFC3339Nano, b) +} + +// SetMTime sets the modification time (mtime) attribute of a node to the given time. +// If the time is nil, the attribute is removed. +func (dtm *Manager) SetMTime(ctx context.Context, n *node.Node, mtime *time.Time) error { + if mtime == nil { + return n.RemoveXattr(ctx, prefixes.MTimeAttr, true) + } + return n.SetXattrString(ctx, prefixes.MTimeAttr, mtime.UTC().Format(time.RFC3339Nano)) +} + +// TMTime retrieves the tree modification time (tmtime) attribute of a node. +// If the attribute is not set, it falls back to the node's modification time (mtime). +func (dtm *Manager) TMTime(ctx context.Context, n *node.Node) (time.Time, error) { + b, err := n.XattrString(ctx, prefixes.TreeMTimeAttr) + if err == nil { + return time.Parse(time.RFC3339Nano, b) + } + + // no tmtime, use mtime + return dtm.MTime(ctx, n) +} + +// SetTMTime sets the tree modification time (tmtime) attribute of a node to the given time. +// If the time is nil, the attribute is removed. +func (dtm *Manager) SetTMTime(ctx context.Context, n *node.Node, tmtime *time.Time) error { + if tmtime == nil { + return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true) + } + return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, tmtime.UTC().Format(time.RFC3339Nano)) +} + +// CTime retrieves the creation time (ctime) attribute of a node. +// Since decomposedfs does not differentiate between ctime and mtime, it falls back to the node's modification time (mtime). +func (dtm *Manager) CTime(ctx context.Context, n *node.Node) (time.Time, error) { + // decomposedfs does not differentiate between ctime and mtime + return dtm.MTime(ctx, n) +} + +// SetCTime sets the creation time (ctime) attribute of a node to the given time. +// Since decomposedfs does not differentiate between ctime and mtime, it sets the modification time (mtime) instead. +func (dtm *Manager) SetCTime(ctx context.Context, n *node.Node, mtime *time.Time) error { + // decomposedfs does not differentiate between ctime and mtime + return dtm.SetMTime(ctx, n, mtime) +} + +// TCTime retrieves the tree creation time (tctime) attribute of a node. +// Since decomposedfs does not differentiate between ctime and mtime, it falls back to the tree modification time (tmtime). +func (dtm *Manager) TCTime(ctx context.Context, n *node.Node) (time.Time, error) { + // decomposedfs does not differentiate between ctime and mtime + return dtm.TMTime(ctx, n) +} + +// SetTCTime sets the tree creation time (tctime) attribute of a node to the given time. +// Since decomposedfs does not differentiate between ctime and mtime, it sets the tree modification time (tmtime) instead. +func (dtm *Manager) SetTCTime(ctx context.Context, n *node.Node, tmtime *time.Time) error { + // decomposedfs does not differentiate between ctime and mtime + return dtm.SetTMTime(ctx, n, tmtime) +} + +// DTime retrieves the deletion time (dtime) attribute of a node. +func (dtm *Manager) DTime(ctx context.Context, n *node.Node) (tmTime time.Time, err error) { + b, err := n.XattrString(ctx, prefixes.DTimeAttr) + if err != nil { + return time.Time{}, err + } + return time.Parse(time.RFC3339Nano, b) +} + +// SetDTime sets the deletion time (dtime) attribute of a node to the given time. +// If the time is nil, the attribute is removed. +func (dtm *Manager) SetDTime(ctx context.Context, n *node.Node, t *time.Time) (err error) { + if t == nil { + return n.RemoveXattr(ctx, prefixes.DTimeAttr, true) + } + return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano)) +} diff --git a/pkg/storage/utils/decomposedfs/trashbin/trashbin.go b/pkg/storage/utils/decomposedfs/trashbin/trashbin.go new file mode 100644 index 0000000000..dd44f6e71a --- /dev/null +++ b/pkg/storage/utils/decomposedfs/trashbin/trashbin.go @@ -0,0 +1,35 @@ +// Copyright 2018-2024 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package trashbin + +import ( + "context" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/storage" +) + +type Trashbin interface { + Setup(storage.FS) error + + ListRecycle(ctx context.Context, ref *provider.Reference, key, relativePath string) ([]*provider.RecycleItem, error) + RestoreRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string, restoreRef *provider.Reference) error + PurgeRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string) error + EmptyRecycle(ctx context.Context, ref *provider.Reference) error +} diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index de070c874f..f10735b664 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -733,7 +733,7 @@ func (t *Tree) removeNode(ctx context.Context, path, timeSuffix string, n *node. return err } - if err := t.lookup.MetadataBackend().Purge(path); err != nil { + if err := t.lookup.MetadataBackend().Purge(ctx, path); err != nil { logger.Error().Err(err).Str("path", t.lookup.MetadataBackend().MetadataPath(path)).Msg("error purging node metadata") return err } @@ -757,7 +757,7 @@ func (t *Tree) removeNode(ctx context.Context, path, timeSuffix string, n *node. continue } - bID, err := t.lookup.ReadBlobIDAttr(ctx, rev) + bID, _, err := t.lookup.ReadBlobIDAndSizeAttr(ctx, rev, nil) if err != nil { logger.Error().Err(err).Str("revision", rev).Msg("error reading blobid attribute") return err diff --git a/pkg/storage/utils/decomposedfs/upload/store.go b/pkg/storage/utils/decomposedfs/upload/store.go index 9abc856fc8..74744e6da9 100644 --- a/pkg/storage/utils/decomposedfs/upload/store.go +++ b/pkg/storage/utils/decomposedfs/upload/store.go @@ -255,15 +255,8 @@ func (store OcisStore) CreateNodeForUpload(session *OcisSession, initAttrs node. return nil, err } - mtime := time.Now() - if !session.MTime().IsZero() { - // overwrite mtime if requested - mtime = session.MTime() - } - // overwrite technical information initAttrs.SetString(prefixes.IDAttr, n.ID) - initAttrs.SetString(prefixes.MTimeAttr, mtime.UTC().Format(time.RFC3339Nano)) initAttrs.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE)) initAttrs.SetString(prefixes.ParentidAttr, n.ParentID) initAttrs.SetString(prefixes.NameAttr, n.Name) @@ -271,6 +264,17 @@ func (store OcisStore) CreateNodeForUpload(session *OcisSession, initAttrs node. initAttrs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize) initAttrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+session.ID()) + // set mtime on the new node + mtime := time.Now() + if !session.MTime().IsZero() { + // overwrite mtime if requested + mtime = session.MTime() + } + err = store.lu.TimeManager().OverrideMtime(ctx, n, &initAttrs, mtime) + if err != nil { + return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime") + } + // update node metadata with new blobid etc err = n.SetXattrsWithContext(ctx, initAttrs, false) if err != nil { @@ -367,7 +371,7 @@ func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSess } // delete old blob - bID, err := session.store.lu.ReadBlobIDAttr(ctx, versionPath) + bID, _, err := session.store.lu.ReadBlobIDAndSizeAttr(ctx, versionPath, nil) if err != nil { return unlock, err } diff --git a/pkg/storage/utils/decomposedfs/upload/store_test.go b/pkg/storage/utils/decomposedfs/upload/store_test.go index 9bfa3b475e..c72b27c9de 100644 --- a/pkg/storage/utils/decomposedfs/upload/store_test.go +++ b/pkg/storage/utils/decomposedfs/upload/store_test.go @@ -14,6 +14,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/timemanager" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" ) @@ -22,7 +23,7 @@ func TestInitNewNode(t *testing.T) { root := t.TempDir() - lookup := lookup.New(metadata.NewMessagePackBackend(root, cache.Config{}), &options.Options{Root: root}) + lookup := lookup.New(metadata.NewMessagePackBackend(root, cache.Config{}), &options.Options{Root: root}, &timemanager.Manager{}) tp := tree.New(lookup, nil, &options.Options{}, nil) aspects := aspects.Aspects{ diff --git a/pkg/storage/utils/decomposedfs/upload_async_test.go b/pkg/storage/utils/decomposedfs/upload_async_test.go index 47ead74e84..f5c0907202 100644 --- a/pkg/storage/utils/decomposedfs/upload_async_test.go +++ b/pkg/storage/utils/decomposedfs/upload_async_test.go @@ -24,6 +24,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/permissions" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/permissions/mocks" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/timemanager" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" treemocks "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/mocks" "github.com/cs3org/reva/v2/pkg/storagespace" @@ -139,7 +140,7 @@ var _ = Describe("Async file uploads", Ordered, func() { }) Expect(err).ToNot(HaveOccurred()) - lu = lookup.New(metadata.NewXattrsBackend(o.Root, cache.Config{}), o) + lu = lookup.New(metadata.NewXattrsBackend(o.Root, cache.Config{}), o, &timemanager.Manager{}) pmock = &mocks.PermissionsChecker{} cs3permissionsclient = &mocks.CS3PermissionsClient{} @@ -177,6 +178,7 @@ var _ = Describe("Async file uploads", Ordered, func() { Tree: tree, Permissions: permissions.NewPermissions(pmock, permissionsSelector), EventStream: stream.Chan{pub, con}, + Trashbin: &DecomposedfsTrashbin{}, } fs, err = New(o, aspects) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/storage/utils/decomposedfs/upload_test.go b/pkg/storage/utils/decomposedfs/upload_test.go index 3ae7033d26..a8bb593924 100644 --- a/pkg/storage/utils/decomposedfs/upload_test.go +++ b/pkg/storage/utils/decomposedfs/upload_test.go @@ -42,6 +42,7 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/permissions" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/permissions/mocks" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/timemanager" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" treemocks "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/mocks" "github.com/cs3org/reva/v2/pkg/storagespace" @@ -104,7 +105,7 @@ var _ = Describe("File uploads", func() { "root": tmpRoot, }) Expect(err).ToNot(HaveOccurred()) - lu = lookup.New(metadata.NewXattrsBackend(o.Root, cache.Config{}), o) + lu = lookup.New(metadata.NewXattrsBackend(o.Root, cache.Config{}), o, &timemanager.Manager{}) pmock = &mocks.PermissionsChecker{} cs3permissionsclient = &mocks.CS3PermissionsClient{} pool.RemoveSelector("PermissionsSelector" + "any") @@ -141,6 +142,7 @@ var _ = Describe("File uploads", func() { Lookup: lu, Tree: tree, Permissions: permissions.NewPermissions(pmock, permissionsSelector), + Trashbin: &decomposedfs.DecomposedfsTrashbin{}, } fs, err = decomposedfs.New(o, aspects) Expect(err).ToNot(HaveOccurred()) diff --git a/tests/acceptance/expected-failures-on-POSIX-storage.md b/tests/acceptance/expected-failures-on-POSIX-storage.md index 9c1f53f09b..3e2aac5235 100644 --- a/tests/acceptance/expected-failures-on-POSIX-storage.md +++ b/tests/acceptance/expected-failures-on-POSIX-storage.md @@ -7,10 +7,10 @@ Basic file management like up and download, move, copy, properties, quota, trash #### [invalid webdav responses for unauthorized requests.](https://github.com/owncloud/product/issues/273) These tests succeed when running against ocis because there we handle the relevant authentication in the proxy. -- [coreApiTrashbin/trashbinFilesFolders.feature:302](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L302) -- [coreApiTrashbin/trashbinFilesFolders.feature:303](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L303) -- [coreApiTrashbin/trashbinFilesFolders.feature:315](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L315) -- [coreApiTrashbin/trashbinFilesFolders.feature:372](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L372) +- [coreApiTrashbin/trashbinFilesFolders.feature:240](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L240) +- [coreApiTrashbin/trashbinFilesFolders.feature:241](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L241) +- [coreApiTrashbin/trashbinFilesFolders.feature:255](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L255) +- [coreApiTrashbin/trashbinFilesFolders.feature:256](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L256) #### [Custom dav properties with namespaces are rendered incorrectly](https://github.com/owncloud/ocis/issues/2140) _ocdav: double check the webdav property parsing when custom namespaces are used_ @@ -215,8 +215,13 @@ _The below features have been added after I last categorized them. AFAICT they a - [coreApiWebdavMove2/moveFile.feature:100](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L100) - [coreApiWebdavMove2/moveFile.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L101) - [coreApiWebdavMove2/moveFile.feature:102](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L102) +- [coreApiWebdavMove1/moveFolder.feature:217](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L217) +- [coreApiWebdavMove1/moveFolder.feature:218](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L218) +- [coreApiWebdavMove1/moveFolder.feature:219](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L219) + ### posixfs doesn't do versions at that point +- [coreApiWebdavEtagPropagation2/restoreVersion.feature:13](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreVersion.feature#L13) - [coreApiWebdavUploadTUS/uploadFile.feature:146](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadFile.feature#L146) - [coreApiWebdavUploadTUS/uploadFile.feature:147](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadFile.feature#L147) - [coreApiWebdavUploadTUS/uploadFile.feature:122](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadFile.feature#L122) @@ -245,130 +250,6 @@ _The below features have been added after I last categorized them. AFAICT they a - [coreApiWebdavMove2/moveFile.feature:42](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L42) - [coreApiWebdavMove2/moveFile.feature:43](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L43) -### posixfs doesn't do trashbin at that point -- [coreApiTrashbin/trashbinDelete.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L49) -- [coreApiTrashbin/trashbinDelete.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L50) -- [coreApiTrashbin/trashbinDelete.feature:72](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L72) -- [coreApiTrashbin/trashbinDelete.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L73) -- [coreApiTrashbin/trashbinDelete.feature:91](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L91) -- [coreApiTrashbin/trashbinDelete.feature:110](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L110) -- [coreApiTrashbin/trashbinDelete.feature:111](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L111) -- [coreApiTrashbin/trashbinDelete.feature:129](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L129) -- [coreApiTrashbin/trashbinDelete.feature:151](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L151) -- [coreApiTrashbin/trashbinDelete.feature:171](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L171) -- [coreApiTrashbin/trashbinDelete.feature:130](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L130) -- [coreApiTrashbin/trashbinDelete.feature:150](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L150) -- [coreApiTrashbin/trashbinDelete.feature:172](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L172) -- [coreApiTrashbin/trashbinDelete.feature:204](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L204) -- [coreApiTrashbin/trashbinDelete.feature:238](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L238) -- [coreApiTrashbin/trashbinDelete.feature:205](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L205) -- [coreApiTrashbin/trashbinDelete.feature:237](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L237) -- [coreApiTrashbin/trashbinDelete.feature:282](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L282) -- [coreApiTrashbin/trashbinDelete.feature:283](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L283) -- [coreApiTrashbin/trashbinFilesFolders.feature:20](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L20) -- [coreApiTrashbin/trashbinFilesFolders.feature:21](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L21) -- [coreApiTrashbin/trashbinFilesFolders.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L32) -- [coreApiTrashbin/trashbinFilesFolders.feature:33](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L33) -- [coreApiTrashbin/trashbinFilesFolders.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L47) -- [coreApiTrashbin/trashbinFilesFolders.feature:48](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L48) -- [coreApiTrashbin/trashbinFilesFolders.feature:122](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L122) -- [coreApiTrashbin/trashbinFilesFolders.feature:123](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L123) -- [coreApiTrashbin/trashbinFilesFolders.feature:141](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L141) -- [coreApiTrashbin/trashbinFilesFolders.feature:142](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L142) -- [coreApiTrashbin/trashbinFilesFolders.feature:316](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L316) -- [coreApiTrashbin/trashbinFilesFolders.feature:240](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L240) -- [coreApiTrashbin/trashbinFilesFolders.feature:241](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L241) -- [coreApiTrashbin/trashbinFilesFolders.feature:255](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L255) -- [coreApiTrashbin/trashbinFilesFolders.feature:256](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L256) -- [coreApiTrashbin/trashbinFilesFolders.feature:269](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L269) -- [coreApiTrashbin/trashbinFilesFolders.feature:270](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L270) -- [coreApiTrashbin/trashbinFilesFolders.feature:271](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L271) -- [coreApiTrashbin/trashbinFilesFolders.feature:272](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L272) -- [coreApiTrashbin/trashbinFilesFolders.feature:273](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L273) -- [coreApiTrashbin/trashbinFilesFolders.feature:274](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L274) -- [coreApiTrashbin/trashbinFilesFolders.feature:286](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L286) -- [coreApiTrashbin/trashbinFilesFolders.feature:287](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L287) -- [coreApiTrashbin/trashbinFilesFolders.feature:373](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L373) -- [coreApiTrashbin/trashbinFilesFolders.feature:429](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L429) -- [coreApiTrashbin/trashbinFilesFolders.feature:430](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L430) -- [coreApiTrashbinRestore/trashbinRestore.feature:34](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L34) -- [coreApiTrashbinRestore/trashbinRestore.feature:35](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L35) -- [coreApiTrashbinRestore/trashbinRestore.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L50) -- [coreApiTrashbinRestore/trashbinRestore.feature:51](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L51) -- [coreApiTrashbinRestore/trashbinRestore.feature:68](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L68) -- [coreApiTrashbinRestore/trashbinRestore.feature:69](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L69) -- [coreApiTrashbinRestore/trashbinRestore.feature:88](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L88) -- [coreApiTrashbinRestore/trashbinRestore.feature:89](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L89) -- [coreApiTrashbinRestore/trashbinRestore.feature:90](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L90) -- [coreApiTrashbinRestore/trashbinRestore.feature:91](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L91) -- [coreApiTrashbinRestore/trashbinRestore.feature:92](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L92) -- [coreApiTrashbinRestore/trashbinRestore.feature:93](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L93) -- [coreApiTrashbinRestore/trashbinRestore.feature:108](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L108) -- [coreApiTrashbinRestore/trashbinRestore.feature:109](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L109) -- [coreApiTrashbinRestore/trashbinRestore.feature:110](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L110) -- [coreApiTrashbinRestore/trashbinRestore.feature:111](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L111) -- [coreApiTrashbinRestore/trashbinRestore.feature:130](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L130) -- [coreApiTrashbinRestore/trashbinRestore.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L131) -- [coreApiTrashbinRestore/trashbinRestore.feature:145](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L145) -- [coreApiTrashbinRestore/trashbinRestore.feature:146](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L146) -- [coreApiTrashbinRestore/trashbinRestore.feature:160](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L160) -- [coreApiTrashbinRestore/trashbinRestore.feature:161](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L161) -- [coreApiTrashbinRestore/trashbinRestore.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L175) -- [coreApiTrashbinRestore/trashbinRestore.feature:176](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L176) -- [coreApiTrashbinRestore/trashbinRestore.feature:190](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L190) -- [coreApiTrashbinRestore/trashbinRestore.feature:191](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L191) -- [coreApiTrashbinRestore/trashbinRestore.feature:192](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L192) -- [coreApiTrashbinRestore/trashbinRestore.feature:193](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L193) -- [coreApiTrashbinRestore/trashbinRestore.feature:194](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L194) -- [coreApiTrashbinRestore/trashbinRestore.feature:195](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L195) -- [coreApiTrashbinRestore/trashbinRestore.feature:212](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L212) -- [coreApiTrashbinRestore/trashbinRestore.feature:213](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L213) -- [coreApiTrashbinRestore/trashbinRestore.feature:230](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L230) -- [coreApiTrashbinRestore/trashbinRestore.feature:231](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L231) -- [coreApiTrashbinRestore/trashbinRestore.feature:250](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L250) -- [coreApiTrashbinRestore/trashbinRestore.feature:251](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L251) -- [coreApiTrashbinRestore/trashbinRestore.feature:270](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L270) -- [coreApiTrashbinRestore/trashbinRestore.feature:271](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L271) -- [coreApiTrashbinRestore/trashbinRestore.feature:304](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L304) -- [coreApiTrashbinRestore/trashbinRestore.feature:305](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L305) -- [coreApiTrashbinRestore/trashbinRestore.feature:343](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L343) -- [coreApiTrashbinRestore/trashbinRestore.feature:344](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L344) -- [coreApiTrashbinRestore/trashbinRestore.feature:387](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L387) -- [coreApiTrashbinRestore/trashbinRestore.feature:388](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L388) -- [coreApiTrashbinRestore/trashbinRestore.feature:405](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L405) -- [coreApiTrashbinRestore/trashbinRestore.feature:406](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L406) -- [coreApiTrashbinRestore/trashbinRestore.feature:424](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L424) -- [coreApiTrashbinRestore/trashbinRestore.feature:425](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L425) -- [coreApiTrashbinRestore/trashbinRestore.feature:448](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L448) -- [coreApiTrashbinRestore/trashbinRestore.feature:449](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L449) -- [coreApiTrashbinRestore/trashbinRestore.feature:467](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L467) -- [coreApiTrashbinRestore/trashbinRestore.feature:468](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L468) -- [coreApiTrashbinRestore/trashbinRestore.feature:482](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L482) -- [coreApiTrashbinRestore/trashbinRestore.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L483) -- [coreApiTrashbinRestore/trashbinRestore.feature:536](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L536) -- [coreApiTrashbinRestore/trashbinRestore.feature:537](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L537) -- [coreApiTrashbinRestore/trashbinRestore.feature:552](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L552) -- [coreApiTrashbinRestore/trashbinRestore.feature:553](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L553) -- [coreApiTrashbinRestore/trashbinRestore.feature:568](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L568) -- [coreApiTrashbinRestore/trashbinRestore.feature:569](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L569) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L28) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L29) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:51](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L51) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:52](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L52) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:72](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L72) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L73) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:95](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L95) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L96) -- [coreApiWebdavEtagPropagation2/restoreVersion.feature:13](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreVersion.feature#L13) -- [coreApiWebdavMove2/moveFile.feature:117](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L117) -- [coreApiWebdavMove2/moveFile.feature:118](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L118) -- [coreApiWebdavMove2/moveFile.feature:119](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L119) -- [coreApiWebdavOperations/listFiles.feature:221](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L221) -- [coreApiWebdavOperations/listFiles.feature:222](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L222) -- [coreApiWebdavOperations/listFiles.feature:223](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L223) -- [coreApiWebdavMove1/moveFolder.feature:235](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L235) -- [coreApiWebdavMove1/moveFolder.feature:236](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L236) -- [coreApiWebdavMove1/moveFolder.feature:237](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove1/moveFolder.feature#L237) ### [COPY file/folder to same name is possible (but 500 code error for folder with spaces path)](https://github.com/owncloud/ocis/issues/8711) diff --git a/tests/oc-integration-tests/drone/storage-users-posixfs.toml b/tests/oc-integration-tests/drone/storage-users-posixfs.toml index aa6bfd4fee..212fabdafe 100644 --- a/tests/oc-integration-tests/drone/storage-users-posixfs.toml +++ b/tests/oc-integration-tests/drone/storage-users-posixfs.toml @@ -26,6 +26,7 @@ treetime_accounting = true treesize_accounting = true personalspacepath_template = "users/{{.User.Username}}" generalspacepath_template = "projects/{{.SpaceId}}" +watch_fs = true [grpc.services.storageprovider.drivers.posix.filemetadatacache] cache_store = "noop" @@ -48,6 +49,7 @@ root = "/drone/src/tmp/reva/data" permissionssvc = "localhost:10000" treetime_accounting = true treesize_accounting = true +scan_debounce_delay = 0 [http.services.dataprovider.drivers.posix.idcache] cache_store = "redis"