Posixfs trashbin

changelog/unreleased/improve-posixfs.md

@@ -0,0 +1,5 @@
+Enhancement: Add trashbin support to posixfs alongside other improvements
+
+We added support for trashbins to posixfs. Posixfs also saw a number of other improvement, bugfixes and optimizations.
+
+https://github.com/cs3org/reva/pull/4779

internal/http/services/owncloud/ocdav/ocdav.go

@@ -178,7 +178,7 @@ func (s *svc) Handler() http.Handler {
 
  var head string
  head, r.URL.Path = router.ShiftPath(r.URL.Path)
- log.Debug().Str("head", head).Str("tail", r.URL.Path).Msg("http routing")
+ log.Debug().Str("method", r.Method).Str("head", head).Str("tail", r.URL.Path).Msg("http routing")
  switch head {
  case "status.php", "status":
  s.doStatus(w, r)

pkg/storage/fs/posix/blobstore/blobstore.go

@@ -41,6 +41,11 @@ func New(root string) (*Blobstore, error) {
 
 // Upload stores some data in the blobstore under the given key
 func (bs *Blobstore) Upload(node *node.Node, source string) error {
+ path := node.InternalPath()
+
+ // preserve the mtime of the file
+ fi, _ := os.Stat(path)
+
  file, err := os.Open(source)
  if err != nil {
  return errors.Wrap(err, "Decomposedfs: oCIS blobstore: Can not open source file to upload")
@@ -59,7 +64,15 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error {
  return errors.Wrapf(err, "could not write blob '%s'", node.InternalPath())
  }
 
- return w.Flush()
+ err = w.Flush()
+ if err != nil {
+ return err
+ }
+
+ if fi != nil {
+ return os.Chtimes(path, fi.ModTime(), fi.ModTime())
+ }
+ return nil
 }
 
 // Download retrieves a blob from the blobstore for reading

pkg/storage/fs/posix/lookup/lookup.go

@@ -72,15 +72,17 @@ type Lookup struct {
  IDCache IDCache
  metadataBackend metadata.Backend
  userMapper usermapper.Mapper
+ tm node.TimeManager
 }
 
 // New returns a new Lookup instance
-func New(b metadata.Backend, um usermapper.Mapper, o *options.Options) *Lookup {
+func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.TimeManager) *Lookup {
  lu := &Lookup{
  Options: o,
  metadataBackend: b,
  IDCache: NewStoreIDCache(&o.Options),
  userMapper: um,
+ tm: tm,
  }
 
  return lu
@@ -122,22 +124,12 @@ func (lu *Lookup) MetadataBackend() metadata.Backend {
  return lu.metadataBackend
 }
 
-// ReadBlobSizeAttr reads the blobsize from the xattrs
-func (lu *Lookup) ReadBlobSizeAttr(ctx context.Context, path string) (int64, error) {
- blobSize, err := lu.metadataBackend.GetInt64(ctx, path, prefixes.BlobsizeAttr)
+func (lu *Lookup) ReadBlobIDAndSizeAttr(ctx context.Context, path string, _ node.Attributes) (string, int64, error) {
+ fi, err := os.Stat(path)
  if err != nil {
- return 0, errors.Wrapf(err, "error reading blobsize xattr")
+ return "", 0, errors.Wrap(err, "error stating file")
  }
- return blobSize, nil
-}
-
-// ReadBlobIDAttr reads the blobsize from the xattrs
-func (lu *Lookup) ReadBlobIDAttr(ctx context.Context, path string) (string, error) {
- attr, err := lu.metadataBackend.Get(ctx, path, prefixes.BlobIDAttr)
- if err != nil {
- return "", errors.Wrapf(err, "error reading blobid xattr")
- }
- return string(attr), nil
+ return "", fi.Size(), nil
 }
 
 // TypeFromPath returns the type of the node at the given path
@@ -421,3 +413,8 @@ func (lu *Lookup) GenerateSpaceID(spaceType string, owner *user.User) (string, e
  return "", fmt.Errorf("unsupported space type: %s", spaceType)
  }
 }
+
+// TimeManager returns the time manager
+func (lu *Lookup) TimeManager() node.TimeManager {
+ return lu.tm
+}

pkg/storage/fs/posix/options/options.go

@@ -19,6 +19,8 @@
 package options
 
 import (
+ "time"
+
  decomposedoptions "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options"
  "github.com/mitchellh/mapstructure"
  "github.com/pkg/errors"
@@ -29,6 +31,8 @@ type Options struct {
 
  UseSpaceGroups bool `mapstructure:"use_space_groups"`
 
+ ScanDebounceDelay time.Duration `mapstructure:"scan_debounce_delay"`
+
  WatchFS bool `mapstructure:"watch_fs"`
  WatchType string `mapstructure:"watch_type"`
  WatchPath string `mapstructure:"watch_path"`

pkg/storage/fs/posix/posix.go

@@ -36,6 +36,8 @@ import (
  "github.com/cs3org/reva/v2/pkg/storage/fs/posix/blobstore"
  "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup"
  "github.com/cs3org/reva/v2/pkg/storage/fs/posix/options"
+ "github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager"
+ "github.com/cs3org/reva/v2/pkg/storage/fs/posix/trashbin"
  "github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree"
  "github.com/cs3org/reva/v2/pkg/storage/fs/registry"
  "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs"
@@ -68,24 +70,39 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) {
  return nil, err
  }
 
- bs, err := blobstore.New(o.Root)
- if err != nil {
- return nil, err
- }
-
+ fs := &posixFS{}
  um := usermapper.NewUnixMapper()
 
  var lu *lookup.Lookup
  switch o.MetadataBackend {
  case "xattrs":
- lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o)
+ lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{})
  case "messagepack":
- lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o)
+ lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{})
  default:
  return nil, fmt.Errorf("unknown metadata backend %s, only 'messagepack' or 'xattrs' (default) supported", o.MetadataBackend)
  }
 
- tp, err := tree.New(lu, bs, um, o, stream, store.Create(
+ trashbin, err := trashbin.New(o, lu)
+ if err != nil {
+ return nil, err
+ }
+ err = trashbin.Setup(fs)
+ if err != nil {
+ return nil, err
+ }
+
+ bs, err := blobstore.New(o.Root)
+ if err != nil {
+ return nil, err
+ }
+
+ switch o.IDCache.Store {
+ case "", "memory", "noop":
+ return nil, fmt.Errorf("the posix driver requires a shared id cache, e.g. nats-js-kv or redis")
+ }
+
+ tp, err := tree.New(lu, bs, um, trashbin, o, stream, store.Create(
  store.Store(o.IDCache.Store),
  store.TTL(o.IDCache.TTL),
  store.Size(o.IDCache.Size),
@@ -113,6 +130,7 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) {
  EventStream: stream,
  UserMapper: um,
  DisableVersioning: true,
+ Trashbin: trashbin,
  }
 
  dfs, err := decomposedfs.New(&o.Options, aspects)
@@ -154,10 +172,8 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) {
  }
 
  mw := middleware.NewFS(dfs, hooks...)
- fs := &posixFS{
- FS: mw,
- um: um,
- }
+ fs.FS = mw
+ fs.um = um
 
  return fs, nil
 }

pkg/storage/fs/posix/posix_test.go

@@ -42,6 +42,9 @@ var _ = Describe("Posix", func() {
  "root": tmpRoot,
  "share_folder": "/Shares",
  "permissionssvc": "any",
+ "idcache": map[string]interface{}{
+ "cache_store": "nats-js-kv",
+ },
  }
  })
 

pkg/storage/fs/posix/testhelpers/helpers.go

@@ -36,6 +36,8 @@ import (
  "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
  "github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup"
  "github.com/cs3org/reva/v2/pkg/storage/fs/posix/options"
+ "github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager"
+ "github.com/cs3org/reva/v2/pkg/storage/fs/posix/trashbin"
  "github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree"
  "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs"
  "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/aspects"
@@ -152,12 +154,13 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) {
  },
  },
  }
+
  var lu *lookup.Lookup
  switch o.MetadataBackend {
  case "xattrs":
- lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o)
+ lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{})
  case "messagepack":
- lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o)
+ lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{})
  default:
  return nil, fmt.Errorf("unknown metadata backend %s", o.MetadataBackend)
  }
@@ -175,14 +178,19 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) {
  )
 
  bs := &treemocks.Blobstore{}
- tree, err := tree.New(lu, bs, um, o, nil, store.Create())
+ tree, err := tree.New(lu, bs, um, &trashbin.Trashbin{}, o, nil, store.Create())
+ if err != nil {
+ return nil, err
+ }
+ tb, err := trashbin.New(o, lu)
  if err != nil {
  return nil, err
  }
  aspects := aspects.Aspects{
  Lookup: lu,
  Tree: tree,
  Permissions: permissions.NewPermissions(pmock, permissionsSelector),
+ Trashbin: tb,
  }
  fs, err := decomposedfs.New(&o.Options, aspects)
  if err != nil {

pkg/storage/fs/posix/timemanager/timemanager.go

@@ -0,0 +1,117 @@
+// Copyright 2018-2024 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package timemanager
+
+import (
+ "context"
+ "os"
+ "syscall"
+ "time"
+
+ "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
+ "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
+)
+
+// Manager is responsible for managing time-related operations on files and directories.
+type Manager struct {
+}
+
+// OverrideMtime overrides the modification time (mtime) of a node with the specified time.
+func (m *Manager) OverrideMtime(ctx context.Context, n *node.Node, _ *node.Attributes, mtime time.Time) error {
+ return os.Chtimes(n.InternalPath(), mtime, mtime)
+}
+
+// MTime returns the modification time (mtime) of a node.
+func (m *Manager) MTime(ctx context.Context, n *node.Node) (time.Time, error) {
+ fi, err := os.Stat(n.InternalPath())
+ if err != nil {
+ return time.Time{}, err
+ }
+ return fi.ModTime(), nil
+}
+
+// SetMTime sets the modification time (mtime) of a node to the specified time.
+func (m *Manager) SetMTime(ctx context.Context, n *node.Node, mtime *time.Time) error {
+ return os.Chtimes(n.InternalPath(), *mtime, *mtime)
+}
+
+// TMTime returns the tree modification time (tmtime) of a node.
+// If the tmtime is not set, it falls back to the modification time (mtime).
+func (m *Manager) TMTime(ctx context.Context, n *node.Node) (time.Time, error) {
+ b, err := n.XattrString(ctx, prefixes.TreeMTimeAttr)
+ if err == nil {
+ return time.Parse(time.RFC3339Nano, b)
+ }
+
+ // no tmtime, use mtime
+ return m.MTime(ctx, n)
+}
+
+// SetTMTime sets the tree modification time (tmtime) of a node to the specified time.
+// If tmtime is nil, the tmtime attribute is removed.
+func (m *Manager) SetTMTime(ctx context.Context, n *node.Node, tmtime *time.Time) error {
+ if tmtime == nil {
+ return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true)
+ }
+ return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, tmtime.UTC().Format(time.RFC3339Nano))
+}
+
+// CTime returns the creation time (ctime) of a node.
+func (m *Manager) CTime(ctx context.Context, n *node.Node) (time.Time, error) {
+ fi, err := os.Stat(n.InternalPath())
+ if err != nil {
+ return time.Time{}, err
+ }
+
+ stat := fi.Sys().(*syscall.Stat_t)
+ //nolint:unconvert
+ return time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)), nil
+}
+
+// TCTime returns the tree creation time (tctime) of a node.
+// Since decomposedfs does not differentiate between ctime and mtime, it falls back to TMTime.
+func (m *Manager) TCTime(ctx context.Context, n *node.Node) (time.Time, error) {
+ // decomposedfs does not differentiate between ctime and mtime
+ return m.TMTime(ctx, n)
+}
+
+// SetTCTime sets the tree creation time (tctime) of a node to the specified time.
+// Since decomposedfs does not differentiate between ctime and mtime, it falls back to SetTMTime.
+func (m *Manager) SetTCTime(ctx context.Context, n *node.Node, tmtime *time.Time) error {
+ // decomposedfs does not differentiate between ctime and mtime
+ return m.SetTMTime(ctx, n, tmtime)
+}
+
+// DTime returns the deletion time (dtime) of a node.
+func (m *Manager) DTime(ctx context.Context, n *node.Node) (tmTime time.Time, err error) {
+ b, err := n.XattrString(ctx, prefixes.DTimeAttr)
+ if err != nil {
+ return time.Time{}, err
+ }
+ return time.Parse(time.RFC3339Nano, b)
+}
+
+// SetDTime sets the deletion time (dtime) of a node to the specified time.
+// If t is nil, the dtime attribute is removed.
+func (m *Manager) SetDTime(ctx context.Context, n *node.Node, t *time.Time) (err error) {
+ if t == nil {
+ return n.RemoveXattr(ctx, prefixes.DTimeAttr, true)
+ }
+ return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano))
+}