Skip to content

Commit

Permalink
Merge pull request #4779 from aduffeck/posixfs-trashbin
Browse files Browse the repository at this point in the history
Posixfs trashbin
  • Loading branch information
butonic committed Sep 16, 2024
2 parents a7bf2bc + a1d58bf commit cedd4ef
Show file tree
Hide file tree
Showing 37 changed files with 1,076 additions and 462 deletions.
5 changes: 5 additions & 0 deletions changelog/unreleased/improve-posixfs.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Enhancement: Add trashbin support to posixfs alongside other improvements

We added support for trashbins to posixfs. Posixfs also saw a number of other improvement, bugfixes and optimizations.

https://github.com/cs3org/reva/pull/4779
2 changes: 1 addition & 1 deletion internal/http/services/owncloud/ocdav/ocdav.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ func (s *svc) Handler() http.Handler {

var head string
head, r.URL.Path = router.ShiftPath(r.URL.Path)
log.Debug().Str("head", head).Str("tail", r.URL.Path).Msg("http routing")
log.Debug().Str("method", r.Method).Str("head", head).Str("tail", r.URL.Path).Msg("http routing")
switch head {
case "status.php", "status":
s.doStatus(w, r)
Expand Down
15 changes: 14 additions & 1 deletion pkg/storage/fs/posix/blobstore/blobstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ func New(root string) (*Blobstore, error) {

// Upload stores some data in the blobstore under the given key
func (bs *Blobstore) Upload(node *node.Node, source string) error {
path := node.InternalPath()

// preserve the mtime of the file
fi, _ := os.Stat(path)

file, err := os.Open(source)
if err != nil {
return errors.Wrap(err, "Decomposedfs: oCIS blobstore: Can not open source file to upload")
Expand All @@ -59,7 +64,15 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error {
return errors.Wrapf(err, "could not write blob '%s'", node.InternalPath())
}

return w.Flush()
err = w.Flush()
if err != nil {
return err
}

if fi != nil {
return os.Chtimes(path, fi.ModTime(), fi.ModTime())
}
return nil
}

// Download retrieves a blob from the blobstore for reading
Expand Down
27 changes: 12 additions & 15 deletions pkg/storage/fs/posix/lookup/lookup.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,17 @@ type Lookup struct {
IDCache IDCache
metadataBackend metadata.Backend
userMapper usermapper.Mapper
tm node.TimeManager
}

// New returns a new Lookup instance
func New(b metadata.Backend, um usermapper.Mapper, o *options.Options) *Lookup {
func New(b metadata.Backend, um usermapper.Mapper, o *options.Options, tm node.TimeManager) *Lookup {
lu := &Lookup{
Options: o,
metadataBackend: b,
IDCache: NewStoreIDCache(&o.Options),
userMapper: um,
tm: tm,
}

return lu
Expand Down Expand Up @@ -122,22 +124,12 @@ func (lu *Lookup) MetadataBackend() metadata.Backend {
return lu.metadataBackend
}

// ReadBlobSizeAttr reads the blobsize from the xattrs
func (lu *Lookup) ReadBlobSizeAttr(ctx context.Context, path string) (int64, error) {
blobSize, err := lu.metadataBackend.GetInt64(ctx, path, prefixes.BlobsizeAttr)
func (lu *Lookup) ReadBlobIDAndSizeAttr(ctx context.Context, path string, _ node.Attributes) (string, int64, error) {
fi, err := os.Stat(path)
if err != nil {
return 0, errors.Wrapf(err, "error reading blobsize xattr")
return "", 0, errors.Wrap(err, "error stating file")
}
return blobSize, nil
}

// ReadBlobIDAttr reads the blobsize from the xattrs
func (lu *Lookup) ReadBlobIDAttr(ctx context.Context, path string) (string, error) {
attr, err := lu.metadataBackend.Get(ctx, path, prefixes.BlobIDAttr)
if err != nil {
return "", errors.Wrapf(err, "error reading blobid xattr")
}
return string(attr), nil
return "", fi.Size(), nil
}

// TypeFromPath returns the type of the node at the given path
Expand Down Expand Up @@ -421,3 +413,8 @@ func (lu *Lookup) GenerateSpaceID(spaceType string, owner *user.User) (string, e
return "", fmt.Errorf("unsupported space type: %s", spaceType)
}
}

// TimeManager returns the time manager
func (lu *Lookup) TimeManager() node.TimeManager {
return lu.tm
}
4 changes: 4 additions & 0 deletions pkg/storage/fs/posix/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
package options

import (
"time"

decomposedoptions "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
Expand All @@ -29,6 +31,8 @@ type Options struct {

UseSpaceGroups bool `mapstructure:"use_space_groups"`

ScanDebounceDelay time.Duration `mapstructure:"scan_debounce_delay"`

WatchFS bool `mapstructure:"watch_fs"`
WatchType string `mapstructure:"watch_type"`
WatchPath string `mapstructure:"watch_path"`
Expand Down
40 changes: 28 additions & 12 deletions pkg/storage/fs/posix/posix.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ import (
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/blobstore"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/options"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/trashbin"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree"
"github.com/cs3org/reva/v2/pkg/storage/fs/registry"
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs"
Expand Down Expand Up @@ -68,24 +70,39 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) {
return nil, err
}

bs, err := blobstore.New(o.Root)
if err != nil {
return nil, err
}

fs := &posixFS{}
um := usermapper.NewUnixMapper()

var lu *lookup.Lookup
switch o.MetadataBackend {
case "xattrs":
lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o)
lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{})
case "messagepack":
lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o)
lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{})
default:
return nil, fmt.Errorf("unknown metadata backend %s, only 'messagepack' or 'xattrs' (default) supported", o.MetadataBackend)
}

tp, err := tree.New(lu, bs, um, o, stream, store.Create(
trashbin, err := trashbin.New(o, lu)
if err != nil {
return nil, err
}
err = trashbin.Setup(fs)
if err != nil {
return nil, err
}

bs, err := blobstore.New(o.Root)
if err != nil {
return nil, err
}

switch o.IDCache.Store {
case "", "memory", "noop":
return nil, fmt.Errorf("the posix driver requires a shared id cache, e.g. nats-js-kv or redis")
}

tp, err := tree.New(lu, bs, um, trashbin, o, stream, store.Create(
store.Store(o.IDCache.Store),
store.TTL(o.IDCache.TTL),
store.Size(o.IDCache.Size),
Expand Down Expand Up @@ -113,6 +130,7 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) {
EventStream: stream,
UserMapper: um,
DisableVersioning: true,
Trashbin: trashbin,
}

dfs, err := decomposedfs.New(&o.Options, aspects)
Expand Down Expand Up @@ -154,10 +172,8 @@ func New(m map[string]interface{}, stream events.Stream) (storage.FS, error) {
}

mw := middleware.NewFS(dfs, hooks...)
fs := &posixFS{
FS: mw,
um: um,
}
fs.FS = mw
fs.um = um

return fs, nil
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/storage/fs/posix/posix_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ var _ = Describe("Posix", func() {
"root": tmpRoot,
"share_folder": "/Shares",
"permissionssvc": "any",
"idcache": map[string]interface{}{
"cache_store": "nats-js-kv",
},
}
})

Expand Down
14 changes: 11 additions & 3 deletions pkg/storage/fs/posix/testhelpers/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ import (
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/options"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/timemanager"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/trashbin"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/tree"
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs"
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/aspects"
Expand Down Expand Up @@ -152,12 +154,13 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) {
},
},
}

var lu *lookup.Lookup
switch o.MetadataBackend {
case "xattrs":
lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o)
lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{})
case "messagepack":
lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o)
lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), um, o, &timemanager.Manager{})
default:
return nil, fmt.Errorf("unknown metadata backend %s", o.MetadataBackend)
}
Expand All @@ -175,14 +178,19 @@ func NewTestEnv(config map[string]interface{}) (*TestEnv, error) {
)

bs := &treemocks.Blobstore{}
tree, err := tree.New(lu, bs, um, o, nil, store.Create())
tree, err := tree.New(lu, bs, um, &trashbin.Trashbin{}, o, nil, store.Create())
if err != nil {
return nil, err
}
tb, err := trashbin.New(o, lu)
if err != nil {
return nil, err
}
aspects := aspects.Aspects{
Lookup: lu,
Tree: tree,
Permissions: permissions.NewPermissions(pmock, permissionsSelector),
Trashbin: tb,
}
fs, err := decomposedfs.New(&o.Options, aspects)
if err != nil {
Expand Down
117 changes: 117 additions & 0 deletions pkg/storage/fs/posix/timemanager/timemanager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Copyright 2018-2024 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

package timemanager

import (
"context"
"os"
"syscall"
"time"

"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node"
)

// Manager is responsible for managing time-related operations on files and directories.
type Manager struct {
}

// OverrideMtime overrides the modification time (mtime) of a node with the specified time.
func (m *Manager) OverrideMtime(ctx context.Context, n *node.Node, _ *node.Attributes, mtime time.Time) error {
return os.Chtimes(n.InternalPath(), mtime, mtime)
}

// MTime returns the modification time (mtime) of a node.
func (m *Manager) MTime(ctx context.Context, n *node.Node) (time.Time, error) {
fi, err := os.Stat(n.InternalPath())
if err != nil {
return time.Time{}, err
}
return fi.ModTime(), nil
}

// SetMTime sets the modification time (mtime) of a node to the specified time.
func (m *Manager) SetMTime(ctx context.Context, n *node.Node, mtime *time.Time) error {
return os.Chtimes(n.InternalPath(), *mtime, *mtime)
}

// TMTime returns the tree modification time (tmtime) of a node.
// If the tmtime is not set, it falls back to the modification time (mtime).
func (m *Manager) TMTime(ctx context.Context, n *node.Node) (time.Time, error) {
b, err := n.XattrString(ctx, prefixes.TreeMTimeAttr)
if err == nil {
return time.Parse(time.RFC3339Nano, b)
}

// no tmtime, use mtime
return m.MTime(ctx, n)
}

// SetTMTime sets the tree modification time (tmtime) of a node to the specified time.
// If tmtime is nil, the tmtime attribute is removed.
func (m *Manager) SetTMTime(ctx context.Context, n *node.Node, tmtime *time.Time) error {
if tmtime == nil {
return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true)
}
return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, tmtime.UTC().Format(time.RFC3339Nano))
}

// CTime returns the creation time (ctime) of a node.
func (m *Manager) CTime(ctx context.Context, n *node.Node) (time.Time, error) {
fi, err := os.Stat(n.InternalPath())
if err != nil {
return time.Time{}, err
}

stat := fi.Sys().(*syscall.Stat_t)
//nolint:unconvert
return time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)), nil
}

// TCTime returns the tree creation time (tctime) of a node.
// Since decomposedfs does not differentiate between ctime and mtime, it falls back to TMTime.
func (m *Manager) TCTime(ctx context.Context, n *node.Node) (time.Time, error) {
// decomposedfs does not differentiate between ctime and mtime
return m.TMTime(ctx, n)
}

// SetTCTime sets the tree creation time (tctime) of a node to the specified time.
// Since decomposedfs does not differentiate between ctime and mtime, it falls back to SetTMTime.
func (m *Manager) SetTCTime(ctx context.Context, n *node.Node, tmtime *time.Time) error {
// decomposedfs does not differentiate between ctime and mtime
return m.SetTMTime(ctx, n, tmtime)
}

// DTime returns the deletion time (dtime) of a node.
func (m *Manager) DTime(ctx context.Context, n *node.Node) (tmTime time.Time, err error) {
b, err := n.XattrString(ctx, prefixes.DTimeAttr)
if err != nil {
return time.Time{}, err
}
return time.Parse(time.RFC3339Nano, b)
}

// SetDTime sets the deletion time (dtime) of a node to the specified time.
// If t is nil, the dtime attribute is removed.
func (m *Manager) SetDTime(ctx context.Context, n *node.Node, t *time.Time) (err error) {
if t == nil {
return n.RemoveXattr(ctx, prefixes.DTimeAttr, true)
}
return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano))
}
Loading

0 comments on commit cedd4ef

Please sign in to comment.