From 2a5caf74e80c37425e6f286bc7e50c1704fc6833 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 19 May 2023 14:42:43 +0200 Subject: [PATCH 1/2] idcache configuration Signed-off-by: jkoberg --- changelog/unreleased/id-cache.md | 5 ++++ services/storage-users/pkg/config/config.go | 10 +++++++ .../storage-users/pkg/revaconfig/drivers.go | 28 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 changelog/unreleased/id-cache.md diff --git a/changelog/unreleased/id-cache.md b/changelog/unreleased/id-cache.md new file mode 100644 index 00000000000..cf5e59dc93e --- /dev/null +++ b/changelog/unreleased/id-cache.md @@ -0,0 +1,5 @@ +Enhancement: Configurable ID Cache + +Makes the integrated idcache (used to reduce reads from disc) configurable with the general cache envvars + +https://github.com/owncloud/ocis/pull/6353 diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 25c7a4bdc42..721efd64a49 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -31,6 +31,7 @@ type Config struct { Events Events `yaml:"events"` StatCache StatCache `yaml:"stat_cache"` FilemetadataCache FilemetadataCache `yaml:"filemetadata_cache"` + IDCache IDCache `yaml:"id_cache"` MountID string `yaml:"mount_id" env:"STORAGE_USERS_MOUNT_ID" desc:"Mount ID of this storage."` ExposeDataServer bool `yaml:"expose_data_server" env:"STORAGE_USERS_EXPOSE_DATA_SERVER" desc:"Exposes the data server directly to users and bypasses the data gateway. Ensure that the data server address is reachable by users."` ReadOnly bool `yaml:"readonly" env:"STORAGE_USERS_READ_ONLY" desc:"Set this storage to be read-only."` @@ -188,6 +189,15 @@ type FilemetadataCache struct { Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_FILEMETADATA_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` } +// IDCache holds cache config +type IDCache struct { + Store string `yaml:"store" env:"OCIS_CACHE_STORE;STORAGE_USERS_ID_CACHE_STORE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."` + Nodes []string `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;STORAGE_USERS_ID_CACHE_STORE_NODES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'memory' or 'ocmem' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."` + Database string `yaml:"database" env:"OCIS_CACHE_DATABASE" desc:"The database name the configured store should use."` + TTL time.Duration `yaml:"ttl" env:"OCIS_CACHE_TTL;STORAGE_USERS_ID_CACHE_TTL" desc:"Default time to live for user info in the user info cache. Only applied when access tokens has no expiration. The duration can be set as number followed by a unit identifier like s, m or h. Defaults to '300s' (300 seconds)."` + Size int `yaml:"size" env:"OCIS_CACHE_SIZE;STORAGE_USERS_ID_CACHE_SIZE" desc:"The maximum quantity of items in the user info cache. Only applies when store type 'ocmem' is configured. Defaults to 512."` +} + // S3Driver is the storage driver configuration when using 's3' storage driver type S3Driver struct { // Root is the absolute path to the location of the data diff --git a/services/storage-users/pkg/revaconfig/drivers.go b/services/storage-users/pkg/revaconfig/drivers.go index d0e15203595..7fb10475788 100644 --- a/services/storage-users/pkg/revaconfig/drivers.go +++ b/services/storage-users/pkg/revaconfig/drivers.go @@ -144,6 +144,13 @@ func Ocis(cfg *config.Config) map[string]interface{} { "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, "cache_size": cfg.FilemetadataCache.Size, }, + "idcache": map[string]interface{}{ + "cache_store": cfg.IDCache.Store, + "cache_nodes": cfg.IDCache.Nodes, + "cache_database": cfg.IDCache.Database, + "cache_ttl": cfg.IDCache.TTL / time.Second, + "cache_size": cfg.IDCache.Size, + }, "events": map[string]interface{}{ "natsaddress": cfg.Events.Addr, "natsclusterid": cfg.Events.ClusterID, @@ -191,6 +198,13 @@ func OcisNoEvents(cfg *config.Config) map[string]interface{} { "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, "cache_size": cfg.FilemetadataCache.Size, }, + "idcache": map[string]interface{}{ + "cache_store": cfg.IDCache.Store, + "cache_nodes": cfg.IDCache.Nodes, + "cache_database": cfg.IDCache.Database, + "cache_ttl": cfg.IDCache.TTL / time.Second, + "cache_size": cfg.IDCache.Size, + }, } } @@ -243,6 +257,13 @@ func S3NG(cfg *config.Config) map[string]interface{} { "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, "cache_size": cfg.FilemetadataCache.Size, }, + "idcache": map[string]interface{}{ + "cache_store": cfg.IDCache.Store, + "cache_nodes": cfg.IDCache.Nodes, + "cache_database": cfg.IDCache.Database, + "cache_ttl": cfg.IDCache.TTL / time.Second, + "cache_size": cfg.IDCache.Size, + }, "events": map[string]interface{}{ "natsaddress": cfg.Events.Addr, "natsclusterid": cfg.Events.ClusterID, @@ -294,5 +315,12 @@ func S3NGNoEvents(cfg *config.Config) map[string]interface{} { "cache_ttl": cfg.FilemetadataCache.TTL / time.Second, "cache_size": cfg.FilemetadataCache.Size, }, + "idcache": map[string]interface{}{ + "cache_store": cfg.IDCache.Store, + "cache_nodes": cfg.IDCache.Nodes, + "cache_database": cfg.IDCache.Database, + "cache_ttl": cfg.IDCache.TTL / time.Second, + "cache_size": cfg.IDCache.Size, + }, } } From 4c2446e22d367a070dc30a767843d50f0f9f8fec Mon Sep 17 00:00:00 2001 From: jkoberg Date: Fri, 19 May 2023 14:37:28 +0200 Subject: [PATCH 2/2] bump reva Signed-off-by: jkoberg --- go.mod | 2 +- go.sum | 4 +- .../utils/decomposedfs/decomposedfs.go | 12 ++++- .../storage/utils/decomposedfs/node/xattrs.go | 14 +++++- .../utils/decomposedfs/options/options.go | 1 + .../storage/utils/decomposedfs/tree/tree.go | 44 +++++++++++++------ vendor/modules.txt | 2 +- 7 files changed, 58 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 8b0f0358ab4..ec0b5c44be3 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.4.0 github.com/cs3org/go-cs3apis v0.0.0-20221012090518-ef2996678965 - github.com/cs3org/reva/v2 v2.13.4-0.20230517171752-b39e78fd7b28 + github.com/cs3org/reva/v2 v2.13.4-0.20230522074943-fde10c4f3dea github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 diff --git a/go.sum b/go.sum index 1e0927a1178..2163c451e4b 100644 --- a/go.sum +++ b/go.sum @@ -627,8 +627,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4= github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc= github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA= -github.com/cs3org/reva/v2 v2.13.4-0.20230517171752-b39e78fd7b28 h1:y8oUHEWGXrs9/jdU0sG3zNRyoDBONyxCRJJfb9Nsbqw= -github.com/cs3org/reva/v2 v2.13.4-0.20230517171752-b39e78fd7b28/go.mod h1:MoymB39kU/myG7LFkaCwqtoXQHct+/8uoZAvJEmNi+I= +github.com/cs3org/reva/v2 v2.13.4-0.20230522074943-fde10c4f3dea h1:UDl1PP9Ydr/0NE1pXPzYSUJrmltFottU9QKcFgf4o40= +github.com/cs3org/reva/v2 v2.13.4-0.20230522074943-fde10c4f3dea/go.mod h1:MoymB39kU/myG7LFkaCwqtoXQHct+/8uoZAvJEmNi+I= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go index c065966b3a4..125b2f15070 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -56,9 +56,11 @@ import ( "github.com/cs3org/reva/v2/pkg/storage/utils/filelocks" "github.com/cs3org/reva/v2/pkg/storage/utils/templates" "github.com/cs3org/reva/v2/pkg/storagespace" + "github.com/cs3org/reva/v2/pkg/store" "github.com/cs3org/reva/v2/pkg/utils" "github.com/jellydator/ttlcache/v2" "github.com/pkg/errors" + microstore "go-micro.dev/v4/store" "golang.org/x/sync/errgroup" ) @@ -117,8 +119,14 @@ func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream) ( return nil, fmt.Errorf("unknown metadata backend %s, only 'messagepack' or 'xattrs' (default) supported", o.MetadataBackend) } - tp := tree.New(lu, bs, o) - + tp := tree.New(lu, bs, o, store.Create( + store.Store(o.IDCache.Store), + store.TTL(time.Duration(o.IDCache.TTL)*time.Second), + store.Size(o.IDCache.Size), + microstore.Nodes(o.IDCache.Nodes...), + microstore.Database(o.IDCache.Database), + microstore.Table(o.IDCache.Table), + )) permissionsClient, err := pool.GetPermissionsClient(o.PermissionsSVC, pool.WithTLSMode(o.PermTLSMode)) if err != nil { return nil, err diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go index 84565663187..3aa1eb3d137 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go @@ -88,6 +88,12 @@ func (n *Node) RemoveXattr(key string) error { // XattrsWithReader returns the extended attributes of the node. If the attributes have already // been cached they are not read from disk again. func (n *Node) XattrsWithReader(r io.Reader) (Attributes, error) { + if n.ID == "" { + // Do not try to read the attribute of an empty node. The InternalPath points to the + // base nodes directory in this case. + return Attributes{}, &xattr.Error{Op: "node.XattrsWithReader", Path: n.InternalPath(), Err: xattr.ENOATTR} + } + if n.xattrsCache != nil { return n.xattrsCache, nil } @@ -116,6 +122,12 @@ func (n *Node) Xattrs() (Attributes, error) { // Xattr returns an extended attribute of the node. If the attributes have already // been cached it is not read from disk again. func (n *Node) Xattr(key string) ([]byte, error) { + if n.ID == "" { + // Do not try to read the attribute of an empty node. The InternalPath points to the + // base nodes directory in this case. + return []byte{}, &xattr.Error{Op: "node.Xattr", Path: n.InternalPath(), Name: key, Err: xattr.ENOATTR} + } + if n.xattrsCache == nil { attrs, err := n.lu.MetadataBackend().All(n.InternalPath()) if err != nil { @@ -128,7 +140,7 @@ func (n *Node) Xattr(key string) ([]byte, error) { return val, nil } // wrap the error as xattr does - return []byte{}, &xattr.Error{Op: "xattr.get", Path: n.InternalPath(), Name: key, Err: xattr.ENOATTR} + return []byte{}, &xattr.Error{Op: "node.Xattr", Path: n.InternalPath(), Name: key, Err: xattr.ENOATTR} } // XattrString returns the string representation of an attribute diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options/options.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options/options.go index ae752b4fc48..f60c912a829 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options/options.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options/options.go @@ -69,6 +69,7 @@ type Options struct { StatCache cache.Config `mapstructure:"statcache"` FileMetadataCache cache.Config `mapstructure:"filemetadatacache"` + IDCache cache.Config `mapstructure:"idcache"` MaxAcquireLockCycles int `mapstructure:"max_acquire_lock_cycles"` LockCycleDurationFactor int `mapstructure:"lock_cycle_duration_factor"` diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go index 90fc96b0514..d092f52d655 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go @@ -32,7 +32,6 @@ import ( "strings" "time" - "github.com/bluele/gcache" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/appctx" "github.com/cs3org/reva/v2/pkg/errtypes" @@ -47,6 +46,7 @@ import ( "github.com/pkg/errors" "github.com/rogpeppe/go-internal/lockedfile" "github.com/rs/zerolog/log" + "go-micro.dev/v4/store" "golang.org/x/sync/errgroup" ) @@ -80,19 +80,19 @@ type Tree struct { options *options.Options - idCache gcache.Cache + idCache store.Store } // PermissionCheckFunc defined a function used to check resource permissions type PermissionCheckFunc func(rp *provider.ResourcePermissions) bool // New returns a new instance of Tree -func New(lu PathLookup, bs Blobstore, o *options.Options) *Tree { +func New(lu PathLookup, bs Blobstore, o *options.Options, cache store.Store) *Tree { return &Tree{ lookup: lu, blobstore: bs, options: o, - idCache: gcache.New(1_000_000).LRU().Build(), + idCache: cache, } } @@ -234,6 +234,9 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) } } + // remove cache entry in any case to avoid inconsistencies + defer func() { _ = t.idCache.Delete(filepath.Join(oldNode.ParentPath(), oldNode.Name)) }() + // Always target the old node ID for xattr updates. // The new node id is empty if the target does not exist // and we need to overwrite the new one when overwriting an existing path. @@ -271,7 +274,6 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) if err != nil { return errors.Wrap(err, "Decomposedfs: could not move child") } - t.idCache.Remove(filepath.Join(oldNode.ParentPath(), oldNode.Name)) // update target parentid and name attribs := node.Attributes{} @@ -361,16 +363,14 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro for i := 0; i < numWorkers; i++ { g.Go(func() error { for name := range work { - nodeID := "" path := filepath.Join(dir, name) - if val, err := t.idCache.Get(path); err == nil { - nodeID = val.(string) - } else { + nodeID := getNodeIDFromCache(path, t.idCache) + if nodeID == "" { nodeID, err = readChildNodeFromLink(path) if err != nil { return err } - err = t.idCache.Set(path, nodeID) + err = storeNodeIDInCache(path, nodeID, t.idCache) if err != nil { return err } @@ -416,6 +416,10 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro // Delete deletes a node in the tree by moving it to the trash func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { + path := filepath.Join(n.ParentPath(), n.Name) + // remove entry from cache immediately to avoid inconsistencies + defer func() { _ = t.idCache.Delete(path) }() + deletingSharedResource := ctx.Value(appctx.DeletingSharedResource) if deletingSharedResource != nil && deletingSharedResource.(bool) { @@ -492,10 +496,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { _ = os.Remove(n.LockFilePath()) // finally remove the entry from the parent dir - path := filepath.Join(n.ParentPath(), n.Name) - err = os.Remove(path) - t.idCache.Remove(path) - if err != nil { + if err = os.Remove(path); err != nil { // To roll back changes // TODO revert the rename // TODO remove symlink @@ -980,3 +981,18 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( return } + +func getNodeIDFromCache(path string, cache store.Store) string { + recs, err := cache.Read(path) + if err == nil && len(recs) > 0 { + return string(recs[0].Value) + } + return "" +} + +func storeNodeIDInCache(path string, nodeID string, cache store.Store) error { + return cache.Write(&store.Record{ + Key: path, + Value: []byte(nodeID), + }) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 3a63944c11f..fe0feb5ffaf 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -349,7 +349,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.13.4-0.20230517171752-b39e78fd7b28 +# github.com/cs3org/reva/v2 v2.13.4-0.20230522074943-fde10c4f3dea ## explicit; go 1.19 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime