From 88b42090beb5823acb6f0c6f9241dc680986e791 Mon Sep 17 00:00:00 2001 From: Andre Duffeck Date: Wed, 26 Apr 2023 15:44:18 +0200 Subject: [PATCH] Cache cleanup (#3820) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * More cache config cleanup * Add changelog * Apply suggestions from code review Co-authored-by: Jörn Friedrich Dreyer --------- Co-authored-by: Jörn Friedrich Dreyer --- changelog/unreleased/refactor-stores.md | 1 + internal/grpc/services/gateway/gateway.go | 48 ++++++++++++++----- .../services/owncloud/ocs/config/config.go | 12 ++--- .../handlers/apps/sharing/shares/shares.go | 2 +- .../apps/sharing/shares/shares_test.go | 2 +- internal/http/services/owncloud/ocs/ocs.go | 4 +- 6 files changed, 46 insertions(+), 23 deletions(-) diff --git a/changelog/unreleased/refactor-stores.md b/changelog/unreleased/refactor-stores.md index ae5a889ef83..4ea4f20b341 100644 --- a/changelog/unreleased/refactor-stores.md +++ b/changelog/unreleased/refactor-stores.md @@ -2,4 +2,5 @@ Change: Streamline stores We refactored and streamlined the different caches and stores. +https://github.com/cs3org/reva/pull/3820 https://github.com/cs3org/reva/pull/3777 diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index 44d6d9524d2..2dc2d44da5a 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -68,15 +68,21 @@ type config struct { DataTransfersFolder string `mapstructure:"data_transfers_folder"` TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"` AllowedUserAgents map[string][]string `mapstructure:"allowed_user_agents"` // map[path][]user-agent - CacheStore string `mapstructure:"cache_store"` - CacheNodes []string `mapstructure:"cache_nodes"` - CacheDatabase string `mapstructure:"cache_database"` + StatCacheStore string `mapstructure:"stat_cache_store"` + StatCacheNodes []string `mapstructure:"stat_cache_nodes"` + StatCacheDatabase string `mapstructure:"stat_cache_database"` + StatCacheTTL int `mapstructure:"stat_cache_ttl"` + StatCacheSize int `mapstructure:"stat_cache_size"` + CreateHomeCacheStore string `mapstructure:"create_home_cache_store"` + CreateHomeCacheNodes []string `mapstructure:"create_home_cache_nodes"` + CreateHomeCacheDatabase string `mapstructure:"create_home_cache_database"` CreateHomeCacheTTL int `mapstructure:"create_home_cache_ttl"` CreateHomeCacheSize int `mapstructure:"create_home_cache_size"` + ProviderCacheStore string `mapstructure:"provider_cache_store"` + ProviderCacheNodes []string `mapstructure:"provider_cache_nodes"` + ProviderCacheDatabase string `mapstructure:"provider_cache_database"` ProviderCacheTTL int `mapstructure:"provider_cache_ttl"` ProviderCacheSize int `mapstructure:"provider_cache_size"` - StatCacheTTL int `mapstructure:"stat_cache_ttl"` - StatCacheSize int `mapstructure:"stat_cache_size"` UseCommonSpaceRootShareLogic bool `mapstructure:"use_common_space_root_share_logic"` } @@ -124,12 +130,28 @@ func (c *config) init() { } // caching needs to be explicitly enabled - if c.CacheStore == "" { - c.CacheStore = "noop" + if c.StatCacheStore == "" { + c.StatCacheStore = "noop" + } + + if c.StatCacheDatabase == "" { + c.StatCacheDatabase = "reva" + } + + if c.ProviderCacheStore == "" { + c.ProviderCacheStore = "noop" + } + + if c.ProviderCacheDatabase == "" { + c.ProviderCacheDatabase = "reva" + } + + if c.CreateHomeCacheStore == "" { + c.CreateHomeCacheStore = "noop" } - if c.CacheDatabase == "" { - c.CacheDatabase = "reva" + if c.CreateHomeCacheDatabase == "" { + c.CreateHomeCacheDatabase = "reva" } } @@ -169,10 +191,10 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { c: c, dataGatewayURL: *u, tokenmgr: tokenManager, - statCache: cache.GetStatCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize), - providerCache: cache.GetProviderCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "provider", time.Duration(c.ProviderCacheTTL)*time.Second, c.ProviderCacheSize), - createHomeCache: cache.GetCreateHomeCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createHome", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize), - createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createPersonalSpace", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize), + statCache: cache.GetStatCache(c.StatCacheStore, c.StatCacheNodes, c.StatCacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize), + providerCache: cache.GetProviderCache(c.ProviderCacheStore, c.ProviderCacheNodes, c.ProviderCacheDatabase, "provider", time.Duration(c.ProviderCacheTTL)*time.Second, c.ProviderCacheSize), + createHomeCache: cache.GetCreateHomeCache(c.CreateHomeCacheStore, c.CreateHomeCacheNodes, c.CreateHomeCacheDatabase, "createHome", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize), + createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CreateHomeCacheStore, c.CreateHomeCacheNodes, c.CreateHomeCacheDatabase, "createPersonalSpace", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize), } return s, nil diff --git a/internal/http/services/owncloud/ocs/config/config.go b/internal/http/services/owncloud/ocs/config/config.go index 91da5fc04fd..9cb7883e685 100644 --- a/internal/http/services/owncloud/ocs/config/config.go +++ b/internal/http/services/owncloud/ocs/config/config.go @@ -37,12 +37,12 @@ type Config struct { AdditionalInfoAttribute string `mapstructure:"additional_info_attribute"` CacheWarmupDriver string `mapstructure:"cache_warmup_driver"` CacheWarmupDrivers map[string]map[string]interface{} `mapstructure:"cache_warmup_drivers"` - ResourceInfoCacheStore string `mapstructure:"resource_info_cache_store"` - ResourceInfoCacheNodes []string `mapstructure:"resource_info_cache_nodes"` - ResourceInfoCacheDatabase string `mapstructure:"resource_info_cache_database"` - ResourceInfoCacheTable string `mapstructure:"resource_info_cache_table"` - ResourceInfoCacheTTL int `mapstructure:"resource_info_cache_ttl"` - ResourceInfoCacheSize int `mapstructure:"resource_info_cache_size"` + StatCacheStore string `mapstructure:"stat_cache_store"` + StatCacheNodes []string `mapstructure:"stat_cache_nodes"` + StatCacheDatabase string `mapstructure:"stat_cache_database"` + StatCacheTable string `mapstructure:"stat_cache_table"` + StatCacheTTL int `mapstructure:"stat_cache_ttl"` + StatCacheSize int `mapstructure:"stat_cache_size"` UserIdentifierCacheTTL int `mapstructure:"user_identifier_cache_ttl"` MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"` SkipUpdatingExistingSharesMountpoints bool `mapstructure:"skip_updating_existing_shares_mountpoint"` diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index 51666b76497..41755087911 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -130,7 +130,7 @@ func (h *Handler) Init(c *config.Config) { h.deniable = c.EnableDenials h.resharing = resharing(c) - h.statCache = cache.GetStatCache(c.ResourceInfoCacheStore, c.ResourceInfoCacheNodes, c.ResourceInfoCacheDatabase, "stat", time.Duration(c.ResourceInfoCacheTTL)*time.Second, c.ResourceInfoCacheSize) + h.statCache = cache.GetStatCache(c.StatCacheStore, c.StatCacheNodes, c.StatCacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize) if c.CacheWarmupDriver != "" { cwm, err := getCacheWarmupManager(c) if err == nil { diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares_test.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares_test.go index 51e62e14fff..af17769e1e9 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares_test.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares_test.go @@ -75,7 +75,7 @@ var _ = Describe("The ocs API", func() { c := &config.Config{} c.GatewaySvc = "gatewaysvc" - c.ResourceInfoCacheDatabase = strconv.FormatInt(rand.Int63(), 10) // Use a fresh database for each test + c.StatCacheDatabase = strconv.FormatInt(rand.Int63(), 10) // Use a fresh database for each test c.Init() h.InitWithGetter(c, func() (gateway.GatewayAPIClient, error) { return client, nil diff --git a/internal/http/services/owncloud/ocs/ocs.go b/internal/http/services/owncloud/ocs/ocs.go index 016f493720a..152bddca4af 100644 --- a/internal/http/services/owncloud/ocs/ocs.go +++ b/internal/http/services/owncloud/ocs/ocs.go @@ -67,9 +67,9 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } - if conf.CacheWarmupDriver == "first-request" && conf.ResourceInfoCacheTTL > 0 { + if conf.CacheWarmupDriver == "first-request" && conf.StatCacheStore != "noop" { s.warmupCacheTracker = ttlcache.NewCache() - _ = s.warmupCacheTracker.SetTTL(time.Second * time.Duration(conf.ResourceInfoCacheTTL)) + _ = s.warmupCacheTracker.SetTTL(time.Second * time.Duration(conf.StatCacheTTL)) } return s, nil