Skip to content

Commit

Permalink
Cache cleanup (cs3org#3820)
Browse files Browse the repository at this point in the history 
* More cache config cleanup

* Add changelog

* Apply suggestions from code review

Co-authored-by: Jörn Friedrich Dreyer <jfd@butonic.de>

---------

Co-authored-by: Jörn Friedrich Dreyer <jfd@butonic.de>
  • Loading branch information
@aduffeck @butonic @2403905
2 people authored and 2403905 committed May 19, 2023
1 parent 2b90336 commit 88b4209
Show file tree
Hide file tree
Showing 6 changed files with 46 additions and 23 deletions.
1 change: 1 addition & 0 deletions changelog/unreleased/refactor-stores.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@ Change: Streamline stores

We refactored and streamlined the different caches and stores.

https://github.com/cs3org/reva/pull/3820
https://github.com/cs3org/reva/pull/3777
48 changes: 35 additions & 13 deletions internal/grpc/services/gateway/gateway.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,15 +68,21 @@ type config struct {
DataTransfersFolder string `mapstructure:"data_transfers_folder"`
TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"`
AllowedUserAgents map[string][]string `mapstructure:"allowed_user_agents"` // map[path][]user-agent
CacheStore string `mapstructure:"cache_store"`
CacheNodes []string `mapstructure:"cache_nodes"`
CacheDatabase string `mapstructure:"cache_database"`
StatCacheStore string `mapstructure:"stat_cache_store"`
StatCacheNodes []string `mapstructure:"stat_cache_nodes"`
StatCacheDatabase string `mapstructure:"stat_cache_database"`
StatCacheTTL int `mapstructure:"stat_cache_ttl"`
StatCacheSize int `mapstructure:"stat_cache_size"`
CreateHomeCacheStore string `mapstructure:"create_home_cache_store"`
CreateHomeCacheNodes []string `mapstructure:"create_home_cache_nodes"`
CreateHomeCacheDatabase string `mapstructure:"create_home_cache_database"`
CreateHomeCacheTTL int `mapstructure:"create_home_cache_ttl"`
CreateHomeCacheSize int `mapstructure:"create_home_cache_size"`
ProviderCacheStore string `mapstructure:"provider_cache_store"`
ProviderCacheNodes []string `mapstructure:"provider_cache_nodes"`
ProviderCacheDatabase string `mapstructure:"provider_cache_database"`
ProviderCacheTTL int `mapstructure:"provider_cache_ttl"`
ProviderCacheSize int `mapstructure:"provider_cache_size"`
StatCacheTTL int `mapstructure:"stat_cache_ttl"`
StatCacheSize int `mapstructure:"stat_cache_size"`
UseCommonSpaceRootShareLogic bool `mapstructure:"use_common_space_root_share_logic"`
}

Expand Down Expand Up @@ -124,12 +130,28 @@ func (c *config) init() {
}

// caching needs to be explicitly enabled
if c.CacheStore == "" {
c.CacheStore = "noop"
if c.StatCacheStore == "" {
c.StatCacheStore = "noop"
}

if c.StatCacheDatabase == "" {
c.StatCacheDatabase = "reva"
}

if c.ProviderCacheStore == "" {
c.ProviderCacheStore = "noop"
}

if c.ProviderCacheDatabase == "" {
c.ProviderCacheDatabase = "reva"
}

if c.CreateHomeCacheStore == "" {
c.CreateHomeCacheStore = "noop"
}

if c.CacheDatabase == "" {
c.CacheDatabase = "reva"
if c.CreateHomeCacheDatabase == "" {
c.CreateHomeCacheDatabase = "reva"
}
}

Expand Down Expand Up @@ -169,10 +191,10 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
c: c,
dataGatewayURL: *u,
tokenmgr: tokenManager,
statCache: cache.GetStatCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize),
providerCache: cache.GetProviderCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "provider", time.Duration(c.ProviderCacheTTL)*time.Second, c.ProviderCacheSize),
createHomeCache: cache.GetCreateHomeCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createHome", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize),
createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CacheStore, c.CacheNodes, c.CacheDatabase, "createPersonalSpace", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize),
statCache: cache.GetStatCache(c.StatCacheStore, c.StatCacheNodes, c.StatCacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize),
providerCache: cache.GetProviderCache(c.ProviderCacheStore, c.ProviderCacheNodes, c.ProviderCacheDatabase, "provider", time.Duration(c.ProviderCacheTTL)*time.Second, c.ProviderCacheSize),
createHomeCache: cache.GetCreateHomeCache(c.CreateHomeCacheStore, c.CreateHomeCacheNodes, c.CreateHomeCacheDatabase, "createHome", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize),
createPersonalSpaceCache: cache.GetCreatePersonalSpaceCache(c.CreateHomeCacheStore, c.CreateHomeCacheNodes, c.CreateHomeCacheDatabase, "createPersonalSpace", time.Duration(c.CreateHomeCacheTTL)*time.Second, c.CreateHomeCacheSize),
}

return s, nil
Expand Down
12 changes: 6 additions & 6 deletions internal/http/services/owncloud/ocs/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,12 @@ type Config struct {
AdditionalInfoAttribute string `mapstructure:"additional_info_attribute"`
CacheWarmupDriver string `mapstructure:"cache_warmup_driver"`
CacheWarmupDrivers map[string]map[string]interface{} `mapstructure:"cache_warmup_drivers"`
ResourceInfoCacheStore string `mapstructure:"resource_info_cache_store"`
ResourceInfoCacheNodes []string `mapstructure:"resource_info_cache_nodes"`
ResourceInfoCacheDatabase string `mapstructure:"resource_info_cache_database"`
ResourceInfoCacheTable string `mapstructure:"resource_info_cache_table"`
ResourceInfoCacheTTL int `mapstructure:"resource_info_cache_ttl"`
ResourceInfoCacheSize int `mapstructure:"resource_info_cache_size"`
StatCacheStore string `mapstructure:"stat_cache_store"`
StatCacheNodes []string `mapstructure:"stat_cache_nodes"`
StatCacheDatabase string `mapstructure:"stat_cache_database"`
StatCacheTable string `mapstructure:"stat_cache_table"`
StatCacheTTL int `mapstructure:"stat_cache_ttl"`
StatCacheSize int `mapstructure:"stat_cache_size"`
UserIdentifierCacheTTL int `mapstructure:"user_identifier_cache_ttl"`
MachineAuthAPIKey string `mapstructure:"machine_auth_apikey"`
SkipUpdatingExistingSharesMountpoints bool `mapstructure:"skip_updating_existing_shares_mountpoint"`
Expand Down
2 changes: 1 addition & 1 deletion internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (h *Handler) Init(c *config.Config) {
h.deniable = c.EnableDenials
h.resharing = resharing(c)

h.statCache = cache.GetStatCache(c.ResourceInfoCacheStore, c.ResourceInfoCacheNodes, c.ResourceInfoCacheDatabase, "stat", time.Duration(c.ResourceInfoCacheTTL)*time.Second, c.ResourceInfoCacheSize)
h.statCache = cache.GetStatCache(c.StatCacheStore, c.StatCacheNodes, c.StatCacheDatabase, "stat", time.Duration(c.StatCacheTTL)*time.Second, c.StatCacheSize)
if c.CacheWarmupDriver != "" {
cwm, err := getCacheWarmupManager(c)
if err == nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ var _ = Describe("The ocs API", func() {

c := &config.Config{}
c.GatewaySvc = "gatewaysvc"
c.ResourceInfoCacheDatabase = strconv.FormatInt(rand.Int63(), 10) // Use a fresh database for each test
c.StatCacheDatabase = strconv.FormatInt(rand.Int63(), 10) // Use a fresh database for each test
c.Init()
h.InitWithGetter(c, func() (gateway.GatewayAPIClient, error) {
return client, nil
Expand Down
4 changes: 2 additions & 2 deletions internal/http/services/owncloud/ocs/ocs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,9 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error)
return nil, err
}

if conf.CacheWarmupDriver == "first-request" && conf.ResourceInfoCacheTTL > 0 {
if conf.CacheWarmupDriver == "first-request" && conf.StatCacheStore != "noop" {
s.warmupCacheTracker = ttlcache.NewCache()
_ = s.warmupCacheTracker.SetTTL(time.Second * time.Duration(conf.ResourceInfoCacheTTL))
_ = s.warmupCacheTracker.SetTTL(time.Second * time.Duration(conf.StatCacheTTL))
}

return s, nil
Expand Down

0 comments on commit 88b4209

Please sign in to comment.