Proxy accesstoken cache store

.drone.star

@@ -1148,6 +1148,7 @@ def e2eTests(ctx):
             "RETRY": "1",
             "WEB_UI_CONFIG": "%s/%s" % (dirs["base"], dirs["ocisConfig"]),
             "LOCAL_UPLOAD_DIR": "/uploads",
+            "SLOW_MO": "50",
         },
         "commands": [
             "cd %s" % dirs["web"],

changelog/unreleased/change-cache-configuration.md

@@ -0,0 +1,10 @@
+Change: Updated Cache Configuration
+
+We updated all cache related environment vars to more closely follow the go micro naming pattern:
+- `{service}_CACHE_STORE_TYPE` becomes `{service}_CACHE_STORE` or `{service}_PERSISTENT_STORE`
+- `{service}_CACHE_STORE_ADDRESS(ES)` becomes `{service}_CACHE_STORE_NODES`
+- The `mem` store implementation name changes to `memory`
+- In yaml files the cache `type` becomes `store`
+We introduced `redis-sentinel` as a store implementation.
+
+https://github.com/owncloud/ocis/pull/5829

go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/go-micro/plugins/v4/wrapper/monitoring/prometheus v1.2.0
 	github.com/go-micro/plugins/v4/wrapper/trace/opencensus v1.1.0
 	github.com/go-ozzo/ozzo-validation/v4 v4.3.0
+	github.com/go-redis/redis/v8 v8.11.5
 	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/golang/protobuf v1.5.3
@@ -69,6 +70,7 @@ require (
 	github.com/prometheus/client_golang v1.14.0
 	github.com/rogpeppe/go-internal v1.8.0
 	github.com/rs/zerolog v1.29.0
+	github.com/shamaton/msgpack/v2 v2.1.1
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.2
@@ -180,7 +182,6 @@ require (
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-redis/redis/v8 v8.11.5 // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
@@ -274,7 +275,6 @@ require (
 	github.com/sciencemesh/meshdirectory-web v1.0.4 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/sethvargo/go-password v0.2.0 // indirect
-	github.com/shamaton/msgpack/v2 v2.1.1 // indirect
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect

ocis-pkg/config/config.go

@@ -54,7 +54,7 @@ type Config struct {
 
 	Tracing        *shared.Tracing        `yaml:"tracing"`
 	Log            *shared.Log            `yaml:"log"`
-	CacheStore     *shared.CacheStore     `yaml:"cache_store"`
+	Cache          *shared.Cache          `yaml:"cache"`
 	GRPCClientTLS  *shared.GRPCClientTLS  `yaml:"grpc_client_tls"`
 	GRPCServiceTLS *shared.GRPCServiceTLS `yaml:"grpc_service_tls"`
 	HTTPServiceTLS shared.HTTPServiceTLS  `yaml:"http_service_tls"`

ocis-pkg/config/parser/parse.go

@@ -49,8 +49,8 @@ func EnsureDefaults(cfg *config.Config) {
 	if cfg.TokenManager == nil {
 		cfg.TokenManager = &shared.TokenManager{}
 	}
-	if cfg.CacheStore == nil {
-		cfg.CacheStore = &shared.CacheStore{}
+	if cfg.Cache == nil {
+		cfg.Cache = &shared.Cache{}
 	}
 	if cfg.GRPCClientTLS == nil {
 		cfg.GRPCClientTLS = &shared.GRPCClientTLS{}
@@ -70,7 +70,7 @@ func EnsureCommons(cfg *config.Config) {
 
 	cfg.Commons.Log = structs.CopyOrZeroValue(cfg.Log)
 	cfg.Commons.Tracing = structs.CopyOrZeroValue(cfg.Tracing)
-	cfg.Commons.CacheStore = structs.CopyOrZeroValue(cfg.CacheStore)
+	cfg.Commons.Cache = structs.CopyOrZeroValue(cfg.Cache)
 
 	if cfg.GRPCClientTLS != nil {
 		cfg.Commons.GRPCClientTLS = cfg.GRPCClientTLS

ocis-pkg/roles/manager.go

@@ -5,33 +5,33 @@ import (
 	"time"
 
 	"github.com/owncloud/ocis/v2/ocis-pkg/log"
-	ocisstore "github.com/owncloud/ocis/v2/ocis-pkg/store"
+	"github.com/owncloud/ocis/v2/ocis-pkg/store"
 	settingsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/settings/v0"
 	settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
-	"go-micro.dev/v4/store"
+	microstore "go-micro.dev/v4/store"
 	"google.golang.org/protobuf/encoding/protojson"
 )
 
 const (
 	cacheDatabase  = "ocis-pkg"
-	cacheTableName = "ocis-pkg/roles"
+	cacheTableName = "roles"
 	cacheTTL       = time.Hour
 )
 
 // Manager manages a cache of roles by fetching unknown roles from the settings.RoleService.
 type Manager struct {
 	logger      log.Logger
-	cache       store.Store
+	roleCache   microstore.Store
 	roleService settingssvc.RoleService
 }
 
 // NewManager returns a new instance of Manager.
 func NewManager(o ...Option) Manager {
 	opts := newOptions(o...)
 
-	nStore := ocisstore.Create(opts.storeOptions...)
+	nStore := store.Create(opts.storeOptions...)
 	return Manager{
-		cache:       nStore,
+		roleCache:   nStore,
 		roleService: opts.roleService,
 	}
 }
@@ -42,7 +42,7 @@ func (m *Manager) List(ctx context.Context, roleIDs []string) []*settingsmsg.Bun
 	result := make([]*settingsmsg.Bundle, 0)
 	lookup := make([]string, 0)
 	for _, roleID := range roleIDs {
-		if records, err := m.cache.Read(roleID, store.ReadFrom(cacheDatabase, cacheTableName)); err != nil {
+		if records, err := m.roleCache.Read(roleID, microstore.ReadFrom(cacheDatabase, cacheTableName)); err != nil {
 			lookup = append(lookup, roleID)
 		} else {
 			role := &settingsmsg.Bundle{}
@@ -77,15 +77,15 @@ func (m *Manager) List(ctx context.Context, roleIDs []string) []*settingsmsg.Bun
 		}
 		for _, role := range res.Bundles {
 			jsonbytes, _ := protojson.Marshal(role)
-			record := &store.Record{
+			record := &microstore.Record{
 				Key:    role.Id,
 				Value:  jsonbytes,
 				Expiry: cacheTTL,
 			}
-			err := m.cache.Write(
+			err := m.roleCache.Write(
 				record,
-				store.WriteTo(cacheDatabase, cacheTableName),
-				store.WriteTTL(cacheTTL),
+				microstore.WriteTo(cacheDatabase, cacheTableName),
+				microstore.WriteTTL(cacheTTL),
 			)
 			if err != nil {
 				m.logger.Debug().Err(err).Msg("failed to cache roles")

ocis-pkg/roles/option.go

@@ -2,13 +2,13 @@ package roles
 
 import (
 	"github.com/owncloud/ocis/v2/ocis-pkg/log"
-	ocisstore "github.com/owncloud/ocis/v2/ocis-pkg/store"
 	settingssvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/settings/v0"
+	"go-micro.dev/v4/store"
 )
 
 // Options are all the possible options.
 type Options struct {
-	storeOptions []ocisstore.Option
+	storeOptions []store.Option
 	logger       log.Logger
 	roleService  settingssvc.RoleService
 }
@@ -31,7 +31,7 @@ func RoleService(rs settingssvc.RoleService) Option {
 }
 
 // StoreOptions are the options for the store
-func StoreOptions(storeOpts []ocisstore.Option) Option {
+func StoreOptions(storeOpts []store.Option) Option {
 	return func(o *Options) {
 		o.storeOptions = storeOpts
 	}

ocis-pkg/shared/shared_types.go

@@ -1,5 +1,7 @@
 package shared
 
+import "time"
+
 // EnvBinding represents a direct binding from an env variable to a go kind. Along with gookit/config, its primal goal
 // is to unpack environment variables into a Go value. We do so with reflection, and this data structure is just a step
 // in between.
@@ -53,18 +55,21 @@ type HTTPServiceTLS struct {
 	Key  string `yaml:"key" env:"OCIS_HTTP_TLS_KEY" desc:"Path/File name for the TLS certificate key (in PEM format) for the server certificate to use for the http services."`
 }
 
-type CacheStore struct {
-	Type    string `yaml:"type" env:"OCIS_CACHE_STORE_TYPE" desc:"The type of the cache store. Valid options are \"noop\", \"ocmem\", \"etcd\" and \"memory\""`
-	Address string `yaml:"address" env:"OCIS_CACHE_STORE_ADDRESS" desc:"A comma-separated list of addresses to connect to. Only valid if the above setting is set to \"etcd\""`
-	Size    int    `yaml:"size" env:"OCIS_CACHE_STORE_SIZE" desc:"Maximum size for the cache store. Only ocmem will use this option, in number of items per table. The rest will ignore the option and can grow indefinitely"`
+type Cache struct {
+	Store    string        `yaml:"store" env:"OCIS_CACHE_STORE;OCIS_CACHE_STORE_TYPE" desc:"The type of the cache store. Supported values are: 'memory', 'ocmem', 'etcd', 'redis', 'redis-sentinel', 'nats-js', 'noop'. See the text description for details."`
+	Nodes    []string      `yaml:"nodes" env:"OCIS_CACHE_STORE_NODES;OCIS_CACHE_STORE_ADDRESSES" desc:"A comma separated list of nodes to access the configured store. This has no effect when 'in-memory' stores are configured. Note that the behaviour how nodes are used is dependent on the library of the configured store."`
+	Database string        `yaml:"database" env:"OCIS_CACHE_STORE_DATABASE" desc:"The database name the configured store should use."`
+	Table    string        `yaml:"table" env:"OCIS_CACHE_STORE_TABLE" desc:"The database table the store should use."`
+	TTL      time.Duration `yaml:"ttl" env:"OCIS_CACHE_STORE_TTL" desc:"Time to live for events in the store. The duration can be set as number followed by a unit identifier like s, m or h."`
+	Size     int           `yaml:"size" env:"OCIS_CACHE_STORE_SIZE" desc:"The maximum quantity of items in the store. Only applies when store type 'ocmem' is configured."`
 }
 
 // Commons holds configuration that are common to all extensions. Each extension can then decide whether
 // to overwrite its values.
 type Commons struct {
 	Log               *Log            `yaml:"log"`
 	Tracing           *Tracing        `yaml:"tracing"`
-	CacheStore        *CacheStore     `yaml:"cache_store"`
+	Cache             *Cache          `yaml:"cache"`
 	GRPCClientTLS     *GRPCClientTLS  `yaml:"grpc_client_tls"`
 	GRPCServiceTLS    *GRPCServiceTLS `yaml:"grpc_service_tls"`
 	HTTPServiceTLS    HTTPServiceTLS  `yaml:"http_service_tls"`