[cherry-pick] fix: support customize cache db for business (#19189)

fix: support customize cache db for business

Support to configure the customized redis db for cache layer and other
misc business for core, by default the behavior is same with
previous(stored in db 0).

Signed-off-by: chlins <chenyuzh@vmware.com>

make/harbor.yml.tmpl

@@ -174,6 +174,17 @@ _version: 2.7.0
 #     password: notary_server_db_password
 #     ssl_mode: disable
 
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+
 # Uncomment external_redis if using external Redis server
 # external_redis:
 #   # support redis, redis+sentinel
@@ -190,6 +201,10 @@ _version: 2.7.0
 #   chartmuseum_db_index: 3
 #   trivy_db_index: 5
 #   idle_timeout_seconds: 30
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
 
 # Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
 # uaa:

make/photon/prepare/migrations/version_2_7_0/harbor.yml.jinja

@@ -371,6 +371,49 @@ external_database:
 #     ssl_mode: disable
 {% endif %}
 
+{% if redis is defined %}
+redis:
+#   # db_index 0 is for core, it's unchangeable
+{% if redis.registry_db_index is defined %}
+  registry_db_index: {{ redis.registry_db_index }}
+{% else %}
+#   # registry_db_index: 1
+{% endif %}
+{% if redis.jobservice_db_index is defined %}
+  jobservice_db_index: {{ redis.jobservice_db_index }}
+{% else %}
+#   # jobservice_db_index: 2
+{% endif %}
+{% if redis.trivy_db_index is defined %}
+  trivy_db_index: {{ redis.trivy_db_index }}
+{% else %}
+#   # trivy_db_index: 5
+{% endif %}
+{% if redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+{% else %}
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+{% endif %}
+{% if redis.cache_layer_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+{% else %}
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+{% else %}
+# Uncomment redis if need to customize redis db
+# redis:
+#   # db_index 0 is for core, it's unchangeable
+#   # registry_db_index: 1
+#   # jobservice_db_index: 2
+#   # trivy_db_index: 5
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
+{% endif %}
+
 {% if external_redis is defined %}
 external_redis:
   # support redis, redis+sentinel
@@ -387,6 +430,18 @@ external_redis:
   chartmuseum_db_index: {{ external_redis.chartmuseum_db_index }}
   trivy_db_index: 5
   idle_timeout_seconds: 30
+  {% if external_redis.harbor_db_index is defined %}
+  harbor_db_index: {{ redis.harbor_db_index }}
+  {% else %}
+# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+# # harbor_db_index: 6
+  {% endif %}
+  {% if external_redis.cache_layer_db_index is defined %}
+  cache_layer_db_index: {{ redis.cache_layer_db_index }}
+  {% else %}
+# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+# # cache_layer_db_index: 7
+  {% endif %}
 {% else %}
 # Umcomments external_redis if using external Redis server
 # external_redis:
@@ -404,6 +459,10 @@ external_redis:
 #   chartmuseum_db_index: 3
 #   trivy_db_index: 5
 #   idle_timeout_seconds: 30
+#   # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
+#   # harbor_db_index: 6
+#   # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
+#   # cache_layer_db_index: 7
 {% endif %}
 
 {% if uaa is defined %}

make/photon/prepare/templates/core/env.jinja

@@ -1,6 +1,9 @@
 CONFIG_PATH=/etc/core/app.conf
 UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
 _REDIS_URL_CORE={{redis_url_core}}
+{% if redis_url_harbor %}
+_REDIS_URL_HARBOR={{redis_url_harbor}}
+{% endif %}
 SYNC_QUOTA=true
 CHART_CACHE_DRIVER={{chart_cache_driver}}
 _REDIS_URL_REG={{redis_url_reg}}
@@ -88,6 +91,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
 {% endif %}
 
 {% if cache.enabled %}
+{% if redis_url_cache_layer %}
+_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
+{% endif %}
 CACHE_ENABLED=true
 CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
 {% endif %}

make/photon/prepare/templates/jobservice/env.jinja

@@ -50,6 +50,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
 
 {% if cache.enabled %}
 _REDIS_URL_CORE={{redis_url_core}}
+{% if redis_url_cache_layer %}
+_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
+{% endif %}
 CACHE_ENABLED=true
 CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
 {% endif %}

make/photon/prepare/utils/configs.py

@@ -312,7 +312,7 @@ def parse_yaml_config(config_file_path, with_notary, with_trivy, with_chartmuseu
         config_dict['external_database'] = False
 
     # update redis configs
-    config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
+    config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy))
 
     # auto generated secret string for core
     config_dict['core_secret'] = generate_random_string(16)
@@ -407,7 +407,7 @@ def get_redis_url_param(redis=None):
     return ""
 
 
-def get_redis_configs(external_redis=None, with_trivy=True):
+def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True):
     """Returns configs for redis
 
     >>> get_redis_configs()['external_redis']
@@ -440,6 +440,8 @@ def get_redis_configs(external_redis=None, with_trivy=True):
     >>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
     True
     """
+
+    internal_redis = internal_redis or {}
     external_redis = external_redis or {}
 
     configs = dict(external_redis=bool(external_redis))
@@ -455,14 +457,23 @@ def get_redis_configs(external_redis=None, with_trivy=True):
         'idle_timeout_seconds': 30,
     }
 
-    # overwriting existing keys by external_redis
-    redis.update({key: value for (key, value) in external_redis.items() if value})
+    if len(internal_redis) > 0:
+        # overwriting existing keys by internal_redis
+        redis.update({key: value for (key, value) in internal_redis.items() if value})
+    else:
+        # overwriting existing keys by external_redis
+        redis.update({key: value for (key, value) in external_redis.items() if value})
 
     configs['redis_url_core'] = get_redis_url(0, redis)
     configs['redis_url_chart'] = get_redis_url(redis['chartmuseum_db_index'], redis)
     configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
     configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
 
+    if redis.get('harbor_db_index'):
+        configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis)
+    if redis.get('cache_layer_db_index'):
+        configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis)
+
     if with_trivy:
         configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)
 

src/core/main.go

@@ -124,25 +124,35 @@ func main() {
 	web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName
 	web.BConfig.MaxMemory = 1 << 35     // (32GB)
 	web.BConfig.MaxUploadSize = 1 << 35 // (32GB)
-
-	redisURL := os.Getenv("_REDIS_URL_CORE")
-	if len(redisURL) > 0 {
-		u, err := url.Parse(redisURL)
+	// the core db used for beego session
+	redisCoreURL := os.Getenv("_REDIS_URL_CORE")
+	if len(redisCoreURL) > 0 {
+		_, err := url.Parse(redisCoreURL)
 		if err != nil {
-			panic("bad _REDIS_URL")
+			panic("bad _REDIS_URL_CORE")
 		}
-
+		// configure the beego session redis
 		web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName
-		web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL
+		web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL
+	}
 
-		log.Info("initializing cache ...")
-		if err := cache.Initialize(u.Scheme, redisURL); err != nil {
-			log.Fatalf("failed to initialize cache: %v", err)
-		}
-		// when config/db init function is called, the cache is not ready,
-		// enable config cache explicitly when the cache is ready
-		dbCfg.EnableConfigCache()
+	log.Info("initializing cache ...")
+	// the harbor db used for harbor business, use core db if not specified
+	redisHarborURL := os.Getenv("_REDIS_URL_HARBOR")
+	if redisHarborURL == "" {
+		redisHarborURL = redisCoreURL
 	}
+	u, err := url.Parse(redisHarborURL)
+	if err != nil {
+		panic("bad _REDIS_URL_HARBOR")
+	}
+	if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil {
+		log.Fatalf("failed to initialize cache: %v", err)
+	}
+	// when config/db init function is called, the cache is not ready,
+	// enable config cache explicitly when the cache is ready
+	dbCfg.EnableConfigCache()
+
 	web.AddTemplateExt("htm")
 
 	log.Info("initializing configurations...")

src/jobservice/main.go

@@ -19,8 +19,6 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"net/url"
-	"os"
 
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/jobservice/common/utils"
@@ -29,7 +27,6 @@ import (
 	"github.com/goharbor/harbor/src/jobservice/job/impl"
 	"github.com/goharbor/harbor/src/jobservice/logger"
 	"github.com/goharbor/harbor/src/jobservice/runtime"
-	"github.com/goharbor/harbor/src/lib/cache"
 	cfgLib "github.com/goharbor/harbor/src/lib/config"
 	tracelib "github.com/goharbor/harbor/src/lib/trace"
 	_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
@@ -44,21 +41,6 @@ func main() {
 		panic(fmt.Sprintf("failed to load configuration, error: %v", err))
 	}
 
-	// init cache if cache layer enabled
-	// gc needs to delete artifact by artifact manager, but the artifact cache store in
-	// core redis db so here require core redis url and init default cache.
-	if cfgLib.CacheEnabled() {
-		cacheURL := os.Getenv("_REDIS_URL_CORE")
-		u, err := url.Parse(cacheURL)
-		if err != nil {
-			panic("bad _REDIS_URL_CORE")
-		}
-
-		if err = cache.Initialize(u.Scheme, cacheURL); err != nil {
-			panic(fmt.Sprintf("failed to initialize cache: %v", err))
-		}
-	}
-
 	// Get parameters
 	configPath := flag.String("c", "", "Specify the yaml config file path")
 	flag.Parse()

src/lib/cache/cache.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"os"
 	"sync"
 	"time"
 
@@ -137,3 +138,37 @@ func Initialize(typ, addr string) error {
 func Default() Cache {
 	return cache
 }
+
+var (
+	// cacheLayer is the global cache layer cache instance.
+	cacheLayer Cache
+	// cacheLayerOnce is the once condition for initializing instance.
+	cacheLayerOnce sync.Once
+)
+
+// LayerCache is the global cache instance for cache layer.
+func LayerCache() Cache {
+	// parse the redis url for cache layer, use the default cache if not specify
+	redisCacheURL := os.Getenv("_REDIS_URL_CACHE_LAYER")
+	if redisCacheURL == "" {
+		if cache != nil {
+			return cache
+		}
+		// use the core url if cache layer url not found
+		redisCacheURL = os.Getenv("_REDIS_URL_CORE")
+	}
+
+	u, err := url.Parse(redisCacheURL)
+	if err != nil {
+		log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE_LAYER")
+	}
+
+	cacheLayerOnce.Do(func() {
+		cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:"))
+		if err != nil {
+			log.Fatalf("failed to initialize cache for cache layer, err: %v", err)
+		}
+	})
+
+	return cacheLayer
+}

src/pkg/cached/base_manager.go

@@ -24,27 +24,27 @@ import (
 )
 
 // innerCache is the default cache client,
-// actually it is a wrapper for cache.Default().
+// actually it is a wrapper for cache.LayerCache().
 var innerCache cache.Cache = &cacheClient{}
 
-// cacheClient is a interceptor for cache.Default, in order to implement specific
+// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific
 // case for cache layer.
 type cacheClient struct{}
 
 func (*cacheClient) Contains(ctx context.Context, key string) bool {
-	return cache.Default().Contains(ctx, key)
+	return cache.LayerCache().Contains(ctx, key)
 }
 
 func (*cacheClient) Delete(ctx context.Context, key string) error {
-	return cache.Default().Delete(ctx, key)
+	return cache.LayerCache().Delete(ctx, key)
 }
 
 func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error {
-	return cache.Default().Fetch(ctx, key, value)
+	return cache.LayerCache().Fetch(ctx, key, value)
 }
 
 func (*cacheClient) Ping(ctx context.Context) error {
-	return cache.Default().Ping(ctx)
+	return cache.LayerCache().Ping(ctx)
 }
 
 func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error {
@@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp
 		return nil
 	}
 
-	return cache.Default().Save(ctx, key, value, expiration...)
+	return cache.LayerCache().Save(ctx, key, value, expiration...)
 }
 
 func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) {
-	return cache.Default().Scan(ctx, match)
+	return cache.LayerCache().Scan(ctx, match)
 }
 
 var _ Manager = &BaseManager{}