✨ Change cacher and locker implemention (#195)

* ✨ Add cacher

* ✨ Add locker

* ✨ Change locker and cacher

* ✅ Fix ci test

* 💚 Fix ci

cmd/distribution.go

@@ -31,7 +31,7 @@ import (
 var distributionCmd = &cobra.Command{
 	Use:     "distribution",
 	Aliases: []string{"ds"},
-	Short:   "Start the XImager distribution server",
+	Short:   "Start the sigma distribution server",
 	PersistentPreRun: func(_ *cobra.Command, _ []string) {
 		initConfig()
 		logger.SetLevel(viper.GetString("log.level"))

cmd/imports/cronjob.go

@@ -0,0 +1,19 @@
+// Copyright 2023 sigma
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package imports
+
+import (
+	_ "github.com/go-sigma/sigma/pkg/cronjob"
+)

cmd/imports/daemon.go

@@ -15,12 +15,9 @@
 package imports
 
 import (
-	_ "github.com/go-sigma/sigma/pkg/cronjob"
 	_ "github.com/go-sigma/sigma/pkg/daemon/builder"
 	_ "github.com/go-sigma/sigma/pkg/daemon/coderepo"
 	_ "github.com/go-sigma/sigma/pkg/daemon/gc"
 	_ "github.com/go-sigma/sigma/pkg/daemon/sbom"
 	_ "github.com/go-sigma/sigma/pkg/daemon/vulnerability"
-	_ "github.com/go-sigma/sigma/pkg/workq/database"
-	_ "github.com/go-sigma/sigma/pkg/workq/kafka"
 )

cmd/imports/modules.go

@@ -0,0 +1,22 @@
+// Copyright 2023 sigma
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package imports
+
+import (
+	_ "github.com/go-sigma/sigma/pkg/modules/locker/database"
+	_ "github.com/go-sigma/sigma/pkg/modules/locker/redis"
+	_ "github.com/go-sigma/sigma/pkg/modules/workq/database"
+	_ "github.com/go-sigma/sigma/pkg/modules/workq/kafka"
+)

conf/config-dev.yaml

@@ -3,6 +3,7 @@
 log:
   level: debug
   proxyLevel: info
+
 database:
   # The database type to use. Supported types are: sqlite3, mysql, postgresql
   type: mysql
@@ -32,15 +33,26 @@ redis:
   url: redis://:sigma@localhost:6379/0
 
 cache:
-  # the cache type available is: redis
-  type: redis
+  # the cache type available is: redis, inmemory, database
+  type: database
   # please attention in multi
-  inmemory: {}
-  redis: {}
+  inmemory:
+    size: 10240
+  redis:
+    ttl: 72h
+  database:
+    size: 10240
+    threshold: 0.2
 
 workqueue:
-  # the workqueue type available: redis
-  type: redis
+  # the workqueue type available: redis, kafka, database
+  type: database
+  redis: {}
+
+locker:
+  # the locker type available: redis, database
+  type: database
+  database: {}
   redis: {}
 
 namespace:

conf/config-full.yaml

@@ -3,6 +3,7 @@
 log:
   level: debug
   proxyLevel: info
+
 database:
   # The database type to use. Supported types are: sqlite3, mysql, postgresql
   type: sqlite3
@@ -32,13 +33,26 @@ redis:
   url: redis://:sigma@localhost:6379/0
 
 cache:
-  # the cache type available is: redis
+  # the cache type available is: redis, inmemory, database
   type: redis
+  inmemory:
+    size: 10240
+  redis:
+    ttl: 72h
+  database:
+    size: 10240
+    threshold: 0.2
 
 workqueue:
-  # the workqueue type available: redis
+  # the workqueue type available: redis, kafka, database
   type: redis
 
+locker:
+  # the locker type available: redis, database
+  type: database
+  database: {}
+  redis: {}
+
 namespace:
   # push image to registry, if namespace not exist, it will be created automatically
   autoCreate: false

conf/config.yaml

@@ -3,6 +3,7 @@
 log:
   level: debug
   proxyLevel: info
+
 database:
   # The database type to use. Supported types are: sqlite3, mysql, postgresql
   type: sqlite3
@@ -32,16 +33,29 @@ redis:
   url: redis://:sigma@localhost:6379/0
 
 cache:
-  # the cache type available is: redis
+  # the cache type available is: redis, inmemory, database
   type: redis
   # please attention in multi
-  inmemory: {}
-  redis: {}
+  inmemory:
+    size: 10240
+  redis:
+    ttl: 72h
+  database:
+    size: 10240
+    threshold: 0.2
 
 workqueue:
-  # the workqueue type available: redis
+  # the workqueue type available: redis, kafka, database
   type: redis
   redis: {}
+  kafka: {}
+  database: {}
+
+locker:
+  # the locker type available: redis, database
+  type: database
+  database: {}
+  redis: {}
 
 namespace:
   # push image to registry, if namespace not exist, it will be created automatically

go.mod

@@ -11,13 +11,13 @@ require (
 	github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible
 	github.com/anchore/syft v0.90.0
 	github.com/aquasecurity/trivy v0.45.1
-	github.com/aws/aws-sdk-go v1.45.11
+	github.com/aws/aws-sdk-go v1.45.12
 	github.com/bytedance/json v0.0.0-20190516032711-0d89175f1949
 	github.com/caarlos0/env/v9 v9.0.0
 	github.com/casbin/casbin/v2 v2.77.2
 	github.com/casbin/gorm-adapter/v3 v3.19.0
 	github.com/deckarep/golang-set/v2 v2.3.1
-	github.com/distribution/distribution/v3 v3.0.0-20230915083636-612ad42609c3
+	github.com/distribution/distribution/v3 v3.0.0-20230918070114-42ce5d4d51ca
 	github.com/distribution/reference v0.5.0
 	github.com/docker/cli v24.0.6+incompatible
 	github.com/docker/docker v24.0.6+incompatible
@@ -28,17 +28,20 @@ require (
 	github.com/go-playground/validator v9.31.0+incompatible
 	github.com/go-redsync/redsync/v4 v4.9.4
 	github.com/go-resty/resty/v2 v2.7.0
+	github.com/go-sql-driver/mysql v1.7.1
 	github.com/golang-jwt/jwt/v5 v5.0.0
 	github.com/golang-migrate/migrate/v4 v4.16.2
 	github.com/google/go-github/v53 v53.2.0
 	github.com/google/uuid v1.3.1
+	github.com/hashicorp/golang-lru/v2 v2.0.6
 	github.com/hibiken/asynq v0.24.1
 	github.com/jackc/pgx/v4 v4.18.1
 	github.com/jinzhu/copier v0.4.0
 	github.com/labstack/echo-contrib v0.15.0
 	github.com/labstack/echo/v4 v4.11.1
 	github.com/matoous/go-nanoid v1.5.0
 	github.com/matoous/go-nanoid/v2 v2.0.0
+	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/moby/buildkit v0.12.2
 	github.com/opencontainers/distribution-spec/specs-go v0.0.0-20230727214836-6bc87156eacf
 	github.com/opencontainers/go-digest v1.0.0
@@ -128,7 +131,6 @@ require (
 	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
@@ -184,7 +186,6 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/microsoft/go-mssqldb v1.5.0 // indirect

go.sum

@@ -130,8 +130,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/aws/aws-sdk-go v1.45.11 h1:8qiSrA12+NRr+2MVpMApi3JxtiFFjDVU1NeWe+80bYg=
-github.com/aws/aws-sdk-go v1.45.11/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.45.12 h1:+bKbbesGNPp+TeGrcqfrWuZoqcIEhjwKyBMHQPp80Jo=
+github.com/aws/aws-sdk-go v1.45.12/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/becheran/wildmatch-go v1.0.0 h1:mE3dGGkTmpKtT4Z+88t8RStG40yN9T+kFEGj2PZFSzA=
 github.com/becheran/wildmatch-go v1.0.0/go.mod h1:gbMvj0NtVdJ15Mg/mH9uxk2R1QCistMyU7d9KFzroX4=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -210,8 +210,8 @@ github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dhui/dktest v0.3.16 h1:i6gq2YQEtcrjKbeJpBkWjE8MmLZPYllcjOFbTZuPDnw=
-github.com/distribution/distribution/v3 v3.0.0-20230915083636-612ad42609c3 h1:NNbaD3nHlCm6c6trUVsYrQqDTsR8sDPaZKJOgZ6PFn8=
-github.com/distribution/distribution/v3 v3.0.0-20230915083636-612ad42609c3/go.mod h1:kwpPDTHXOnL7Ei4I8QaP0rOH/KP9SYIuXABAvv5jw0c=
+github.com/distribution/distribution/v3 v3.0.0-20230918070114-42ce5d4d51ca h1:gYakwCAlCMcSUcrdNRMI9vPAkJI8U5amLSt9ILAffEU=
+github.com/distribution/distribution/v3 v3.0.0-20230918070114-42ce5d4d51ca/go.mod h1:kwpPDTHXOnL7Ei4I8QaP0rOH/KP9SYIuXABAvv5jw0c=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
@@ -514,6 +514,8 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM=
+github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=

pkg/cmds/server/server.go

@@ -36,7 +36,9 @@ import (
 	"github.com/go-sigma/sigma/pkg/daemon"
 	"github.com/go-sigma/sigma/pkg/handlers"
 	"github.com/go-sigma/sigma/pkg/middlewares"
+	"github.com/go-sigma/sigma/pkg/modules/workq"
 	"github.com/go-sigma/sigma/pkg/storage"
+	"github.com/go-sigma/sigma/pkg/types/enums"
 	"github.com/go-sigma/sigma/pkg/utils/serializer"
 	"github.com/go-sigma/sigma/web"
 )
@@ -49,7 +51,7 @@ type ServerConfig struct {
 }
 
 // Serve starts the server
-func Serve(config ServerConfig) error {
+func Serve(serverConfig ServerConfig) error {
 	e := echo.New()
 	e.HideBanner = true
 	e.HidePort = true
@@ -77,10 +79,10 @@ func Serve(config ServerConfig) error {
 		pprof.Register(e)
 	}
 
-	if !config.WithoutDistribution {
+	if !serverConfig.WithoutDistribution {
 		handlers.InitializeDistribution(e)
 	}
-	if !config.WithoutWorker {
+	if !serverConfig.WithoutWorker {
 		err := builder.Initialize()
 		if err != nil {
 			return err
@@ -89,8 +91,16 @@ func Serve(config ServerConfig) error {
 		if err != nil {
 			return err
 		}
+		err = workq.Initialize(configs.Configuration{
+			WorkQueue: configs.ConfigurationWorkQueue{
+				Type: enums.WorkQueueTypeDatabase,
+			},
+		})
+		if err != nil {
+			return err
+		}
 	}
-	if !config.WithoutWeb {
+	if !serverConfig.WithoutWeb {
 		web.RegisterHandlers(e)
 	}
 

pkg/cmds/worker/worker.go

@@ -27,9 +27,12 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/go-sigma/sigma/pkg/builder"
+	"github.com/go-sigma/sigma/pkg/configs"
 	"github.com/go-sigma/sigma/pkg/consts"
 	"github.com/go-sigma/sigma/pkg/daemon"
 	"github.com/go-sigma/sigma/pkg/middlewares"
+	"github.com/go-sigma/sigma/pkg/modules/workq"
+	"github.com/go-sigma/sigma/pkg/types/enums"
 )
 
 // Worker is the worker initialization
@@ -44,6 +47,15 @@ func Worker() error {
 		return err
 	}
 
+	err = workq.Initialize(configs.Configuration{
+		WorkQueue: configs.ConfigurationWorkQueue{
+			Type: enums.WorkQueueTypeDatabase,
+		},
+	})
+	if err != nil {
+		return err
+	}
+
 	e := echo.New()
 	e.HideBanner = true
 	e.HidePort = true