Merge pull request #1 from xxxibgdrgnmm/refactor-repository

feat: refactor repository factory, add readme

.gitignore

@@ -0,0 +1,2 @@
+.idea/
+reverse-registry

Dockerfile

@@ -0,0 +1,8 @@
+FROM cgr.dev/chainguard/go AS builder
+COPY . /app
+RUN cd /app && CGO_ENABLED=0 GOOS=linux go build -o reverse-registry .
+
+FROM cgr.dev/chainguard/glibc-dynamic
+COPY --from=builder /app/reverse-registry /usr/bin/
+COPY ./config/config.local.yaml /etc/reverse-registry/config.local.yaml
+CMD ["/usr/bin/go-digester", "server", "--config=/etc/reverse-registry/config.local.yaml"]

README.md

@@ -0,0 +1,8 @@
+# Reverse registry
+
+
+This is a simple registry redirect service that redirect our.domain.com/* to cgr.dev/chainguard/*. It also run a background process to periodically hash image index to sha256 checksum and save to a local database (default in-mem sqlite is used. For production purpose, mysql is the recommended choice)
+
+Deploying
+
+[![Run on Google Cloud](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run)

app.json

@@ -0,0 +1,25 @@
+{
+    "name": "reverse-registry",
+    "env": {
+        "WORKER_FETCH_INTERVAL": {
+            "description": "specify interval to fetch manifest from cgr",
+            "value": "30s",
+            "required": true
+        }
+    },
+    "options": {
+        "allow-unauthenticated": false,
+        "memory": "512Mi",
+        "cpu": "1",
+        "port": 443,
+        "http2": false,
+        "concurrency": 80,
+        "max-instances": 1
+    },
+    "build": {
+        "skip": false,
+        "buildpacks": {
+            "builder": "gcr.io/buildpacks/builder:v1"
+        }
+    }
+}

app/app.go

@@ -58,11 +58,15 @@ func RunFetcher(conf config.Config) error {
 	if err != nil {
 		return err
 	}
+	d, err := time.ParseDuration(conf.WorkerFetchInterval)
+	if err != nil {
+		return err
+	}
 	fetcher := digestfetcher.New(digestfetcher.Options{
 		Storage:       storage,
 		Registry:      registryClient,
 		Log:           log,
-		FetchInterval: conf.WorkerFetchInterval * time.Second,
+		FetchInterval: d,
 	})
 	return fetcher.Fetch(conf.Images)
 }

cmd/root.go

@@ -28,6 +28,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/xxxibgdrgnmm/reverse-registry/config"
+	"github.com/xxxibgdrgnmm/reverse-registry/constant"
 )
 
 var cfgFile string
@@ -92,5 +93,13 @@ func initConfig() {
 		if err = viper.Unmarshal(&c); err != nil {
 			panic(fmt.Sprintf("can not marshal config %v", err))
 		}
+		wF := os.Getenv(constant.WorkerFetchIntervalEnv)
+		mP := os.Getenv(constant.MySQLPassWordEnv)
+		if wF != "" {
+			c.WorkerFetchInterval = wF
+		}
+		if mP != "" {
+			c.DBConfig.Password = mP
+		}
 	}
 }

config/config.go

@@ -1,7 +1,5 @@
 package config
 
-import "time"
-
 type MysqlConfig struct {
 	Host     string `mapstructure:"host"`
 	User     string `mapstructure:"user"`
@@ -10,9 +8,10 @@ type MysqlConfig struct {
 }
 
 type Config struct {
-	DBConfig            MysqlConfig   `mapstructure:"dbConfig"`
-	Images              []Image       `mapstructure:"images"`
-	WorkerFetchInterval time.Duration `mapstructure:"workerFetchInterval"`
+	DB                  string      `mapstructure:"db"`
+	DBConfig            MysqlConfig `mapstructure:"dbConfig"`
+	Images              []Image     `mapstructure:"images"`
+	WorkerFetchInterval string      `mapstructure:"workerFetchInterval"`
 }
 
 type Image struct {

config/config.local.yaml

@@ -1,3 +1,4 @@
+db: sqlite
 dbConfig:
   host: localhost
   user: root

constant/const.go

@@ -0,0 +1,6 @@
+package constant
+
+const (
+	WorkerFetchIntervalEnv = "WORKER_FETCH_INTERVAL"
+	MySQLPassWordEnv       = "MYSQL_PASSWORD"
+)

driver/mysql_driver.go

@@ -15,9 +15,6 @@ func NewMySQLDB(host string, user string, password string, dbName string) (*gorm
 		Logger: logger.Default.LogMode(logger.Silent),
 	})
 	if err != nil {
-		msg := fmt.Sprintf("cannot connect to database. host: %s, user: %s, db: %s", host, user, dbName)
-		fmt.Println(msg)
-
 		return nil, err
 	}
 

driver/sqlte_driver.go

@@ -0,0 +1,21 @@
+package driver
+
+import (
+	"github.com/xxxibgdrgnmm/reverse-registry/model"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+func NewSqliteDB() (*gorm.DB, error) {
+	db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		return nil, err
+	}
+	db.AutoMigrate(
+		&model.ImageModel{},
+	)
+	return db, nil
+}

go.mod

@@ -14,10 +14,13 @@ require (
 	github.com/test-go/testify v1.1.4
 	golang.org/x/sync v0.3.0
 	gorm.io/driver/mysql v1.5.1
-	gorm.io/gorm v1.25.1
+	gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55
 )
 
-require github.com/sigstore/cosign/v2 v2.0.3-0.20230619102641-b0072d56686b
+require (
+	github.com/sigstore/cosign/v2 v2.0.3-0.20230619102641-b0072d56686b
+	gorm.io/driver/sqlite v1.5.2
+)
 
 require (
 	cloud.google.com/go/compute v1.19.3 // indirect
@@ -129,6 +132,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect

go.sum

@@ -594,6 +594,8 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
+github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -1268,8 +1270,11 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gorm.io/driver/mysql v1.5.1 h1:WUEH5VF9obL/lTtzjmML/5e6VfFR/788coz2uaVCAZw=
 gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5o=
-gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64=
+gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc=
+gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
 gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
+gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 h1:sC1Xj4TYrLqg1n3AN10w871An7wJM0gzgcm8jkIkECQ=
+gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

handler/handler.go

@@ -10,7 +10,7 @@ import (
 	"github.com/docker/distribution/reference"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
-	repository "github.com/xxxibgdrgnmm/reverse-registry/repository/storage"
+	"github.com/xxxibgdrgnmm/reverse-registry/repository"
 	containerregistry "github.com/xxxibgdrgnmm/reverse-registry/services/container-registry"
 	"github.com/xxxibgdrgnmm/reverse-registry/utils"
 )

inject/inject.go

@@ -5,31 +5,38 @@ import (
 
 	"github.com/xxxibgdrgnmm/reverse-registry/config"
 	"github.com/xxxibgdrgnmm/reverse-registry/driver"
-	repository "github.com/xxxibgdrgnmm/reverse-registry/repository/storage"
-	"github.com/xxxibgdrgnmm/reverse-registry/repository/storage/mysql"
+	"github.com/xxxibgdrgnmm/reverse-registry/repository"
 	containerregistry "github.com/xxxibgdrgnmm/reverse-registry/services/container-registry"
 )
 
-var imageMySQLStorage *mysql.MySQLStorage
-var muImageMySQLStorage sync.Mutex
+var imageStorage *repository.Storage
+var muImageStorage sync.Mutex
 
 func GetStorage(conf config.Config) (repository.Interface, error) {
-	muImageMySQLStorage.Lock()
-	defer muImageMySQLStorage.Unlock()
-	if imageMySQLStorage != nil {
-		return imageMySQLStorage, nil
+	muImageStorage.Lock()
+	defer muImageStorage.Unlock()
+	if imageStorage != nil {
+		return imageStorage, nil
 	}
 	dbConfig := conf.DBConfig
 	host := dbConfig.Host
 	user := dbConfig.User
 	password := dbConfig.Password
 	dbName := dbConfig.DBName
-	db, err := driver.NewMySQLDB(host, user, password, dbName)
+	if conf.DB == "mysql" {
+		db, err := driver.NewMySQLDB(host, user, password, dbName)
+		if err != nil {
+			return nil, err
+		}
+		imageStorage := repository.NewStorage(db)
+		return imageStorage, nil
+	}
+	db, err := driver.NewSqliteDB()
 	if err != nil {
 		return nil, err
 	}
-	imageMySQL := mysql.NewMySQLStorage(db)
-	return imageMySQL, nil
+	imageStorage := repository.NewStorage(db)
+	return imageStorage, nil
 }
 
 var registryClient *containerregistry.Client

repository/storage/mysql/repo_impl.go → repository/repo_impl.go

@@ -1,22 +1,21 @@
-package mysql
+package repository
 
 import (
 	"github.com/xxxibgdrgnmm/reverse-registry/model"
-	repository "github.com/xxxibgdrgnmm/reverse-registry/repository/storage"
 	"gorm.io/gorm"
 )
 
-type MySQLStorage struct {
+type Storage struct {
 	db *gorm.DB
 }
 
-func NewMySQLStorage(db *gorm.DB) repository.Interface {
-	return &MySQLStorage{
+func NewStorage(db *gorm.DB) Interface {
+	return &Storage{
 		db,
 	}
 }
 
-func (s *MySQLStorage) FindByNameTag(nameWithTag string) (*model.ImageModel, error) {
+func (s *Storage) FindByNameTag(nameWithTag string) (*model.ImageModel, error) {
 	var iM model.ImageModel
 	query := s.db.Model(&model.ImageModel{})
 	query = query.Where("name=?", nameWithTag)
@@ -27,7 +26,7 @@ func (s *MySQLStorage) FindByNameTag(nameWithTag string) (*model.ImageModel, err
 	return &iM, nil
 }
 
-func (s *MySQLStorage) FindByDigest(hashedIndex string) (*model.ImageModel, error) {
+func (s *Storage) FindByDigest(hashedIndex string) (*model.ImageModel, error) {
 	var iM model.ImageModel
 	query := s.db.Model(&model.ImageModel{})
 	query = query.Where("hashed_index=?", hashedIndex)
@@ -38,7 +37,7 @@ func (s *MySQLStorage) FindByDigest(hashedIndex string) (*model.ImageModel, erro
 	return &iM, nil
 }
 
-func (s *MySQLStorage) SaveDigest(nameWithTag string, hashedIndex string) error {
+func (s *Storage) SaveDigest(nameWithTag string, hashedIndex string) error {
 	var iM model.ImageModel
 	iM.Name = nameWithTag
 	iM.HashedIndex = hashedIndex

repository/storage/mysql/repo_impl_test.go → repository/repo_impl_test.go

@@ -1,4 +1,4 @@
-package mysql
+package repository
 
 import (
 	"testing"
@@ -10,7 +10,7 @@ import (
 func TestRepoImpl(t *testing.T) {
 	db, err := driver.NewMySQLDB("localhost", "root", "my-secret-pw", "test")
 	assert.NoError(t, err)
-	imageModelStorage := NewMySQLStorage(db)
+	imageModelStorage := NewStorage(db)
 	err = imageModelStorage.SaveDigest("172.20.10.2:8080/nginx:1.25.1-r0", "sha256:81bed54c9e507503766c0f8f030f869705dae486f37c2a003bb5b12bcfcc713f")
 	assert.NoError(t, err)
 	res, err := imageModelStorage.FindByNameTag("172.20.10.2:8080/nginx:1.25.1-r0")

services/digest-fetcher/digest_fetcher.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/xxxibgdrgnmm/reverse-registry/config"
-	repository "github.com/xxxibgdrgnmm/reverse-registry/repository/storage"
+	repository "github.com/xxxibgdrgnmm/reverse-registry/repository"
 	containerregistry "github.com/xxxibgdrgnmm/reverse-registry/services/container-registry"
 	"github.com/xxxibgdrgnmm/reverse-registry/utils"
 )
@@ -91,7 +91,7 @@ func (c *client) Fetch(images []config.Image) error {
 						if err != nil {
 							c.log.Errorf("hash index %v", err)
 						}
-						if err := c.storage.SaveDigest(img, "sha256:"+fmt.Sprintf("digest: %x\n", digest)); err != nil {
+						if err := c.storage.SaveDigest(img, "sha256:"+fmt.Sprintf("%x", digest)); err != nil {
 							c.log.Errorf("save digest to db %v", err)
 							break
 						}

services/digest-fetcher/digetst_fetcher_test.go

@@ -18,6 +18,7 @@ func TestFetch(t *testing.T) {
 		Constraint:  "^1.2.*",
 		MainPackage: "nginx",
 	})
+
 	conf := config.Config{
 		DBConfig: config.MysqlConfig{
 			Host:     "localhost",
@@ -26,17 +27,19 @@ func TestFetch(t *testing.T) {
 			DBName:   "test",
 		},
 		Images:              images,
-		WorkerFetchInterval: 10,
+		WorkerFetchInterval: "10s",
 	}
+	d, err := time.ParseDuration(conf.WorkerFetchInterval)
 	storage, err := inject.GetStorage(conf)
 	assert.NoError(t, err)
 	registryClient, err := inject.GetContainerRegistryClient()
 	assert.NoError(t, err)
+	assert.NoError(t, err)
 	fetcher := New(Options{
 		Storage:       storage,
 		Registry:      registryClient,
 		Log:           log,
-		FetchInterval: conf.WorkerFetchInterval * time.Second,
+		FetchInterval: d,
 	})
 	fetcher.Fetch(conf.Images)
 }