Merge branch 'main' into main

Signed-off-by: Tosone <i@tosone.cn>

conf/config-dev.yaml

@@ -78,6 +78,8 @@ http:
   # if http.tls.enabled is true, internalEndpoint should start with https://
   # eg: http://sigma.test.io, http://sigma.test.io:3000, https://sigma.test.io:30080
   internalEndpoint: http://192.168.31.198:3000
+  # eg: http://sigma-distribution:3000
+  internalDistributionEndpoint:
   tls:
     enabled: false
     certificate: ./conf/sigma.test.io.crt

conf/config-full.yaml

@@ -77,6 +77,8 @@ http:
   # if http.tls.enabled is true, internalEndpoint should start with https://
   # eg: http://sigma.test.io, http://sigma.test.io:3000, https://sigma.test.io:30080
   internalEndpoint:
+  # eg: http://sigma-distribution:3000
+  internalDistributionEndpoint:
   tls:
     enabled: false
     certificate: ./conf/sigma.test.io.crt

conf/config.yaml

@@ -80,10 +80,13 @@ http:
   # if http.tls.enabled is true, internalEndpoint should start with https://
   # eg: http://sigma.test.io, http://sigma.test.io:3000, https://sigma.test.io:30080
   internalEndpoint:
+  # eg: http://sigma-distribution:3000
+  internalDistributionEndpoint:
   tls:
     enabled: false
     certificate: ./conf/sigma.test.io.crt
     key: ./conf/sigma.test.io.key
+
 storage:
   rootdirectory: ./storage
   type: filesystem

go.mod

@@ -48,6 +48,7 @@ require (
 	github.com/moby/buildkit v0.12.2
 	github.com/opencontainers/distribution-spec/specs-go v0.0.0-20231016131659-3940529fe6c0
 	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.1.0-rc5
 	github.com/redis/go-redis/v9 v9.2.1
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/rs/zerolog v1.31.0
@@ -218,7 +219,6 @@ require (
 	github.com/mozillazg/go-httpheader v0.4.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
 	github.com/opencontainers/runc v1.1.7 // indirect
 	github.com/opencontainers/runtime-spec v1.1.0-rc.2 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect

pkg/configs/configuration.go

@@ -165,9 +165,10 @@ type ConfigurationHttpTLS struct {
 
 // ConfigurationHTTP ...
 type ConfigurationHTTP struct {
-	Endpoint         string               `yaml:"endpoint"`
-	InternalEndpoint string               `yaml:"internalEndpoint"`
-	TLS              ConfigurationHttpTLS `yaml:"tls"`
+	Endpoint                     string               `yaml:"endpoint"`
+	InternalEndpoint             string               `yaml:"internalEndpoint"`
+	InternalDistributionEndpoint string               `yaml:"internalDistributionEndpoint"`
+	TLS                          ConfigurationHttpTLS `yaml:"tls"`
 }
 
 // ConfigurationStorageFilesystem ...

pkg/configs/default.go

@@ -18,6 +18,8 @@ import (
 	"time"
 
 	"github.com/spf13/viper"
+
+	"github.com/go-sigma/sigma/pkg/types/enums"
 )
 
 func defaultSettings() {
@@ -36,4 +38,16 @@ func defaultSettings() {
 	if configuration.HTTP.InternalEndpoint == "" {
 		configuration.HTTP.InternalEndpoint = "http://127.0.0.1:3000"
 	}
+	if configuration.Auth.Jwt.Ttl == 0 {
+		configuration.Auth.Jwt.Ttl = time.Hour
+	}
+	if configuration.Auth.Jwt.RefreshTtl == 0 {
+		configuration.Auth.Jwt.RefreshTtl = time.Hour * 24
+	}
+	if configuration.Namespace.Visibility.String() == "" {
+		configuration.Namespace.Visibility = enums.VisibilityPrivate
+	}
+	if configuration.Cache.Ttl == 0 {
+		configuration.Cache.Ttl = 72 * time.Hour
+	}
 }

pkg/consts/consts.go

@@ -20,6 +20,8 @@ import (
 	"time"
 
 	pwdvalidate "github.com/wagslane/go-password-validator"
+
+	"github.com/go-sigma/sigma/pkg/version"
 )
 
 const (
@@ -63,7 +65,7 @@ const (
 )
 
 // UserAgent represents the user agent
-var UserAgent = fmt.Sprintf("sigma/%s (https://github.com/go-sigma/sigma)", APIVersion)
+var UserAgent = fmt.Sprintf("sigma/%s (https://github.com/go-sigma/sigma)", version.Version)
 
 const (
 	// AuthModel represents the auth model
@@ -82,7 +84,7 @@ const (
 	e = some(where (p.eft == allow)) && !some(where (p.eft == deny))
 
 	[matchers]
-	m = g(r.sub, p.sub, r.ns) && keyMatch(r.ns, p.ns) && urlMatch(r.url, p.url) && regexMatch(r.visibility, p.visibility) && regexMatch(r.method, p.method) && p.effect == "allow" || r.sub == "admin"`
+	m = g(r.sub, p.sub, r.ns) && keyMatch(r.ns, p.ns) && urlMatch(r.url, p.url) && regexMatch(r.visibility, p.visibility) && regexMatch(r.method, p.method) && p.effect == "allow" || r.sub == "admin" || r.sub == "root"`
 )
 
 var (

pkg/daemon/decorator_artifact.go

@@ -16,6 +16,7 @@ package daemon
 
 import (
 	"context"
+	"sync"
 
 	"github.com/rs/zerolog/log"
 	"github.com/tidwall/gjson"
@@ -52,13 +53,17 @@ func DecoratorArtifact(runner func(context.Context, *models.Artifact, chan Decor
 			return err
 		}
 
+		var waitAllEvents = &sync.WaitGroup{}
+		waitAllEvents.Add(1)
+
 		var statusChan = make(chan DecoratorArtifactStatus, 1)
-		defer close(statusChan)
 		go func() {
+			defer waitAllEvents.Done()
+			ctx := log.Logger.WithContext(ctx)
 			for status := range statusChan {
 				switch status.Daemon {
 				case enums.DaemonVulnerability:
-					err = artifactService.UpdateVulnerability(context.Background(), id,
+					err = artifactService.UpdateVulnerability(ctx, id,
 						map[string]any{
 							query.ArtifactVulnerability.Raw.ColumnName().String():     status.Raw,
 							query.ArtifactVulnerability.Result.ColumnName().String():  status.Result,
@@ -69,7 +74,7 @@ func DecoratorArtifact(runner func(context.Context, *models.Artifact, chan Decor
 						},
 					)
 				case enums.DaemonSbom:
-					err = artifactService.UpdateSbom(context.Background(),
+					err = artifactService.UpdateSbom(ctx,
 						id,
 						map[string]any{
 							query.ArtifactSbom.Raw.ColumnName().String():     status.Raw,
@@ -94,6 +99,8 @@ func DecoratorArtifact(runner func(context.Context, *models.Artifact, chan Decor
 			return err
 		}
 
+		waitAllEvents.Wait()
+
 		return nil
 	}
 }

pkg/daemon/sbom/sbom.go

@@ -65,6 +65,7 @@ type Report struct {
 }
 
 func runner(ctx context.Context, artifact *models.Artifact, statusChan chan daemon.DecoratorArtifactStatus) error {
+	defer close(statusChan)
 	statusChan <- daemon.DecoratorArtifactStatus{Daemon: enums.DaemonSbom, Status: enums.TaskCommonStatusDoing, Message: ""}
 
 	config := ptr.To(configs.GetConfiguration())

pkg/daemon/vulnerability/vulnerability.go

@@ -59,6 +59,7 @@ type Report struct {
 }
 
 func runner(ctx context.Context, artifact *models.Artifact, statusChan chan daemon.DecoratorArtifactStatus) error {
+	defer close(statusChan)
 	statusChan <- daemon.DecoratorArtifactStatus{Daemon: enums.DaemonVulnerability, Status: enums.TaskCommonStatusDoing, Message: ""}
 
 	config := ptr.To(configs.GetConfiguration())

pkg/dal/dao/artifact.go

@@ -81,6 +81,8 @@ type ArtifactService interface {
 	GetNamespaceSize(ctx context.Context, namespaceID int64) (int64, error)
 	// GetRepositorySize get the specific repository size
 	GetRepositorySize(ctx context.Context, repositoryID int64) (int64, error)
+	// GetReferrers ...
+	GetReferrers(ctx context.Context, repositoryID int64, digest string, artifactTypes []string) ([]*models.Artifact, error)
 }
 
 type artifactService struct {
@@ -334,7 +336,7 @@ func (s *artifactService) UpdateSbom(ctx context.Context, artifactID int64, upda
 	if len(updates) == 0 {
 		return nil
 	}
-	_, err := s.tx.ArtifactSbom.WithContext(ctx).Where(s.tx.ArtifactSbom.ID.Eq(artifactID)).UpdateColumns(updates)
+	_, err := s.tx.ArtifactSbom.WithContext(ctx).Where(s.tx.ArtifactSbom.ArtifactID.Eq(artifactID)).UpdateColumns(updates)
 	return err
 }
 
@@ -343,7 +345,7 @@ func (s *artifactService) UpdateVulnerability(ctx context.Context, artifactID in
 	if len(updates) == 0 {
 		return nil
 	}
-	_, err := s.tx.ArtifactVulnerability.WithContext(ctx).Where(s.tx.ArtifactVulnerability.ID.Eq(artifactID)).UpdateColumns(updates)
+	_, err := s.tx.ArtifactVulnerability.WithContext(ctx).Where(s.tx.ArtifactVulnerability.ArtifactID.Eq(artifactID)).UpdateColumns(updates)
 	return err
 }
 
@@ -364,3 +366,17 @@ func (s *artifactService) GetRepositorySize(ctx context.Context, repositoryID in
 	}
 	return result.Size, nil
 }
+
+// GetReferrers ...
+func (s *artifactService) GetReferrers(ctx context.Context, repositoryID int64, digest string, artifactTypes []string) ([]*models.Artifact, error) {
+	artifactObj, err := s.tx.Artifact.WithContext(ctx).Where(s.tx.Artifact.RepositoryID.Eq(repositoryID)).
+		Where(s.tx.Artifact.Digest.Eq(digest)).First()
+	if err != nil {
+		return nil, err
+	}
+	query := s.tx.Artifact.WithContext(ctx).Where(s.tx.Artifact.RepositoryID.Eq(repositoryID))
+	if len(artifactTypes) > 0 {
+		query = query.Where(s.tx.Artifact.ConfigMediaType.In(artifactTypes...))
+	}
+	return query.Where(s.tx.Artifact.ReferrerID.Eq(artifactObj.ID)).Find()
+}

pkg/dal/dao/tag.go

@@ -241,6 +241,8 @@ func (s *tagService) ListTag(ctx context.Context, repositoryID int64, name *stri
 	}
 	query = query.Preload(s.tx.Tag.Artifact.Vulnerability)
 	query = query.Preload(s.tx.Tag.Artifact.Sbom)
+	query.UnderlyingDB().Preload("Artifact.ArtifactIndexes.Vulnerability")
+	query.UnderlyingDB().Preload("Artifact.ArtifactIndexes.Sbom")
 	return query.FindByPage(ptr.To(pagination.Limit)*(ptr.To(pagination.Page)-1), ptr.To(pagination.Limit))
 }
 

pkg/dal/dao/user.go

@@ -45,6 +45,8 @@ type UserService interface {
 	UpdateUser3rdParty(ctx context.Context, id int64, updates map[string]any) error
 	// List all users with pagination
 	List(ctx context.Context, name *string, pagination types.Pagination, sort types.Sortable) ([]*models.User, int64, error)
+	// ListWithoutUsername all users with pagination, and without specific username
+	ListWithoutUsername(ctx context.Context, expect string, name *string, pagination types.Pagination, sort types.Sortable) ([]*models.User, int64, error)
 	// UpdateByID updates the namespace with the specified namespace ID.
 	UpdateByID(ctx context.Context, id int64, updates map[string]interface{}) error
 	// Count gets the total number of users.
@@ -131,6 +133,29 @@ func (s *userService) UpdateUser3rdParty(ctx context.Context, id int64, updates
 	return nil
 }
 
+// ListWithoutUsername all users with pagination, and without specific username
+func (s *userService) ListWithoutUsername(ctx context.Context, expect string, name *string, pagination types.Pagination, sort types.Sortable) ([]*models.User, int64, error) {
+	pagination = utils.NormalizePagination(pagination)
+	query := s.tx.User.WithContext(ctx).Where(s.tx.User.Username.Neq(expect))
+	if name != nil {
+		query = query.Where(s.tx.User.Username.Like(fmt.Sprintf("%%%s%%", ptr.To(name))))
+	}
+	field, ok := s.tx.User.GetFieldByName(ptr.To(sort.Sort))
+	if ok {
+		switch ptr.To(sort.Method) {
+		case enums.SortMethodDesc:
+			query = query.Order(field.Desc())
+		case enums.SortMethodAsc:
+			query = query.Order(field)
+		default:
+			query = query.Order(s.tx.User.UpdatedAt.Desc())
+		}
+	} else {
+		query = query.Order(s.tx.User.UpdatedAt.Desc())
+	}
+	return query.FindByPage(ptr.To(pagination.Limit)*(ptr.To(pagination.Page)-1), ptr.To(pagination.Limit))
+}
+
 // List all users with pagination
 func (s *userService) List(ctx context.Context, name *string, pagination types.Pagination, sort types.Sortable) ([]*models.User, int64, error) {
 	pagination = utils.NormalizePagination(pagination)

pkg/dal/migrations/mysql/0001_initialize.up.sql

@@ -3,10 +3,19 @@ CREATE TABLE IF NOT EXISTS `users` (
   `username` varchar(64) NOT NULL,
   `password` varchar(256),
   `email` varchar(256),
+  `last_login` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `namespace_limit` bigint NOT NULL DEFAULT 0,
+  `namespace_count` bigint NOT NULL DEFAULT 0,
+  `status` ENUM ('Active', 'Inactive') NOT NULL DEFAULT 'Active',
+  `role` ENUM ('Root', 'Admin', 'User') NOT NULL DEFAULT 'User',
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `deleted_at` bigint NOT NULL DEFAULT 0,
-  CONSTRAINT `users_unique_with_username` UNIQUE (`username`, `deleted_at`)
+  UNIQUE KEY `users_unique_with_username` (`username`, `deleted_at`),
+  KEY `users_idx_created_at` (`created_at`),
+  KEY `users_idx_status` (`status`),
+  KEY `users_idx_role` (`role`),
+  KEY `users_idx_last_login` (`last_login`)
 );
 
 CREATE TABLE IF NOT EXISTS `user_3rdparty` (
@@ -158,10 +167,12 @@ CREATE TABLE IF NOT EXISTS `artifacts` (
   `pushed_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `last_pull` timestamp,
   `pull_times` bigint NOT NULL DEFAULT 0,
+  `referrer_id` bigint,
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `updated_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `deleted_at` bigint NOT NULL DEFAULT 0,
   FOREIGN KEY (`repository_id`) REFERENCES `repositories` (`id`),
+  FOREIGN KEY (`referrer_id`) REFERENCES `artifacts` (`id`),
   CONSTRAINT `artifacts_unique_with_repo` UNIQUE (`repository_id`, `digest`, `deleted_at`)
 );