✨ Set network for builder

README.md

@@ -15,16 +15,17 @@ Runs on aws ec2 (2C2G, Disk 20G), Linux distribution is Debian 12.1, Docker vers
 
 ``` sh
 # Install Docker from get.docker.com
-./scripts/samples/init.sh
+sh ./scripts/samples/init.sh
 
 # If your docker running with rootless mode,
 # make sure add net.ipv4.ip_unprivileged_port_start=0 to /etc/sysctl.conf and run sudo sysctl --system.
+docker network create sigma
 docker run --name sigma -v /home/admin/config:/etc/sigma \
   -v /var/run/docker.sock:/var/run/docker.sock -p 443:3000 \
-  -d ghcr.io/go-sigma/sigma:nightly-alpine
+  -d --net sigma ghcr.io/go-sigma/sigma:nightly-alpine
 
 # Add sample data
-./scripts/samples/samples.sh
+sh ./scripts/samples/samples.sh
 ```
 
 Visit: <https://sigma.tosone.cn>, username/password: sigma/sigma

build/Dockerfile

@@ -68,6 +68,7 @@ FROM alpine:${ALPINE_VERSION}
 COPY --from=syft /usr/local/bin/syft /usr/local/bin/syft
 COPY --from=trivy /usr/local/bin/trivy /usr/local/bin/trivy
 COPY --from=trivy /opt/trivy/trivy.db /opt/trivy/db/trivy.db
+COPY --from=trivy /opt/trivy/metadata.json /opt/trivy/db/metadata.json
 COPY ./conf/config.yaml /etc/sigma/config.yaml
 COPY --from=builder /go/src/github.com/go-sigma/sigma/bin/sigma /usr/local/bin/sigma
 

build/Dockerfile.debian

@@ -85,6 +85,7 @@ RUN set -eux && \
 COPY --from=syft /usr/local/bin/syft /usr/local/bin/syft
 COPY --from=trivy /usr/local/bin/trivy /usr/local/bin/trivy
 COPY --from=trivy /opt/trivy/trivy.db /opt/trivy/db/trivy.db
+COPY --from=trivy /opt/trivy/metadata.json /opt/trivy/db/metadata.json
 COPY ./conf/config.yaml /etc/sigma/config.yaml
 COPY --from=builder /go/src/github.com/go-sigma/sigma/bin/sigma /usr/local/bin/sigma
 

cmd/imports/builder.go

@@ -16,7 +16,7 @@ package imports
 
 import (
 	_ "github.com/go-sigma/sigma/pkg/builder/docker"
-	_ "github.com/go-sigma/sigma/pkg/builder/k8s"
+	_ "github.com/go-sigma/sigma/pkg/builder/kubernetes"
 	_ "github.com/go-sigma/sigma/pkg/builder/logger/database"
 	_ "github.com/go-sigma/sigma/pkg/builder/logger/obs"
 	_ "github.com/go-sigma/sigma/pkg/builder/podman"

conf/config-dev.yaml

@@ -75,6 +75,7 @@ http:
   # you can leave it blank and it will use http://127.0.0.1:3000 as internal domain by default.
   # in k8s cluster, it will be set to the distribution service which is used to pull image from registry, eg: http://registry.default.svc.cluster.local:3000
   # in docker-compose, it will be set to the registry service which is used to pull image from registry, eg: http://registry:3000
+  # if http.tls.enabled is true, internalEndpoint should start with https://
   # eg: http://sigma.test.io, http://sigma.test.io:3000, https://sigma.test.io:30080
   internalEndpoint: http://192.168.31.198:3000
   tls:
@@ -93,19 +94,32 @@ storage:
     region: cn-north-1
     bucket: sigma
     forcePathStyle: true
+
 # Notice: the tag never update after the first pulled from remote registry, unless you delete the image and pull again.
 proxy:
   enabled: false
   endpoint: https://registry-1.docker.io
   tlsVerify: true
   username: ""
   password: ""
+
+# daemon task config
 daemon:
   gc:
     # if blob not associate with artifact
     retention: 72h
     # At 02:00 on Saturday
     cron: 0 2 * * 6
+  builder:
+    image: sigma-builder:latest
+    type: docker
+    docker:
+      sock:
+      network: sigma
+    kubernetes:
+      kubeconfig:
+      namespace:
+
 auth:
   internalUser:
     username: internal-sigma

conf/config-full.yaml

@@ -74,6 +74,7 @@ http:
   # you can leave it blank and it will use http://127.0.0.1:3000 as internal domain by default.
   # in k8s cluster, it will be set to the distribution service which is used to pull image from registry, eg: http://registry.default.svc.cluster.local:3000
   # in docker-compose, it will be set to the registry service which is used to pull image from registry, eg: http://registry:3000
+  # if http.tls.enabled is true, internalEndpoint should start with https://
   # eg: http://sigma.test.io, http://sigma.test.io:3000, https://sigma.test.io:30080
   internalEndpoint:
   tls:
@@ -99,12 +100,24 @@ proxy:
   tlsVerify: true
   username: ""
   password: ""
+
+# daemon task config
 daemon:
   gc:
     # if blob not associate with artifact
     retention: 72h
     # At 02:00 on Saturday
     cron: 0 2 * * 6
+  builder:
+    image: sigma-builder:latest
+    type: docker
+    docker:
+      sock:
+      network: sigma
+    kubernetes:
+      kubeconfig:
+      namespace:
+
 auth:
   internalUser:
     username: internal-sigma

conf/config.yaml

@@ -29,6 +29,8 @@ deploy: single
 
 redis:
   # redis type available: none, external
+  # none: means never use redis
+  # external: means use the specific redis instance
   type: none
   url: redis://:sigma@localhost:6379/0
 
@@ -75,6 +77,7 @@ http:
   # you can leave it blank and it will use http://127.0.0.1:3000 as internal domain by default.
   # in k8s cluster, it will be set to the distribution service which is used to pull image from registry, eg: http://registry.default.svc.cluster.local:3000
   # in docker-compose, it will be set to the registry service which is used to pull image from registry, eg: http://registry:3000
+  # if http.tls.enabled is true, internalEndpoint should start with https://
   # eg: http://sigma.test.io, http://sigma.test.io:3000, https://sigma.test.io:30080
   internalEndpoint:
   tls:
@@ -105,12 +108,24 @@ proxy:
   tlsVerify: true
   username: ""
   password: ""
+
+# daemon task config
 daemon:
   gc:
     # if blob not associate with artifact
     retention: 72h
     # At 02:00 on Saturday
     cron: 0 2 * * 6
+  builder:
+    image: sigma-builder:latest
+    type: docker
+    docker:
+      sock:
+      network: sigma
+    kubernetes:
+      kubeconfig:
+      namespace:
+
 auth:
   internalUser:
     username: internal-sigma

docs/docs/configuration.mdx

@@ -76,6 +76,7 @@ http:
   # you can leave it blank and it will use http://127.0.0.1:3000 as internal domain by default.
   # in k8s cluster, it will be set to the distribution service which is used to pull image from registry, eg: http://registry.default.svc.cluster.local:3000
   # in docker-compose, it will be set to the registry service which is used to pull image from registry, eg: http://registry:3000
+  # if http.tls.enabled is true, internalEndpoint should start with 'https://'
   # eg: http://sigma.test.io, http://sigma.test.io:3000, https://sigma.test.io:30080
   internalEndpoint:
   tls:
@@ -101,12 +102,24 @@ proxy:
   tlsVerify: true
   username: ""
   password: ""
+
+# daemon task config
 daemon:
   gc:
     # if blob not associate with artifact
     retention: 72h
     # At 02:00 on Saturday
     cron: 0 2 * * 6
+  builder:
+    image: sigma-builder:latest
+    type: docker
+    docker:
+      sock:
+      network: sigma
+    kubernetes:
+      kubeconfig:
+      namespace:
+
 auth:
   internalUser:
     username: internal-sigma
@@ -125,5 +138,4 @@ auth:
     github:
       clientId: "e5f9fa9e372dfac66aed"
       clientSecret: "49ab83f4d0665f8579516f7a3f2f753a6a57189b"
-
 ```

pkg/builder/builder.go

@@ -60,18 +60,18 @@ type Factory interface {
 // DriverFactories ...
 var DriverFactories = make(map[string]Factory)
 
-func Initialize() error {
-	typ := "docker"
-	factory, ok := DriverFactories[typ]
+func Initialize(config configs.Configuration) error {
+	typ := config.Daemon.Builder.Type
+	factory, ok := DriverFactories[typ.String()]
 	if !ok {
-		return fmt.Errorf("builder driver %q not registered", typ)
+		return fmt.Errorf("builder driver %s not registered", typ.String())
 	}
 	var err error
 	err = logger.Initialize()
 	if err != nil {
 		return err
 	}
-	Driver, err = factory.New(configs.Configuration{})
+	Driver, err = factory.New(config)
 	if err != nil {
 		return err
 	}

pkg/builder/docker/docker.go

@@ -54,6 +54,7 @@
 		return nil, fmt.Errorf("Create docker client failed: %v", err)
 	}
 	i := &instance{
+		config:                config,
 		client:                cli,
 		controlled:            mapset.NewSet[string](),
 		builderServiceFactory: dao.NewBuilderServiceFactory(),
@@ -67,6 +68,7 @@
 }
 
 type instance struct {
+	config                configs.Configuration
 	client                *client.Client
 	controlled            mapset.Set[string] // the controlled container in docker container
 	builderServiceFactory dao.BuilderServiceFactory
@@ -82,7 +84,7 @@
 	}
 
 	containerConfig := &container.Config{
-		Image:      "docker.io/library/builder:dev",
+		Image:      i.config.Daemon.Builder.Image,
 		Entrypoint: []string{},
 		Cmd:        []string{"sigma-builder"},
 		Env:        envs,
@@ -94,6 +96,7 @@
 	}
 	hostConfig := &container.HostConfig{
 		SecurityOpt: []string{"seccomp=unconfined", "apparmor=unconfined"},
+		NetworkMode: container.NetworkMode(i.config.Daemon.Builder.Docker.Network),
 	}
 	_, err = i.client.ContainerCreate(ctx, containerConfig, hostConfig, nil, nil, i.genContainerID(builderConfig.BuilderID, builderConfig.RunnerID))
 	if err != nil {

pkg/builder/k8s/informer.go → pkg/builder/kubernetes/informer.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package k8s
+package kubernetes
 
 import (
 	"context"

pkg/builder/k8s/k8s.go → pkg/builder/kubernetes/k8s.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package k8s
+package kubernetes
 
 import (
 	"context"

pkg/cmds/server/server.go

@@ -97,14 +97,10 @@ func Serve(serverConfig ServerConfig) error {
 		handlers.InitializeDistribution(e)
 	}
 	if !serverConfig.WithoutWorker {
-		err := builder.Initialize()
+		err := builder.Initialize(config)
 		if err != nil {
 			return err
 		}
-		// err = daemon.InitializeServer()
-		// if err != nil {
-		// 	return err
-		// }
 		err = workq.Initialize(configs.Configuration{
 			WorkQueue: configs.ConfigurationWorkQueue{
 				Type: enums.WorkQueueTypeDatabase,

pkg/cmds/worker/worker.go

@@ -32,20 +32,17 @@ import (
 	"github.com/go-sigma/sigma/pkg/middlewares"
 	"github.com/go-sigma/sigma/pkg/modules/workq"
 	"github.com/go-sigma/sigma/pkg/types/enums"
+	"github.com/go-sigma/sigma/pkg/utils/ptr"
 )
 
 // Worker is the worker initialization
 func Worker() error {
-	err := builder.Initialize()
+	config := ptr.To(configs.GetConfiguration())
+	err := builder.Initialize(config)
 	if err != nil {
 		return err
 	}
 
-	// err = daemon.InitializeServer()
-	// if err != nil {
-	// 	return err
-	// }
-
 	err = workq.Initialize(configs.Configuration{
 		WorkQueue: configs.ConfigurationWorkQueue{
 			Type: enums.WorkQueueTypeDatabase,

pkg/configs/configuration.go

@@ -229,9 +229,30 @@ type ConfigurationDaemonGc struct {
 	Cron      string `yaml:"cron"`
 }
 
+// ConfigurationDaemonDocker ...
+type ConfigurationDaemonDocker struct {
+	Sock    *string `yaml:"sock"`
+	Network string  `yaml:"network"`
+}
+
+// ConfigurationDaemonKubernetes ...
+type ConfigurationDaemonKubernetes struct {
+	Kubeconfig *string `yaml:"kubeconfig"`
+	Namespace  *string `yaml:"namespace"`
+}
+
+// ConfigurationDaemonBuilder ...
+type ConfigurationDaemonBuilder struct {
+	Type       enums.BuilderType             `yaml:"type"`
+	Image      string                        `yaml:"image"`
+	Docker     ConfigurationDaemonDocker     `yaml:"docker"`
+	Kubernetes ConfigurationDaemonKubernetes `yaml:"kubernetes"`
+}
+
 // ConfigurationDaemon ...
 type ConfigurationDaemon struct {
-	Gc ConfigurationDaemonGc `yaml:"gc"`
+	Gc      ConfigurationDaemonGc      `yaml:"gc"`
+	Builder ConfigurationDaemonBuilder `yaml:"builder"`
 }
 
 // ConfigurationAuthInternalUser ...