From c6cc57f085e2b292f70b2087d4b0ddda880ddbd8 Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Fri, 19 Jan 2024 17:59:13 +0000 Subject: [PATCH 1/4] Use Single job starter workflow --- .semaphore/semaphore.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .semaphore/semaphore.yml diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml new file mode 100644 index 0000000..27b5115 --- /dev/null +++ b/.semaphore/semaphore.yml @@ -0,0 +1,13 @@ +version: v1.0 +name: Initial Pipeline +agent: + machine: + type: e1-standard-2 + os_image: ubuntu2004 +blocks: + - name: 'Block #1' + task: + jobs: + - name: 'Job #1' + commands: + - checkout From 75328d5a7190b083192ac1e0f564207a2710a1fb Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Fri, 19 Jan 2024 15:24:04 -0300 Subject: [PATCH 2/4] update README and semaphore.yaml --- .semaphore/semaphore.yml | 9 +++-- Makefile | 3 ++ README.md | 40 ++++++++++----------- lint.toml | 27 ++++++++++++++ main.go | 13 +++---- pkg/{agent_types => agenttypes}/registry.go | 2 +- pkg/controller/controller.go | 6 ++-- pkg/controller/job_scheduler.go | 8 ++--- resources.yml | 34 ------------------ 9 files changed, 67 insertions(+), 75 deletions(-) create mode 100644 lint.toml rename pkg/{agent_types => agenttypes}/registry.go (99%) delete mode 100644 resources.yml diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 27b5115..3e8039b 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -2,12 +2,15 @@ version: v1.0 name: Initial Pipeline agent: machine: - type: e1-standard-2 + type: e2-standard-2 os_image: ubuntu2004 blocks: - - name: 'Block #1' + - name: 'Tests' task: jobs: - - name: 'Job #1' + - name: Lint commands: + - sem-version go 1.21 - checkout + - go install github.com/mgechev/revive@latest + - make lint diff --git a/Makefile b/Makefile index d9ef100..5abf3f9 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,9 @@ check.deps: check.prepare registry.semaphoreci.com/ruby:2.7 \ bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/dependencies --language go -d' +lint: + revive -formatter friendly -config lint.toml ./... + build: rm -rf build env GOOS=linux go build -o build/controller main.go diff --git a/README.md b/README.md index 3986d21..7e25b55 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,22 @@ -### Testing locally +# Semaphore agent controller for Kubernetes -```bash -# Create k8s cluster -sem-version go 1.21 -curl -sLO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && install minikube-linux-amd64 /tmp/ -/tmp/minikube-linux-amd64 config set WantUpdateNotification false -/tmp/minikube-linux-amd64 start --driver=docker -eval $(/tmp/minikube-linux-amd64 docker-env) +A Kubernetes controller that runs Semaphore jobs in Kubernetes. -# Create required k8s resources -kubectl apply -f resources.yml +## Installation -# Expose configuration parameters -export SEMAPHORE_API_TOKEN=??? -export SEMAPHORE_ENDPOINT=rtx.sxpreprod.com -export KUBERNETES_NAMESPACE=default -export SEMAPHORE_AGENT_IMAGE=semaphoreci/agent:v2.2.14 -export KUBERNETES_SERVICE_ACCOUNT=semaphore-agent-svc-account -export MAX_PARALLEL_JOBS=10 -export SEMAPHORE_AGENT_STARTUP_PARAMETERS='--kubernetes-executor-pod-spec WHATEVER --pre-job-hook-path /opt/semaphore/agent/hooks/pre-job.sh --source-pre-job-hook' +### Requirements -# Build and start controller -go build -o controller main.go -./controller &>/tmp/controller.logs -``` +- A Kubernetes cluster +- A Semaphore API token + +### Configuration + +All the configuration for the controller is provided through environment variables: +- **SEMAPHORE_API_TOKEN**: the Semaphore API token used by the controller to inspect the job queues. +- **SEMAPHORE_ENDPOINT**: the Semaphore endpoint, e.g. `.semaphoreci.com`. +- **KUBERNETES_NAMESPACE**: in which Kubernetes namespace the controller should create the resources for the Semaphore jobs. If nothing is specified, the default namespace is used. +- **SEMAPHORE_AGENT_IMAGE**: the [Semaphore agent](https://github.com/semaphoreci/agent) image to use when creating agents. If nothing is specified, `semaphoreci/agent:latest` is used. +- **MAX_PARALLEL_JOBS**: how much Semaphore jobs to run in parallel. By default, 10. +- **KUBERNETES_SERVICE_ACCOUNT**: the Kubernetes service account name to use on the pods created to run the [Semaphore agent](https://github.com/semaphoreci/agent). +- **SEMAPHORE_AGENT_LABELS**: a comma-separated list of Kubernetes labels to apply on all resources created by the controller. +- **SEMAPHORE_AGENT_STARTUP_PARAMETERS**: any additional [Semaphore agent configuration parameters](https://docs.semaphoreci.com/ci-cd-environment/configure-self-hosted-agent/) to pass to the agents being created. diff --git a/lint.toml b/lint.toml new file mode 100644 index 0000000..f28910b --- /dev/null +++ b/lint.toml @@ -0,0 +1,27 @@ +ignoreGeneratedHeader = false +severity = "warning" +confidence = 0.8 +errorCode = 1 +warningCode = 1 + +[rule.blank-imports] +[rule.context-as-argument] +[rule.context-keys-type] +[rule.dot-imports] +[rule.error-return] +[rule.error-strings] +[rule.error-naming] +# [rule.exported] +[rule.if-return] +[rule.increment-decrement] +[rule.var-naming] +[rule.var-declaration] +[rule.range] +[rule.receiver-naming] +[rule.time-naming] +[rule.unexported-return] +[rule.indent-error-flow] +[rule.errorf] + +[rule.package-comments] + Disabled = true diff --git a/main.go b/main.go index b9920ce..bdd358d 100644 --- a/main.go +++ b/main.go @@ -107,17 +107,14 @@ func NewInformerFactory(clientset kubernetes.Interface, cfg *config.Config) (inf func buildConfig(endpoint string) (*config.Config, error) { k8sNamespace := os.Getenv("KUBERNETES_NAMESPACE") if k8sNamespace == "" { - return nil, fmt.Errorf("no KUBERNETES_NAMESPACE specified") - } - - svcAccountName := os.Getenv("KUBERNETES_SERVICE_ACCOUNT") - if svcAccountName == "" { - return nil, fmt.Errorf("no KUBERNETES_SERVICE_ACCOUNT specified") + k8sNamespace = "default" + klog.Warningf("no KUBERNETES_NAMESPACE specified - using '%s'", k8sNamespace) } agentImage := os.Getenv("SEMAPHORE_AGENT_IMAGE") if agentImage == "" { - return nil, fmt.Errorf("no SEMAPHORE_AGENT_IMAGE specified") + agentImage = "semaphoreci/agent:latest" + klog.Warningf("no SEMAPHORE_AGENT_IMAGE specified - using '%s'", agentImage) } maxParallelJobs := 10 @@ -145,7 +142,7 @@ func buildConfig(endpoint string) (*config.Config, error) { return &config.Config{ SemaphoreEndpoint: endpoint, Namespace: k8sNamespace, - ServiceAccountName: svcAccountName, + ServiceAccountName: os.Getenv("KUBERNETES_SERVICE_ACCOUNT"), AgentImage: agentImage, AgentStartupParameters: agentStartupParameters, MaxParallelJobs: maxParallelJobs, diff --git a/pkg/agent_types/registry.go b/pkg/agenttypes/registry.go similarity index 99% rename from pkg/agent_types/registry.go rename to pkg/agenttypes/registry.go index f5c5c36..f7cbc26 100644 --- a/pkg/agent_types/registry.go +++ b/pkg/agenttypes/registry.go @@ -1,4 +1,4 @@ -package agent_types +package agenttypes import ( "fmt" diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 5708f8c..fdc1f13 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -10,7 +10,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" - agentTypes "github.com/renderedtext/agent-k8s-stack/pkg/agent_types" + "github.com/renderedtext/agent-k8s-stack/pkg/agenttypes" "github.com/renderedtext/agent-k8s-stack/pkg/config" "github.com/renderedtext/agent-k8s-stack/pkg/semaphore" "k8s.io/client-go/kubernetes" @@ -18,7 +18,7 @@ import ( type Controller struct { semaphoreClient *semaphore.Client - agentTypeRegistry *agentTypes.Registry + agentTypeRegistry *agenttypes.Registry clientset kubernetes.Interface jobScheduler *JobScheduler } @@ -30,7 +30,7 @@ func New( semaphoreClient *semaphore.Client, clientset kubernetes.Interface) (*Controller, error) { - agentTypeRegistry, err := agentTypes.NewRegistry() + agentTypeRegistry, err := agenttypes.NewRegistry() if err != nil { return nil, err } diff --git a/pkg/controller/job_scheduler.go b/pkg/controller/job_scheduler.go index f8da064..8d5961e 100644 --- a/pkg/controller/job_scheduler.go +++ b/pkg/controller/job_scheduler.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - agentTypes "github.com/renderedtext/agent-k8s-stack/pkg/agent_types" + "github.com/renderedtext/agent-k8s-stack/pkg/agenttypes" "github.com/renderedtext/agent-k8s-stack/pkg/config" "github.com/renderedtext/agent-k8s-stack/pkg/semaphore" batchv1 "k8s.io/api/batch/v1" @@ -41,7 +41,7 @@ func (s *JobScheduler) RegisterInformer(informerFactory informers.SharedInformer return nil } -func (s *JobScheduler) Create(ctx context.Context, req semaphore.JobRequest, agentType *agentTypes.AgentType) error { +func (s *JobScheduler) Create(ctx context.Context, req semaphore.JobRequest, agentType *agenttypes.AgentType) error { s.mu.Lock() defer s.mu.Unlock() @@ -77,7 +77,7 @@ func (s *JobScheduler) jobName(jobID string) string { return fmt.Sprintf("semaphore-agent-%s", jobID) } -func (s *JobScheduler) buildJob(job semaphore.JobRequest, agentType *agentTypes.AgentType) *batchv1.Job { +func (s *JobScheduler) buildJob(job semaphore.JobRequest, agentType *agenttypes.AgentType) *batchv1.Job { parallelism := int32(1) retries := int32(0) activeDeadlineSeconds := int64(60 * 60 * 24) // 1 day @@ -157,7 +157,7 @@ func (s *JobScheduler) buildLabels(job semaphore.JobRequest) map[string]string { return labels } -func (s *JobScheduler) buildAgentStartupParameters(agentType *agentTypes.AgentType, jobID string) []string { +func (s *JobScheduler) buildAgentStartupParameters(agentType *agenttypes.AgentType, jobID string) []string { labels := []string{ fmt.Sprintf("%s=%s", config.AgentTypeLabel, agentType.AgentTypeName), } diff --git a/resources.yml b/resources.yml deleted file mode 100644 index 8146591..0000000 --- a/resources.yml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: semaphore-agent-svc-account ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: semaphore-agent-svc-account -rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "create", "patch", "delete"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["pods/exec"] - verbs: ["create"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: semaphore-agent-svc-account -subjects: -- kind: ServiceAccount - name: semaphore-agent-svc-account -roleRef: - kind: Role - apiGroup: rbac.authorization.k8s.io - name: semaphore-agent-svc-account From a170c2205f9071d013129da23f204bcf84bd94a6 Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Fri, 19 Jan 2024 15:37:04 -0300 Subject: [PATCH 3/4] run security checks --- .semaphore/semaphore.yml | 28 +++++++++++++++++++++++++++- Makefile | 7 +++++++ README.md | 21 ++++++++++++--------- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 3e8039b..a49fb47 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -5,7 +5,8 @@ agent: type: e2-standard-2 os_image: ubuntu2004 blocks: - - name: 'Tests' + - name: 'Lint' + dependencies: [] task: jobs: - name: Lint @@ -14,3 +15,28 @@ blocks: - checkout - go install github.com/mgechev/revive@latest - make lint + - name: "Security checks" + dependencies: [] + task: + secrets: + - name: security-toolbox-shared-read-access + prologue: + commands: + - checkout + - mv ~/.ssh/security-toolbox ~/.ssh/id_rsa + - sudo chmod 600 ~/.ssh/id_rsa + jobs: + - name: Check dependencies + commands: + - make check.deps + - name: Check code + commands: + - make check.static + - name: Check docker + commands: + - make docker.build + - make check.docker + epilogue: + always: + commands: + - 'if [ -f results.xml ]; then test-results publish --name="Security checks" results.xml; fi' diff --git a/Makefile b/Makefile index 5abf3f9..648cb87 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,13 @@ check.deps: check.prepare registry.semaphoreci.com/ruby:2.7 \ bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/dependencies --language go -d' +check.docker: check.prepare + docker run -it -v $$(pwd):/app \ + -v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \ + -v /var/run/docker.sock:/var/run/docker.sock \ + registry.semaphoreci.com/ruby:2.7 \ + bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/docker -d --image $(REGISTRY):latest' + lint: revive -formatter friendly -config lint.toml ./... diff --git a/README.md b/README.md index 7e25b55..2b7a8f5 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,15 @@ A Kubernetes controller that runs Semaphore jobs in Kubernetes. ### Configuration -All the configuration for the controller is provided through environment variables: -- **SEMAPHORE_API_TOKEN**: the Semaphore API token used by the controller to inspect the job queues. -- **SEMAPHORE_ENDPOINT**: the Semaphore endpoint, e.g. `.semaphoreci.com`. -- **KUBERNETES_NAMESPACE**: in which Kubernetes namespace the controller should create the resources for the Semaphore jobs. If nothing is specified, the default namespace is used. -- **SEMAPHORE_AGENT_IMAGE**: the [Semaphore agent](https://github.com/semaphoreci/agent) image to use when creating agents. If nothing is specified, `semaphoreci/agent:latest` is used. -- **MAX_PARALLEL_JOBS**: how much Semaphore jobs to run in parallel. By default, 10. -- **KUBERNETES_SERVICE_ACCOUNT**: the Kubernetes service account name to use on the pods created to run the [Semaphore agent](https://github.com/semaphoreci/agent). -- **SEMAPHORE_AGENT_LABELS**: a comma-separated list of Kubernetes labels to apply on all resources created by the controller. -- **SEMAPHORE_AGENT_STARTUP_PARAMETERS**: any additional [Semaphore agent configuration parameters](https://docs.semaphoreci.com/ci-cd-environment/configure-self-hosted-agent/) to pass to the agents being created. +All the configuration for the controller is provided through environment variables. + +| Environment variable | Description | +|------------------------------------|-------------| +| SEMAPHORE_API_TOKEN | The Semaphore API token used to inspect the job queues. | +| SEMAPHORE_ENDPOINT | The Semaphore control plane endpoint, e.g. `.semaphoreci.com`. | +| KUBERNETES_NAMESPACE | The Kubernetes namespace where the resources for Semaphore jobs will be created. By default, the default namespace is used. | +| SEMAPHORE_AGENT_IMAGE | The [Semaphore agent](https://github.com/semaphoreci/agent) image to use when creating agents. By default, `semaphoreci/agent:latest`. | +| MAX_PARALLEL_JOBS | The max number of Semaphore jobs to run in parallel. By default, 10. | +| KUBERNETES_SERVICE_ACCOUNT | The Kubernetes service account to attach to the pods created for the [Semaphore agent](https://github.com/semaphoreci/agent). | +| SEMAPHORE_AGENT_LABELS | A comma-separated list of Kubernetes labels to apply on all resources created by the controller. | +| SEMAPHORE_AGENT_STARTUP_PARAMETERS | Any additional [Semaphore agent configuration parameters](https://docs.semaphoreci.com/ci-cd-environment/configure-self-hosted-agent/) to pass to the agents being created. | From 7dcd8f3189ca347a3f8fab568c2a16038db49cb8 Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Fri, 19 Jan 2024 15:46:08 -0300 Subject: [PATCH 4/4] fix code and docker image checks --- Dockerfile | 16 +++++++++++++++- README.md | 2 -- pkg/agenttypes/registry.go | 4 ++-- pkg/controller/job_scheduler.go | 4 ++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index dd865d7..bcaa742 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,17 @@ -FROM alpine:3.14 +FROM ubuntu:22.04 + +ARG USERNAME=semaphore +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +# Create the user +RUN groupadd --gid $USER_GID $USERNAME && \ + useradd --uid $USER_UID --gid $USER_GID -m $USERNAME + COPY build/controller / + +USER $USERNAME +WORKDIR /home/semaphore +HEALTHCHECK NONE + ENTRYPOINT ["/controller"] diff --git a/README.md b/README.md index 2b7a8f5..2ec06b4 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,6 @@ A Kubernetes controller that runs Semaphore jobs in Kubernetes. ### Configuration -All the configuration for the controller is provided through environment variables. - | Environment variable | Description | |------------------------------------|-------------| | SEMAPHORE_API_TOKEN | The Semaphore API token used to inspect the job queues. | diff --git a/pkg/agenttypes/registry.go b/pkg/agenttypes/registry.go index f7cbc26..f567073 100644 --- a/pkg/agenttypes/registry.go +++ b/pkg/agenttypes/registry.go @@ -29,8 +29,8 @@ func NewRegistry() (*Registry, error) { func (r *Registry) RegisterInformer(informerFactory informers.SharedInformerFactory) error { informer := informerFactory.Core().V1().Secrets() - informer.Informer().AddEventHandler(r) - return nil + _, err := informer.Informer().AddEventHandler(r) + return err } func (r *Registry) OnAdd(obj interface{}, _ bool) { diff --git a/pkg/controller/job_scheduler.go b/pkg/controller/job_scheduler.go index 8d5961e..47a2be8 100644 --- a/pkg/controller/job_scheduler.go +++ b/pkg/controller/job_scheduler.go @@ -37,8 +37,8 @@ func NewJobScheduler(clientset kubernetes.Interface, config *config.Config) *Job func (s *JobScheduler) RegisterInformer(informerFactory informers.SharedInformerFactory) error { informer := informerFactory.Batch().V1().Jobs() - informer.Informer().AddEventHandler(s) - return nil + _, err := informer.Informer().AddEventHandler(s) + return err } func (s *JobScheduler) Create(ctx context.Context, req semaphore.JobRequest, agentType *agenttypes.AgentType) error {