diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml new file mode 100644 index 0000000..a49fb47 --- /dev/null +++ b/.semaphore/semaphore.yml @@ -0,0 +1,42 @@ +version: v1.0 +name: Initial Pipeline +agent: + machine: + type: e2-standard-2 + os_image: ubuntu2004 +blocks: + - name: 'Lint' + dependencies: [] + task: + jobs: + - name: Lint + commands: + - sem-version go 1.21 + - checkout + - go install github.com/mgechev/revive@latest + - make lint + - name: "Security checks" + dependencies: [] + task: + secrets: + - name: security-toolbox-shared-read-access + prologue: + commands: + - checkout + - mv ~/.ssh/security-toolbox ~/.ssh/id_rsa + - sudo chmod 600 ~/.ssh/id_rsa + jobs: + - name: Check dependencies + commands: + - make check.deps + - name: Check code + commands: + - make check.static + - name: Check docker + commands: + - make docker.build + - make check.docker + epilogue: + always: + commands: + - 'if [ -f results.xml ]; then test-results publish --name="Security checks" results.xml; fi' diff --git a/Dockerfile b/Dockerfile index dd865d7..bcaa742 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,17 @@ -FROM alpine:3.14 +FROM ubuntu:22.04 + +ARG USERNAME=semaphore +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +# Create the user +RUN groupadd --gid $USER_GID $USERNAME && \ + useradd --uid $USER_UID --gid $USER_GID -m $USERNAME + COPY build/controller / + +USER $USERNAME +WORKDIR /home/semaphore +HEALTHCHECK NONE + ENTRYPOINT ["/controller"] diff --git a/Makefile b/Makefile index d9ef100..648cb87 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,16 @@ check.deps: check.prepare registry.semaphoreci.com/ruby:2.7 \ bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/dependencies --language go -d' +check.docker: check.prepare + docker run -it -v $$(pwd):/app \ + -v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \ + -v /var/run/docker.sock:/var/run/docker.sock \ + registry.semaphoreci.com/ruby:2.7 \ + bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/docker -d --image $(REGISTRY):latest' + +lint: + revive -formatter friendly -config lint.toml ./... + build: rm -rf build env GOOS=linux go build -o build/controller main.go diff --git a/README.md b/README.md index 3986d21..2ec06b4 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,23 @@ -### Testing locally +# Semaphore agent controller for Kubernetes -```bash -# Create k8s cluster -sem-version go 1.21 -curl -sLO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && install minikube-linux-amd64 /tmp/ -/tmp/minikube-linux-amd64 config set WantUpdateNotification false -/tmp/minikube-linux-amd64 start --driver=docker -eval $(/tmp/minikube-linux-amd64 docker-env) +A Kubernetes controller that runs Semaphore jobs in Kubernetes. -# Create required k8s resources -kubectl apply -f resources.yml +## Installation -# Expose configuration parameters -export SEMAPHORE_API_TOKEN=??? -export SEMAPHORE_ENDPOINT=rtx.sxpreprod.com -export KUBERNETES_NAMESPACE=default -export SEMAPHORE_AGENT_IMAGE=semaphoreci/agent:v2.2.14 -export KUBERNETES_SERVICE_ACCOUNT=semaphore-agent-svc-account -export MAX_PARALLEL_JOBS=10 -export SEMAPHORE_AGENT_STARTUP_PARAMETERS='--kubernetes-executor-pod-spec WHATEVER --pre-job-hook-path /opt/semaphore/agent/hooks/pre-job.sh --source-pre-job-hook' +### Requirements -# Build and start controller -go build -o controller main.go -./controller &>/tmp/controller.logs -``` +- A Kubernetes cluster +- A Semaphore API token + +### Configuration + +| Environment variable | Description | +|------------------------------------|-------------| +| SEMAPHORE_API_TOKEN | The Semaphore API token used to inspect the job queues. | +| SEMAPHORE_ENDPOINT | The Semaphore control plane endpoint, e.g. `.semaphoreci.com`. | +| KUBERNETES_NAMESPACE | The Kubernetes namespace where the resources for Semaphore jobs will be created. By default, the default namespace is used. | +| SEMAPHORE_AGENT_IMAGE | The [Semaphore agent](https://github.com/semaphoreci/agent) image to use when creating agents. By default, `semaphoreci/agent:latest`. | +| MAX_PARALLEL_JOBS | The max number of Semaphore jobs to run in parallel. By default, 10. | +| KUBERNETES_SERVICE_ACCOUNT | The Kubernetes service account to attach to the pods created for the [Semaphore agent](https://github.com/semaphoreci/agent). | +| SEMAPHORE_AGENT_LABELS | A comma-separated list of Kubernetes labels to apply on all resources created by the controller. | +| SEMAPHORE_AGENT_STARTUP_PARAMETERS | Any additional [Semaphore agent configuration parameters](https://docs.semaphoreci.com/ci-cd-environment/configure-self-hosted-agent/) to pass to the agents being created. | diff --git a/lint.toml b/lint.toml new file mode 100644 index 0000000..f28910b --- /dev/null +++ b/lint.toml @@ -0,0 +1,27 @@ +ignoreGeneratedHeader = false +severity = "warning" +confidence = 0.8 +errorCode = 1 +warningCode = 1 + +[rule.blank-imports] +[rule.context-as-argument] +[rule.context-keys-type] +[rule.dot-imports] +[rule.error-return] +[rule.error-strings] +[rule.error-naming] +# [rule.exported] +[rule.if-return] +[rule.increment-decrement] +[rule.var-naming] +[rule.var-declaration] +[rule.range] +[rule.receiver-naming] +[rule.time-naming] +[rule.unexported-return] +[rule.indent-error-flow] +[rule.errorf] + +[rule.package-comments] + Disabled = true diff --git a/main.go b/main.go index b9920ce..bdd358d 100644 --- a/main.go +++ b/main.go @@ -107,17 +107,14 @@ func NewInformerFactory(clientset kubernetes.Interface, cfg *config.Config) (inf func buildConfig(endpoint string) (*config.Config, error) { k8sNamespace := os.Getenv("KUBERNETES_NAMESPACE") if k8sNamespace == "" { - return nil, fmt.Errorf("no KUBERNETES_NAMESPACE specified") - } - - svcAccountName := os.Getenv("KUBERNETES_SERVICE_ACCOUNT") - if svcAccountName == "" { - return nil, fmt.Errorf("no KUBERNETES_SERVICE_ACCOUNT specified") + k8sNamespace = "default" + klog.Warningf("no KUBERNETES_NAMESPACE specified - using '%s'", k8sNamespace) } agentImage := os.Getenv("SEMAPHORE_AGENT_IMAGE") if agentImage == "" { - return nil, fmt.Errorf("no SEMAPHORE_AGENT_IMAGE specified") + agentImage = "semaphoreci/agent:latest" + klog.Warningf("no SEMAPHORE_AGENT_IMAGE specified - using '%s'", agentImage) } maxParallelJobs := 10 @@ -145,7 +142,7 @@ func buildConfig(endpoint string) (*config.Config, error) { return &config.Config{ SemaphoreEndpoint: endpoint, Namespace: k8sNamespace, - ServiceAccountName: svcAccountName, + ServiceAccountName: os.Getenv("KUBERNETES_SERVICE_ACCOUNT"), AgentImage: agentImage, AgentStartupParameters: agentStartupParameters, MaxParallelJobs: maxParallelJobs, diff --git a/pkg/agent_types/registry.go b/pkg/agenttypes/registry.go similarity index 97% rename from pkg/agent_types/registry.go rename to pkg/agenttypes/registry.go index f5c5c36..f567073 100644 --- a/pkg/agent_types/registry.go +++ b/pkg/agenttypes/registry.go @@ -1,4 +1,4 @@ -package agent_types +package agenttypes import ( "fmt" @@ -29,8 +29,8 @@ func NewRegistry() (*Registry, error) { func (r *Registry) RegisterInformer(informerFactory informers.SharedInformerFactory) error { informer := informerFactory.Core().V1().Secrets() - informer.Informer().AddEventHandler(r) - return nil + _, err := informer.Informer().AddEventHandler(r) + return err } func (r *Registry) OnAdd(obj interface{}, _ bool) { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 5708f8c..fdc1f13 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -10,7 +10,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" - agentTypes "github.com/renderedtext/agent-k8s-stack/pkg/agent_types" + "github.com/renderedtext/agent-k8s-stack/pkg/agenttypes" "github.com/renderedtext/agent-k8s-stack/pkg/config" "github.com/renderedtext/agent-k8s-stack/pkg/semaphore" "k8s.io/client-go/kubernetes" @@ -18,7 +18,7 @@ import ( type Controller struct { semaphoreClient *semaphore.Client - agentTypeRegistry *agentTypes.Registry + agentTypeRegistry *agenttypes.Registry clientset kubernetes.Interface jobScheduler *JobScheduler } @@ -30,7 +30,7 @@ func New( semaphoreClient *semaphore.Client, clientset kubernetes.Interface) (*Controller, error) { - agentTypeRegistry, err := agentTypes.NewRegistry() + agentTypeRegistry, err := agenttypes.NewRegistry() if err != nil { return nil, err } diff --git a/pkg/controller/job_scheduler.go b/pkg/controller/job_scheduler.go index f8da064..47a2be8 100644 --- a/pkg/controller/job_scheduler.go +++ b/pkg/controller/job_scheduler.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - agentTypes "github.com/renderedtext/agent-k8s-stack/pkg/agent_types" + "github.com/renderedtext/agent-k8s-stack/pkg/agenttypes" "github.com/renderedtext/agent-k8s-stack/pkg/config" "github.com/renderedtext/agent-k8s-stack/pkg/semaphore" batchv1 "k8s.io/api/batch/v1" @@ -37,11 +37,11 @@ func NewJobScheduler(clientset kubernetes.Interface, config *config.Config) *Job func (s *JobScheduler) RegisterInformer(informerFactory informers.SharedInformerFactory) error { informer := informerFactory.Batch().V1().Jobs() - informer.Informer().AddEventHandler(s) - return nil + _, err := informer.Informer().AddEventHandler(s) + return err } -func (s *JobScheduler) Create(ctx context.Context, req semaphore.JobRequest, agentType *agentTypes.AgentType) error { +func (s *JobScheduler) Create(ctx context.Context, req semaphore.JobRequest, agentType *agenttypes.AgentType) error { s.mu.Lock() defer s.mu.Unlock() @@ -77,7 +77,7 @@ func (s *JobScheduler) jobName(jobID string) string { return fmt.Sprintf("semaphore-agent-%s", jobID) } -func (s *JobScheduler) buildJob(job semaphore.JobRequest, agentType *agentTypes.AgentType) *batchv1.Job { +func (s *JobScheduler) buildJob(job semaphore.JobRequest, agentType *agenttypes.AgentType) *batchv1.Job { parallelism := int32(1) retries := int32(0) activeDeadlineSeconds := int64(60 * 60 * 24) // 1 day @@ -157,7 +157,7 @@ func (s *JobScheduler) buildLabels(job semaphore.JobRequest) map[string]string { return labels } -func (s *JobScheduler) buildAgentStartupParameters(agentType *agentTypes.AgentType, jobID string) []string { +func (s *JobScheduler) buildAgentStartupParameters(agentType *agenttypes.AgentType, jobID string) []string { labels := []string{ fmt.Sprintf("%s=%s", config.AgentTypeLabel, agentType.AgentTypeName), } diff --git a/resources.yml b/resources.yml deleted file mode 100644 index 8146591..0000000 --- a/resources.yml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: semaphore-agent-svc-account ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: semaphore-agent-svc-account -rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "create", "patch", "delete"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get"] - - apiGroups: [""] - resources: ["pods/exec"] - verbs: ["create"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: semaphore-agent-svc-account -subjects: -- kind: ServiceAccount - name: semaphore-agent-svc-account -roleRef: - kind: Role - apiGroup: rbac.authorization.k8s.io - name: semaphore-agent-svc-account