From 446e87d84cddffcf9da1de48a5801589d599c80b Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Fri, 5 Jan 2024 17:39:32 -0300 Subject: [PATCH] feat: support for agent startup parameters (#2) --- .gitignore | 1 + Dockerfile | 3 ++ Makefile | 39 ++++++++++++++++ README.md | 1 + main.go | 25 +++++++---- pkg/controller/agent_type_finder.go | 26 ++++++----- pkg/controller/controller.go | 70 ++++++++++++++++------------- 7 files changed, 115 insertions(+), 50 deletions(-) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d163863 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +build/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dd865d7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,3 @@ +FROM alpine:3.14 +COPY build/controller / +ENTRYPOINT ["/controller"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d9ef100 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +.PHONY: build test + +REGISTRY=semaphoreci/controller +LATEST_VERSION=$(shell git tag | sort --version-sort | tail -n 1) + +SECURITY_TOOLBOX_BRANCH ?= master +SECURITY_TOOLBOX_TMP_DIR ?= /tmp/security-toolbox + +check.prepare: + rm -rf $(SECURITY_TOOLBOX_TMP_DIR) + git clone git@github.com:renderedtext/security-toolbox.git $(SECURITY_TOOLBOX_TMP_DIR) && (cd $(SECURITY_TOOLBOX_TMP_DIR) && git checkout $(SECURITY_TOOLBOX_BRANCH) && cd -) + +check.static: check.prepare + docker run -it -v $$(pwd):/app \ + -v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \ + registry.semaphoreci.com/ruby:2.7 \ + bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/code --language go -d' + +check.deps: check.prepare + docker run -it -v $$(pwd):/app \ + -v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \ + registry.semaphoreci.com/ruby:2.7 \ + bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/dependencies --language go -d' + +build: + rm -rf build + env GOOS=linux go build -o build/controller main.go + +docker.build: build + docker build -t $(REGISTRY):latest . + +docker.push: + @if [ -z "$(LATEST_VERSION)" ]; then \ + docker push $(REGISTRY):latest; \ + else \ + docker tag $(REGISTRY):latest $(REGISTRY):$(LATEST_VERSION); \ + docker push $(REGISTRY):$(LATEST_VERSION); \ + docker push $(REGISTRY):latest; \ + fi \ No newline at end of file diff --git a/README.md b/README.md index 15db491..3986d21 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ export KUBERNETES_NAMESPACE=default export SEMAPHORE_AGENT_IMAGE=semaphoreci/agent:v2.2.14 export KUBERNETES_SERVICE_ACCOUNT=semaphore-agent-svc-account export MAX_PARALLEL_JOBS=10 +export SEMAPHORE_AGENT_STARTUP_PARAMETERS='--kubernetes-executor-pod-spec WHATEVER --pre-job-hook-path /opt/semaphore/agent/hooks/pre-job.sh --source-pre-job-hook' # Build and start controller go build -o controller main.go diff --git a/main.go b/main.go index 0d2d8c4..9e793fd 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "github.com/renderedtext/agent-k8s-stack/pkg/controller" "github.com/renderedtext/agent-k8s-stack/pkg/semaphore" @@ -72,12 +73,18 @@ func buildConfig(endpoint string) (*controller.Config, error) { } } + agentStartupParameters := []string{} + if os.Getenv("SEMAPHORE_AGENT_STARTUP_PARAMETERS") != "" { + agentStartupParameters = strings.Split(os.Getenv("SEMAPHORE_AGENT_STARTUP_PARAMETERS"), " ") + } + return &controller.Config{ - SemaphoreEndpoint: endpoint, - Namespace: k8sNamespace, - ServiceAccountName: svcAccountName, - AgentImage: agentImage, - MaxParallelJobs: maxParallelJobs, + SemaphoreEndpoint: endpoint, + Namespace: k8sNamespace, + ServiceAccountName: svcAccountName, + AgentImage: agentImage, + AgentStartupParameters: agentStartupParameters, + MaxParallelJobs: maxParallelJobs, }, nil } @@ -88,7 +95,7 @@ func newK8sClientset() (kubernetes.Interface, error) { clientset, err = newClientsetFromConfig() if err != nil { - return nil, fmt.Errorf("error creating kubernetes clientset: %v\n", err) + return nil, fmt.Errorf("error creating kubernetes clientset: %v", err) } } @@ -98,18 +105,18 @@ func newK8sClientset() (kubernetes.Interface, error) { func newClientsetFromConfig() (kubernetes.Interface, error) { homeDir, err := os.UserHomeDir() if err != nil { - return nil, fmt.Errorf("error getting user home directory: %v\n", err) + return nil, fmt.Errorf("error getting user home directory: %v", err) } kubeConfigPath := filepath.Join(homeDir, ".kube", "config") kubeConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) if err != nil { - return nil, fmt.Errorf("error getting Kubernetes config: %v\n", err) + return nil, fmt.Errorf("error getting Kubernetes config: %v", err) } clientset, err := kubernetes.NewForConfig(kubeConfig) if err != nil { - return nil, fmt.Errorf("error creating kubernetes clientset from config file: %v\n", err) + return nil, fmt.Errorf("error creating kubernetes clientset from config file: %v", err) } return clientset, nil diff --git a/pkg/controller/agent_type_finder.go b/pkg/controller/agent_type_finder.go index b8b4708..8cff4be 100644 --- a/pkg/controller/agent_type_finder.go +++ b/pkg/controller/agent_type_finder.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "strings" "time" "github.com/dgraph-io/ristretto" @@ -14,10 +15,10 @@ import ( ) type AgentType struct { - SecretName string - AgentTypeName string - RegistrationToken string - PodSpecConfigMap string + SecretName string + AgentTypeName string + RegistrationToken string + AgentStartupParameters []string } // We cache the agent type secret information @@ -117,15 +118,18 @@ func (f *AgentTypeFinder) secretToAgentType(secret *v1.Secret) (*AgentType, erro return nil, fmt.Errorf("no registrationToken field in secret '%s'", secret.GetName()) } - podSpecConfigMap, ok := secret.Data["podSpecConfigMap"] - if !ok { - podSpecConfigMap = []byte{} + agentStartupParameters := []string{} + if parameters, ok := secret.Data["agentStartupParameters"]; ok && string(parameters) != "" { + for _, v := range strings.Split(string(parameters), " ") { + parameter := strings.Trim(strings.Trim(v, "\n"), " ") + agentStartupParameters = append(agentStartupParameters, parameter) + } } return &AgentType{ - SecretName: secret.GetName(), - AgentTypeName: string(agentTypeName), - RegistrationToken: string(registrationToken), - PodSpecConfigMap: string(podSpecConfigMap), + SecretName: secret.GetName(), + AgentTypeName: string(agentTypeName), + RegistrationToken: string(registrationToken), + AgentStartupParameters: agentStartupParameters, }, nil } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index c541cd5..3cc78c3 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -15,11 +15,12 @@ import ( ) type Config struct { - Namespace string - ServiceAccountName string - AgentImage string - MaxParallelJobs int - SemaphoreEndpoint string + Namespace string + ServiceAccountName string + AgentImage string + AgentStartupParameters []string + MaxParallelJobs int + SemaphoreEndpoint string } type Controller struct { @@ -224,25 +225,15 @@ func (c *Controller) buildJob(job semaphore.JobRequest, agentTypes []*AgentType) ObjectMeta: v1.ObjectMeta{ Name: c.jobName(job.JobID), Namespace: c.cfg.Namespace, - Labels: map[string]string{ - "app": "semaphore", - "semaphoreci.com/agent-type": job.MachineType, - }, + Labels: c.buildLabels(job), }, Spec: batchv1.JobSpec{ Parallelism: ¶llelism, Completions: ¶llelism, BackoffLimit: &retries, ActiveDeadlineSeconds: &activeDeadlineSeconds, - // Selector: ???, - // TTLSecondsAfterFinished: ???, Template: corev1.PodTemplateSpec{ - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app": "semaphore", - "semaphoreci.com/agent-type": job.MachineType, - }, - }, + ObjectMeta: v1.ObjectMeta{Labels: c.buildLabels(job)}, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, ServiceAccountName: c.cfg.ServiceAccountName, @@ -255,19 +246,7 @@ func (c *Controller) buildJob(job semaphore.JobRequest, agentTypes []*AgentType) "/opt/semaphore/agent", "start", }, - Args: []string{ - "--endpoint", - c.cfg.SemaphoreEndpoint, - // TODO: do not pass registration token in plain text like this, use environment variable - "--token", - agentType.RegistrationToken, - "--name-from-env", - "KUBERNETES_POD_NAME", - "--job-id", - job.JobID, - "--kubernetes-executor", - "--disconnect-after-job", - }, + Args: c.buildAgentStartupParameters(agentType, job.JobID), Env: []corev1.EnvVar{ { Name: "KUBERNETES_NAMESPACE", @@ -289,3 +268,34 @@ func (c *Controller) buildJob(job semaphore.JobRequest, agentTypes []*AgentType) }, }, nil } + +func (c *Controller) buildLabels(job semaphore.JobRequest) map[string]string { + return map[string]string{ + "app": "semaphore", + "semaphoreci.com/agent-type": job.MachineType, + } +} + +// TODO: do not pass registration token in plain text like this, use environment variable +func (c *Controller) buildAgentStartupParameters(agentType *AgentType, jobID string) []string { + parameters := []string{ + "--kubernetes-executor", + "--disconnect-after-job", + "--name-from-env", + "KUBERNETES_POD_NAME", + "--endpoint", + c.cfg.SemaphoreEndpoint, + "--token", + agentType.RegistrationToken, + "--job-id", + jobID, + } + + // If agent type does not specify startup parameters, use the controller's defaults. + if len(agentType.AgentStartupParameters) == 0 { + return append(parameters, c.cfg.AgentStartupParameters...) + } + + // Otherwise, use the agent type's startup parameters. + return append(parameters, agentType.AgentStartupParameters...) +}