feat: support for agent startup parameters (#2)

.gitignore

@@ -0,0 +1 @@
+build/

Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:3.14
+COPY build/controller /
+ENTRYPOINT ["/controller"]

Makefile

@@ -0,0 +1,39 @@
+.PHONY: build test
+
+REGISTRY=semaphoreci/controller
+LATEST_VERSION=$(shell git tag | sort --version-sort | tail -n 1)
+
+SECURITY_TOOLBOX_BRANCH ?= master
+SECURITY_TOOLBOX_TMP_DIR ?= /tmp/security-toolbox
+
+check.prepare:
+	rm -rf $(SECURITY_TOOLBOX_TMP_DIR)
+	git clone git@github.com:renderedtext/security-toolbox.git $(SECURITY_TOOLBOX_TMP_DIR) && (cd $(SECURITY_TOOLBOX_TMP_DIR) && git checkout $(SECURITY_TOOLBOX_BRANCH) && cd -)
+
+check.static: check.prepare
+	docker run -it -v $$(pwd):/app \
+		-v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \
+		registry.semaphoreci.com/ruby:2.7 \
+		bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/code --language go -d'
+
+check.deps: check.prepare
+	docker run -it -v $$(pwd):/app \
+		-v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \
+		registry.semaphoreci.com/ruby:2.7 \
+		bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/dependencies --language go -d'
+
+build:
+	rm -rf build
+	env GOOS=linux go build -o build/controller main.go
+
+docker.build: build
+	docker build -t $(REGISTRY):latest .
+
+docker.push:
+	@if [ -z "$(LATEST_VERSION)" ]; then \
+		docker push $(REGISTRY):latest; \
+	else \
+		docker tag $(REGISTRY):latest $(REGISTRY):$(LATEST_VERSION); \
+		docker push $(REGISTRY):$(LATEST_VERSION); \
+		docker push $(REGISTRY):latest; \
+	fi

README.md

@@ -18,6 +18,7 @@ export KUBERNETES_NAMESPACE=default
 export SEMAPHORE_AGENT_IMAGE=semaphoreci/agent:v2.2.14
 export KUBERNETES_SERVICE_ACCOUNT=semaphore-agent-svc-account
 export MAX_PARALLEL_JOBS=10
+export SEMAPHORE_AGENT_STARTUP_PARAMETERS='--kubernetes-executor-pod-spec WHATEVER --pre-job-hook-path /opt/semaphore/agent/hooks/pre-job.sh --source-pre-job-hook'
 
 # Build and start controller
 go build -o controller main.go

main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	"github.com/renderedtext/agent-k8s-stack/pkg/controller"
 	"github.com/renderedtext/agent-k8s-stack/pkg/semaphore"
@@ -72,12 +73,18 @@ func buildConfig(endpoint string) (*controller.Config, error) {
 		}
 	}
 
+	agentStartupParameters := []string{}
+	if os.Getenv("SEMAPHORE_AGENT_STARTUP_PARAMETERS") != "" {
+		agentStartupParameters = strings.Split(os.Getenv("SEMAPHORE_AGENT_STARTUP_PARAMETERS"), " ")
+	}
+
 	return &controller.Config{
-		SemaphoreEndpoint:  endpoint,
-		Namespace:          k8sNamespace,
-		ServiceAccountName: svcAccountName,
-		AgentImage:         agentImage,
-		MaxParallelJobs:    maxParallelJobs,
+		SemaphoreEndpoint:      endpoint,
+		Namespace:              k8sNamespace,
+		ServiceAccountName:     svcAccountName,
+		AgentImage:             agentImage,
+		AgentStartupParameters: agentStartupParameters,
+		MaxParallelJobs:        maxParallelJobs,
 	}, nil
 }
 
@@ -88,7 +95,7 @@ func newK8sClientset() (kubernetes.Interface, error) {
 
 		clientset, err = newClientsetFromConfig()
 		if err != nil {
-			return nil, fmt.Errorf("error creating kubernetes clientset: %v\n", err)
+			return nil, fmt.Errorf("error creating kubernetes clientset: %v", err)
 		}
 	}
 
@@ -98,18 +105,18 @@ func newK8sClientset() (kubernetes.Interface, error) {
 func newClientsetFromConfig() (kubernetes.Interface, error) {
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
-		return nil, fmt.Errorf("error getting user home directory: %v\n", err)
+		return nil, fmt.Errorf("error getting user home directory: %v", err)
 	}
 
 	kubeConfigPath := filepath.Join(homeDir, ".kube", "config")
 	kubeConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
 	if err != nil {
-		return nil, fmt.Errorf("error getting Kubernetes config: %v\n", err)
+		return nil, fmt.Errorf("error getting Kubernetes config: %v", err)
 	}
 
 	clientset, err := kubernetes.NewForConfig(kubeConfig)
 	if err != nil {
-		return nil, fmt.Errorf("error creating kubernetes clientset from config file: %v\n", err)
+		return nil, fmt.Errorf("error creating kubernetes clientset from config file: %v", err)
 	}
 
 	return clientset, nil

pkg/controller/agent_type_finder.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	"github.com/dgraph-io/ristretto"
@@ -14,10 +15,10 @@ import (
 )
 
 type AgentType struct {
-	SecretName        string
-	AgentTypeName     string
-	RegistrationToken string
-	PodSpecConfigMap  string
+	SecretName             string
+	AgentTypeName          string
+	RegistrationToken      string
+	AgentStartupParameters []string
 }
 
 // We cache the agent type secret information
@@ -117,15 +118,18 @@ func (f *AgentTypeFinder) secretToAgentType(secret *v1.Secret) (*AgentType, erro
 		return nil, fmt.Errorf("no registrationToken field in secret '%s'", secret.GetName())
 	}
 
-	podSpecConfigMap, ok := secret.Data["podSpecConfigMap"]
-	if !ok {
-		podSpecConfigMap = []byte{}
+	agentStartupParameters := []string{}
+	if parameters, ok := secret.Data["agentStartupParameters"]; ok && string(parameters) != "" {
+		for _, v := range strings.Split(string(parameters), " ") {
+			parameter := strings.Trim(strings.Trim(v, "\n"), " ")
+			agentStartupParameters = append(agentStartupParameters, parameter)
+		}
 	}
 
 	return &AgentType{
-		SecretName:        secret.GetName(),
-		AgentTypeName:     string(agentTypeName),
-		RegistrationToken: string(registrationToken),
-		PodSpecConfigMap:  string(podSpecConfigMap),
+		SecretName:             secret.GetName(),
+		AgentTypeName:          string(agentTypeName),
+		RegistrationToken:      string(registrationToken),
+		AgentStartupParameters: agentStartupParameters,
 	}, nil
 }

pkg/controller/controller.go

@@ -15,11 +15,12 @@ import (
 )
 
 type Config struct {
-	Namespace          string
-	ServiceAccountName string
-	AgentImage         string
-	MaxParallelJobs    int
-	SemaphoreEndpoint  string
+	Namespace              string
+	ServiceAccountName     string
+	AgentImage             string
+	AgentStartupParameters []string
+	MaxParallelJobs        int
+	SemaphoreEndpoint      string
 }
 
 type Controller struct {
@@ -224,25 +225,15 @@ func (c *Controller) buildJob(job semaphore.JobRequest, agentTypes []*AgentType)
 		ObjectMeta: v1.ObjectMeta{
 			Name:      c.jobName(job.JobID),
 			Namespace: c.cfg.Namespace,
-			Labels: map[string]string{
-				"app":                        "semaphore",
-				"semaphoreci.com/agent-type": job.MachineType,
-			},
+			Labels:    c.buildLabels(job),
 		},
 		Spec: batchv1.JobSpec{
 			Parallelism:           &parallelism,
 			Completions:           &parallelism,
 			BackoffLimit:          &retries,
 			ActiveDeadlineSeconds: &activeDeadlineSeconds,
-			// Selector: ???,
-			// TTLSecondsAfterFinished: ???,
 			Template: corev1.PodTemplateSpec{
-				ObjectMeta: v1.ObjectMeta{
-					Labels: map[string]string{
-						"app":                        "semaphore",
-						"semaphoreci.com/agent-type": job.MachineType,
-					},
-				},
+				ObjectMeta: v1.ObjectMeta{Labels: c.buildLabels(job)},
 				Spec: corev1.PodSpec{
 					RestartPolicy:                 corev1.RestartPolicyNever,
 					ServiceAccountName:            c.cfg.ServiceAccountName,
@@ -255,19 +246,7 @@ func (c *Controller) buildJob(job semaphore.JobRequest, agentTypes []*AgentType)
 								"/opt/semaphore/agent",
 								"start",
 							},
-							Args: []string{
-								"--endpoint",
-								c.cfg.SemaphoreEndpoint,
-								// TODO: do not pass registration token in plain text like this, use environment variable
-								"--token",
-								agentType.RegistrationToken,
-								"--name-from-env",
-								"KUBERNETES_POD_NAME",
-								"--job-id",
-								job.JobID,
-								"--kubernetes-executor",
-								"--disconnect-after-job",
-							},
+							Args: c.buildAgentStartupParameters(agentType, job.JobID),
 							Env: []corev1.EnvVar{
 								{
 									Name: "KUBERNETES_NAMESPACE",
@@ -289,3 +268,34 @@ func (c *Controller) buildJob(job semaphore.JobRequest, agentTypes []*AgentType)
 		},
 	}, nil
 }
+
+func (c *Controller) buildLabels(job semaphore.JobRequest) map[string]string {
+	return map[string]string{
+		"app":                        "semaphore",
+		"semaphoreci.com/agent-type": job.MachineType,
+	}
+}
+
+// TODO: do not pass registration token in plain text like this, use environment variable
+func (c *Controller) buildAgentStartupParameters(agentType *AgentType, jobID string) []string {
+	parameters := []string{
+		"--kubernetes-executor",
+		"--disconnect-after-job",
+		"--name-from-env",
+		"KUBERNETES_POD_NAME",
+		"--endpoint",
+		c.cfg.SemaphoreEndpoint,
+		"--token",
+		agentType.RegistrationToken,
+		"--job-id",
+		jobID,
+	}
+
+	// If agent type does not specify startup parameters, use the controller's defaults.
+	if len(agentType.AgentStartupParameters) == 0 {
+		return append(parameters, c.cfg.AgentStartupParameters...)
+	}
+
+	// Otherwise, use the agent type's startup parameters.
+	return append(parameters, agentType.AgentStartupParameters...)
+}