Allow to define custom scripts in independent Elastic Agents (#1822)

Allow to define custom scripts in independent Elastic Agents. This
allows developers to customize Elastic Agent bootstrap for instance
installing new dependencies or libraries.

.buildkite/pipeline.trigger.integration.tests.sh

@@ -3,6 +3,10 @@
 # exit immediately on failure, or if an undefined variable is used
 set -eu
 
+echoerr() {
+    echo "$@" 1>&2
+}
+
 # begin the pipeline.yml file
 echo "steps:"
 echo "  - group: \":terminal: Integration test suite\""
@@ -89,6 +93,17 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
         label_suffix=" (independent agent)"
     fi
     package_name=$(basename "${package}")
+
+    if [[ "$independent_agent" == "false" && "$package_name" == "oracle" ]]; then
+        echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing required software)."
+        continue
+    fi
+
+    if [[ "$independent_agent" == "false" && "$package_name" == "custom_entrypoint" ]]; then
+        echoerr "Package \"${package_name}\" skipped: not supported with Elastic Agent running in the stack (missing required files deployed in provisioning)."
+        continue
+    fi
+
     echo "      - label: \":go: Integration test: ${package_name}${label_suffix}\""
     echo "        key: \"integration-parallel-${package_name}-agent-${independent_agent}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-parallel -p ${package_name}"

internal/agentdeployer/_static/custom-entrypoint.sh.tmpl

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -eu
+
+# Pre-start commands
+# Currently, these commands must be a "sh" script
+{{ fact "pre_start_script_contents" }}
+
+# Required to add "-s" since tini is not running as PID 1
+/usr/bin/tini -s -- /usr/local/bin/docker-entrypoint $@

internal/agentdeployer/_static/docker-agent-base.yml.tmpl

@@ -1,35 +1,44 @@
+{{- $user := fact "user" -}}
+{{- $capabilities:= fact "capabilities" -}}
+{{- $pid_mode := fact "pid_mode" -}}
+{{- $ports := fact "ports" -}}
+{{- $dockerfile_hash := fact "dockerfile_hash" -}}
+{{- $stack_version := fact "stack_version" }}
 services:
   elastic-agent:
     hostname: ${AGENT_HOSTNAME}
+    {{ if ne $dockerfile_hash "" }}
+    image: "elastic-package-test-elastic-agent-complete:{{ $stack_version }}-{{ $dockerfile_hash }}"
+    build:
+      context: .
+      args:
+        ES_AGENT_IMAGE: "${ELASTIC_AGENT_IMAGE_REF}"
+      dockerfile: "Dockerfile"
+    {{ else }}
     image: "${ELASTIC_AGENT_IMAGE_REF}"
+    {{ end }}
     healthcheck:
       test: "elastic-agent status"
       retries: 180
       interval: 1s
-    {{ if ne .pidMode "" }}
-    pid: {{ .pidMode }}
+    {{ if ne $pid_mode "" }}
+    pid: {{ $pid_mode }}
     {{ end }}
-    {{ if ne .user "" }}
-    user: {{ .user }}
+    {{ if ne $user "" }}
+    user: {{ $user }}
     {{ end }}
-    {{ if .capabilities }}
-    cap_add:
-    {{- range .capabilities }}
-      - {{ . }}
-    {{- end }}
+    {{ if $capabilities }}
+    cap_add: [{{ $capabilities }}]
     {{ end }}
     cap_drop:
       - ALL
-    {{ if .ports }}
-    ports:
-    {{- range .ports }}
-      - "{{ . }}"
-    {{- end }}
+    {{ if ne $ports "" }}
+    ports: [{{ $ports }}]
     {{ end }}
     environment:
       - FLEET_ENROLL=1
-      - FLEET_URL=https://fleet-server:8220
-      - KIBANA_HOST=https://kibana:5601
+      - FLEET_URL={{ fact "fleet_url" }}
+      - KIBANA_HOST={{ fact "kibana_host" }}
       - FLEET_TOKEN_POLICY_NAME=${FLEET_TOKEN_POLICY_NAME}
     volumes:
       - type: bind

internal/agentdeployer/_static/dockerfile.tmpl

@@ -0,0 +1,32 @@
+ARG ES_AGENT_IMAGE
+FROM $ES_AGENT_IMAGE
+
+{{- $pre_start_script_contents := fact "pre_start_script_contents" }}
+{{- $provisioning_script_filename := fact "provisioning_script_filename" }}
+{{- $provisioning_script_language := fact "provisioning_script_language" }}
+{{- $provisioning_script_contents := fact "provisioning_script_contents" }}
+{{- $entrypoint_script_filename := fact "entrypoint_script_filename" }}
+{{- $agent_name := fact "agent_name" }}
+
+{{- $entrypoint_path := (print "/usr/local/bin/" $agent_name "-entrypoint.sh") }}
+
+USER root
+
+{{ if ne $provisioning_script_contents "" }}
+COPY {{ $provisioning_script_filename }} .
+RUN chmod u+x {{ $provisioning_script_filename }} && \
+  {{ $provisioning_script_language }} ./{{ $provisioning_script_filename }} && \
+  rm {{ $provisioning_script_filename }}
+{{ end }}
+
+{{ if ne $pre_start_script_contents "" }}
+COPY {{ $entrypoint_script_filename }} {{ $entrypoint_path }}
+RUN chmod a+x {{ $entrypoint_path }}
+{{ end }}
+
+USER elastic-agent
+
+{{ if ne $pre_start_script_contents "" }}
+ENTRYPOINT ["{{ $entrypoint_path }}"]
+CMD [""]
+{{ end }}

internal/agentdeployer/agent.go

@@ -6,13 +6,17 @@ package agentdeployer
 
 import (
 	"context"
-	_ "embed"
+	"crypto/md5"
+	"embed"
+	"encoding/hex"
 	"fmt"
 	"os"
 	"path/filepath"
-	"text/template"
+	"strings"
 	"time"
 
+	"github.com/elastic/go-resource"
+
 	"github.com/elastic/elastic-package/internal/compose"
 	"github.com/elastic/elastic-package/internal/docker"
 	"github.com/elastic/elastic-package/internal/files"
@@ -25,11 +29,16 @@ import (
 const (
 	dockerTestAgentServiceName   = "elastic-agent"
 	dockerTestAgentDockerCompose = "docker-agent-base.yml"
+	dockerTestAgentDockerfile    = "Dockerfile"
+	customScriptFilename         = "script.sh"
+	customEntrypointFilename     = "custom-entrypoint.sh"
 	defaultAgentPolicyName       = "Elastic-Agent (elastic-package)"
 )
 
-//go:embed _static/docker-agent-base.yml.tmpl
-var dockerTestAgentDockerComposeTemplate string
+//go:embed _static
+var static embed.FS
+
+var staticSource = resource.NewSourceFS(static)
 
 // CustomAgentDeployer knows how to deploy a custom elastic-agent defined via
 // a Docker Compose file.
@@ -111,7 +120,7 @@ func (d *DockerComposeAgentDeployer) SetUp(ctx context.Context, agentInfo AgentI
 		fmt.Sprintf("%s=%s", agentHostnameEnv, d.agentHostname()),
 	)
 
-	configDir, err := d.installDockerfile(agentInfo)
+	configDir, err := d.installDockerCompose(agentInfo)
 	if err != nil {
 		return nil, fmt.Errorf("could not create resources for custom agent: %w", err)
 	}
@@ -233,36 +242,117 @@ func (d *DockerComposeAgentDeployer) agentName() string {
 	return name
 }
 
-// installDockerfile creates the files needed to run the custom elastic agent and returns
+// installDockerCompose creates the files needed to run the custom elastic agent and returns
 // the directory with these files.
-func (d *DockerComposeAgentDeployer) installDockerfile(agentInfo AgentInfo) (string, error) {
+func (d *DockerComposeAgentDeployer) installDockerCompose(agentInfo AgentInfo) (string, error) {
 	customAgentDir, err := CreateDeployerDir(d.profile, fmt.Sprintf("docker-agent-%s-%s", d.agentName(), d.agentRunID))
 	if err != nil {
 		return "", fmt.Errorf("failed to create directory for custom agent files: %w", err)
 	}
 
-	customAgentDockerfile := filepath.Join(customAgentDir, dockerTestAgentDockerCompose)
-	file, err := os.Create(customAgentDockerfile)
-	if err != nil {
-		return "", fmt.Errorf("failed to create file (name %s): %w", customAgentDockerfile, err)
+	hashDockerfile := []byte{}
+	if agentInfo.Agent.ProvisioningScript.Contents != "" || agentInfo.Agent.PreStartScript.Contents != "" {
+		err = d.installDockerfileResources(agentInfo.Agent.AgentSettings, customAgentDir)
+		if err != nil {
+			return "", fmt.Errorf("failed to create dockerfile resources: %w", err)
+		}
+		hashDockerfile, err = hashFile(filepath.Join(customAgentDir, dockerTestAgentDockerfile))
+		if err != nil {
+			return "", fmt.Errorf("failed to obtain has for Elastic Agent Dockerfile: %w", err)
+		}
 	}
-	defer file.Close()
-
-	tmpl := template.Must(template.New(dockerTestAgentDockerCompose).Parse(dockerTestAgentDockerComposeTemplate))
-	err = tmpl.Execute(file, map[string]any{
-		"user":         agentInfo.Agent.User,
-		"capabilities": agentInfo.Agent.LinuxCapabilities,
-		"runtime":      agentInfo.Agent.Runtime,
-		"pidMode":      agentInfo.Agent.PidMode,
-		"ports":        agentInfo.Agent.Ports,
+
+	resourceManager := resource.NewManager()
+	resourceManager.AddFacter(resource.StaticFacter{
+		"user":            agentInfo.Agent.User,
+		"capabilities":    strings.Join(agentInfo.Agent.LinuxCapabilities, ","),
+		"runtime":         agentInfo.Agent.Runtime,
+		"pid_mode":        agentInfo.Agent.PidMode,
+		"ports":           strings.Join(agentInfo.Agent.Ports, ","),
+		"kibana_host":     "https://kibana:5601",
+		"fleet_url":       "https://fleet-server:8220",
+		"dockerfile_hash": hex.EncodeToString(hashDockerfile),
+		"stack_version":   d.stackVersion,
 	})
+
+	resourceManager.RegisterProvider("file", &resource.FileProvider{
+		Prefix: customAgentDir,
+	})
+
+	agentResources := []resource.Resource{
+		&resource.File{
+			Path:    dockerTestAgentDockerCompose,
+			Content: staticSource.Template("_static/docker-agent-base.yml.tmpl"),
+		},
+	}
+	results, err := resourceManager.Apply(agentResources)
 	if err != nil {
-		return "", fmt.Errorf("failed to create contents of the docker-compose file %q: %w", customAgentDockerfile, err)
+		return "", fmt.Errorf("%w: %s", err, processApplyErrors(results))
 	}
 
 	return customAgentDir, nil
 }
 
+func (d *DockerComposeAgentDeployer) installDockerfileResources(agentSettings AgentSettings, folder string) error {
+	agentResources := []resource.Resource{
+		&resource.File{
+			Path:    dockerTestAgentDockerfile,
+			Content: staticSource.Template("_static/dockerfile.tmpl"),
+		},
+	}
+	if agentSettings.ProvisioningScript.Contents != "" {
+		agentResources = append(agentResources, &resource.File{
+			Path:    customScriptFilename,
+			Mode:    resource.FileMode(0o755),
+			Content: resource.FileContentLiteral(agentSettings.ProvisioningScript.Contents),
+		})
+	}
+	if agentSettings.PreStartScript.Contents != "" {
+		agentResources = append(agentResources, &resource.File{
+			Path:    customEntrypointFilename,
+			Mode:    resource.FileMode(0o755),
+			Content: staticSource.Template("_static/custom-entrypoint.sh.tmpl"),
+		})
+	}
+	resourceManager := resource.NewManager()
+	resourceManager.AddFacter(resource.StaticFacter{
+		"provisioning_script_contents": agentSettings.ProvisioningScript.Contents,
+		"provisioning_script_language": agentSettings.ProvisioningScript.Language,
+		"provisioning_script_filename": customScriptFilename,
+		"pre_start_script_contents":    agentSettings.PreStartScript.Contents,
+		"entrypoint_script_filename":   customEntrypointFilename,
+		"agent_name":                   d.agentName(),
+	})
+
+	resourceManager.RegisterProvider("file", &resource.FileProvider{
+		Prefix: folder,
+	})
+	results, err := resourceManager.Apply(agentResources)
+	if err != nil {
+		return fmt.Errorf("%w: %s", err, processApplyErrors(results))
+	}
+	return nil
+}
+
+func hashFile(path string) ([]byte, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return []byte{}, err
+	}
+	dockerfileMD5 := md5.Sum(data)
+	return dockerfileMD5[:], nil
+}
+
+func processApplyErrors(results resource.ApplyResults) string {
+	var errors []string
+	for _, result := range results {
+		if err := result.Err(); err != nil {
+			errors = append(errors, err.Error())
+		}
+	}
+	return strings.Join(errors, ", ")
+}
+
 // ExitCode returns true if the agent is exited and its exit code.
 func (s *dockerComposeDeployedAgent) ExitCode(ctx context.Context) (bool, int, error) {
 	p, err := compose.NewProject(s.project, s.ymlPaths...)

internal/agentdeployer/info.go

@@ -11,8 +11,39 @@ const (
 	fleetPolicyEnv      = "FLEET_TOKEN_POLICY_NAME"
 	agentHostnameEnv    = "AGENT_HOSTNAME"
 	elasticAgentTagsEnv = "ELASTIC_AGENT_TAGS"
+
+	DefaultAgentRuntime             = "docker"
+	DefaultAgentProgrammingLanguage = "sh"
 )
 
+type AgentScript struct {
+	// Language defines the programming language used for the script.
+	Language string `config:"language"`
+	// Contents is the code script.
+	Contents string `config:"contents"`
+}
+
+type AgentSettings struct {
+	// User user to run Elastic Agent process
+	User string `config:"user"`
+	// PidMode selects the host PID mode
+	// (From docker-compose docs) Turns on sharing between container and the host
+	// operating system the PID address space
+	PidMode string `config:"pid_mode"`
+	// Runtime is the selected runtime to run the Elastic Agent process
+	Runtime string `config:"runtime"`
+	// LinuxCapabilities is a list of the capabilities needed to run the Elastic Agent process
+	LinuxCapabilities []string `config:"linux_capabilities"`
+	// Ports is a list of ports to make them available to communicate to the Elastic Agent process
+	Ports []string `config:"ports"`
+	// ProvisioningScript allows to define a script to modify Elastic Agent environment with the required
+	// libraries or dependencies (container, vm, ...),
+	ProvisioningScript AgentScript `config:"provisioning_script"`
+	// PreStartScript allows to define a script to update/modify Elastic Agent process (container, vm, ...)
+	// Example update environment variables like PATH
+	PreStartScript AgentScript `config:"pre_start_script"`
+}
+
 // AgentInfo encapsulates context that is both available to a AgentDeployer and
 // populated by a DeployedAgent. The fields in AgentInfo may be used in handlebars
 // templates in system test configuration files, for example: {{ Hostname }}.
@@ -72,18 +103,7 @@ type AgentInfo struct {
 			NamePrefix string
 		}
 
-		// User user to run Elastic Agent process
-		User string
-		// PidMode selects the host PID mode
-		// (From docker-compose docs) Turns on sharing between container and the host
-		// operating system the PID address space
-		PidMode string
-		// Runtime is the selected runtime to run the Elastic Agent process
-		Runtime string
-		// LinuxCapabilities is a list of the capabilities needed to run the Elastic Agent process
-		LinuxCapabilities []string
-		// Ports is a list of ports to make them available to communicate to the Elastic Agent process
-		Ports []string
+		AgentSettings
 	}
 
 	// CustomProperties store additional data used to boot up the service, e.g. AWS credentials.

internal/testrunner/runners/system/runner.go

@@ -367,11 +367,8 @@ func (r *runner) createAgentInfo(policy *kibana.Policy, config *testConfig, runI
 	info.Policy.Name = policy.Name
 	info.Policy.ID = policy.ID
 
-	info.Agent.User = config.Agent.User
-	info.Agent.LinuxCapabilities = config.Agent.LinuxCapabilities
-	info.Agent.Runtime = config.Agent.Runtime
-	info.Agent.PidMode = config.Agent.PidMode
-	info.Agent.Ports = config.Agent.Ports
+	// Copy all agent settings from the test configuration file
+	info.Agent.AgentSettings = config.Agent.AgentSettings
 
 	// If user is defined in the configuration file, it has preference
 	// and it should not be overwritten by the value in the manifest