Move service container status to a file

Author: mrodm

.buildkite/pipeline.trigger.integration.tests.sh

@@ -44,6 +44,7 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "          - build/test-results/*.xml"
     echo "          - build/elastic-stack-dump/check-*/logs/*.log"
     echo "          - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*"
+    echo "          - build/container-status/*.log"
     if [[ $test =~ with-kind$ ]]; then
         echo "          - build/kubectl-dump.txt"
     fi
@@ -61,6 +62,7 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
+    echo "          - build/container-status/*.log"
 done
 
 popd > /dev/null

internal/benchrunner/runners/system/servicedeployer/compose.go

@@ -5,6 +5,7 @@
 package servicedeployer
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -73,7 +74,10 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer
 	if err != nil {
 		return nil, fmt.Errorf("could not boot up service using Docker Compose: %w", err)
 	}
-	err = p.WaitForHealthy(opts)
+	statuses, err := p.WaitForHealthy(opts)
+	if statusErr := writeServiceContainerStatus(outCtxt.Name, statuses); statusErr != nil {
+		logger.Errorf("failed to create container status file: %v", statusErr)
+	}
 	if err != nil {
 		processServiceContainerLogs(p, compose.CommandOptions{
 			Env: opts.Env,
@@ -205,11 +209,38 @@ func writeServiceContainerLogs(serviceName string, content []byte) error {
 		return fmt.Errorf("can't create directory for service container logs (path: %s): %w", containerLogsDir, err)
 	}
 
-	containerLogsFilepath := filepath.Join(containerLogsDir, fmt.Sprintf("%s-%d.log", serviceName, time.Now().UnixNano()))
+	containerLogsFilepath := filepath.Join(containerLogsDir, fmt.Sprintf("benchmark-%s-%d.log", serviceName, time.Now().UnixNano()))
 	logger.Infof("Write container logs to file: %s", containerLogsFilepath)
 	err = os.WriteFile(containerLogsFilepath, content, 0644)
 	if err != nil {
 		return fmt.Errorf("can't write container logs to file (path: %s): %w", containerLogsFilepath, err)
 	}
 	return nil
 }
+
+func writeServiceContainerStatus(serviceName string, content [][]byte) error {
+	if len(content) == 0 {
+		return nil
+	}
+
+	buildDir, err := builder.BuildDirectory()
+	if err != nil {
+		return fmt.Errorf("locating build directory failed: %w", err)
+	}
+
+	containerStatusDir := filepath.Join(buildDir, "container-status")
+	err = os.MkdirAll(containerStatusDir, 0755)
+	if err != nil {
+		return fmt.Errorf("can't create directory for service container status (path: %s): %w", containerStatusDir, err)
+	}
+
+	containerStatusFilepath := filepath.Join(containerStatusDir, fmt.Sprintf("benchmark-%s-%d.log", serviceName, time.Now().UnixNano()))
+	logger.Infof("Write container status to file: %s", containerStatusFilepath)
+
+	contentFile := bytes.Join(content, []byte("\n"))
+	err = os.WriteFile(containerStatusFilepath, contentFile, 0644)
+	if err != nil {
+		return fmt.Errorf("can't write container status to file (path: %s): %w", containerStatusFilepath, err)
+	}
+	return nil
+}

internal/compose/compose.go

@@ -314,28 +314,29 @@ func (p *Project) Logs(opts CommandOptions) ([]byte, error) {
 }
 
 // WaitForHealthy method waits until all containers are healthy.
-func (p *Project) WaitForHealthy(opts CommandOptions) error {
+func (p *Project) WaitForHealthy(opts CommandOptions) ([][]byte, error) {
 	// Read container IDs
 	args := p.baseArgs()
 	args = append(args, "ps")
 	args = append(args, "-q")
 
 	var b bytes.Buffer
 	if err := p.runDockerComposeCmd(dockerComposeOptions{args: args, env: opts.Env, stdout: &b}); err != nil {
-		return err
+		return nil, err
 	}
 
 	startTime := time.Now()
 	timeout := startTime.Add(waitForHealthyTimeout)
 
 	containerIDs := strings.Fields(b.String())
+	containerStatuses := [][]byte{}
 	for {
 		if time.Now().After(timeout) {
-			return errors.New("timeout waiting for healthy container")
+			return containerStatuses, errors.New("timeout waiting for healthy container")
 		}
 
 		if signal.SIGINT() {
-			return errors.New("SIGINT: cancel waiting for policy assigned")
+			return containerStatuses, errors.New("SIGINT: cancel waiting for policy assigned")
 		}
 
 		// NOTE: healthy must be reinitialized at each iteration
@@ -344,11 +345,11 @@ func (p *Project) WaitForHealthy(opts CommandOptions) error {
 		logger.Debugf("Wait for healthy containers: %s", strings.Join(containerIDs, ","))
 		descriptions, err := docker.InspectContainers(containerIDs...)
 		if err != nil {
-			return err
+			return containerStatuses, err
 		}
 
 		for _, containerDescription := range descriptions {
-			logger.Debugf("Container status: %s", containerDescription.String())
+			containerStatuses = append(containerStatuses, []byte(containerDescription.String()))
 
 			// No healthcheck defined for service
 			if containerDescription.State.Status == "running" && containerDescription.State.Health == nil {
@@ -367,7 +368,7 @@ func (p *Project) WaitForHealthy(opts CommandOptions) error {
 
 			// Container exited with code > 0
 			if containerDescription.State.Status == "exited" && containerDescription.State.ExitCode > 0 {
-				return fmt.Errorf("container (ID: %s) exited with code %d", containerDescription.ID, containerDescription.State.ExitCode)
+				return containerStatuses, fmt.Errorf("container (ID: %s) exited with code %d", containerDescription.ID, containerDescription.State.ExitCode)
 			}
 
 			// Any different status is considered unhealthy
@@ -383,7 +384,7 @@ func (p *Project) WaitForHealthy(opts CommandOptions) error {
 		time.Sleep(waitForHealthyInterval)
 	}
 
-	return nil
+	return containerStatuses, nil
 }
 
 func (p *Project) baseArgs() []string {

internal/testrunner/runners/system/servicedeployer/compose.go

@@ -5,6 +5,7 @@
 package servicedeployer
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -89,7 +90,10 @@ func (d *DockerComposeServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedSer
 		return nil, errors.Wrap(err, "could not boot up service using Docker Compose")
 	}
 
-	err = p.WaitForHealthy(opts)
+	statuses, err := p.WaitForHealthy(opts)
+	if statusErr := writeServiceContainerStatus(outCtxt.Name, statuses); statusErr != nil {
+		logger.Errorf("failed to create container status file: %v", statusErr)
+	}
 	if err != nil {
 		processServiceContainerLogs(p, compose.CommandOptions{
 			Env: opts.Env,
@@ -234,3 +238,30 @@ func writeServiceContainerLogs(serviceName string, content []byte) error {
 	}
 	return nil
 }
+
+func writeServiceContainerStatus(serviceName string, content [][]byte) error {
+	if len(content) == 0 {
+		return nil
+	}
+
+	buildDir, err := builder.BuildDirectory()
+	if err != nil {
+		return fmt.Errorf("locating build directory failed: %w", err)
+	}
+
+	containerStatusDir := filepath.Join(buildDir, "container-status")
+	err = os.MkdirAll(containerStatusDir, 0755)
+	if err != nil {
+		return fmt.Errorf("can't create directory for service container status (path: %s): %w", containerStatusDir, err)
+	}
+
+	containerStatusFilepath := filepath.Join(containerStatusDir, fmt.Sprintf("%s-%d.log", serviceName, time.Now().UnixNano()))
+	logger.Infof("Write container status to file: %s", containerStatusFilepath)
+
+	contentFile := bytes.Join(content, []byte("\n"))
+	err = os.WriteFile(containerStatusFilepath, contentFile, 0644)
+	if err != nil {
+		return fmt.Errorf("can't write container status to file (path: %s): %w", containerStatusFilepath, err)
+	}
+	return nil
+}

internal/testrunner/runners/system/servicedeployer/custom_agent.go

@@ -129,7 +129,10 @@ func (d *CustomAgentDeployer) SetUp(inCtxt ServiceContext) (DeployedService, err
 		return nil, errors.Wrapf(err, "can't attach service container to the stack network")
 	}
 
-	err = p.WaitForHealthy(opts)
+	statuses, err := p.WaitForHealthy(opts)
+	if statusErr := writeServiceContainerStatus(outCtxt.Name, statuses); statusErr != nil {
+		logger.Errorf("failed to create container status file: %v", statusErr)
+	}
 	if err != nil {
 		processServiceContainerLogs(p, compose.CommandOptions{
 			Env: opts.Env,

internal/testrunner/runners/system/servicedeployer/terraform.go

@@ -148,7 +148,10 @@ func (tsd TerraformServiceDeployer) SetUp(inCtxt ServiceContext) (DeployedServic
 		return nil, errors.Wrap(err, "could not boot up service using Docker Compose")
 	}
 
-	err = p.WaitForHealthy(opts)
+	statuses, err := p.WaitForHealthy(opts)
+	if statusErr := writeServiceContainerStatus(outCtxt.Name, statuses); statusErr != nil {
+		logger.Errorf("failed to create container status file: %v", statusErr)
+	}
 	if err != nil {
 		processServiceContainerLogs(p, compose.CommandOptions{
 			Env: opts.Env,