Improve health check for Fleet Server (#1824)

Fleet Server restarts once after being healthy. Agents enrolling during this restart fail and need to be restarted. Improve the healthcheck so it expects to have three healthy healthchecks before considering Fleet Server ready to enroll.
elastic · May 8, 2024 · db4c611 · db4c611
1 parent e53417a
commit db4c611
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 7 deletions.
diff --git a/internal/stack/_static/docker-compose-stack.yml.tmpl b/internal/stack/_static/docker-compose-stack.yml.tmpl
@@ -94,7 +94,7 @@ services:
       kibana:
         condition: service_healthy
     healthcheck:
-      test: "curl --cacert /etc/ssl/elastic-agent/ca-cert.pem -f https://localhost:8220/api/status | grep -i healthy 2>&1 >/dev/null"
+      test: "bash /healthcheck.sh"
       start_period: 60s
       interval: 5s
     hostname: docker-fleet-server
@@ -112,8 +112,9 @@ services:
     - "KIBANA_FLEET_SETUP=1"
     - "KIBANA_HOST={{ fact "kibana_host" }}"
     volumes:
-      - "../certs/ca-cert.pem:/etc/ssl/certs/elastic-package.pem"
-      - "../certs/fleet-server:/etc/ssl/elastic-agent"
+      - "../certs/ca-cert.pem:/etc/ssl/certs/elastic-package.pem:ro"
+      - "../certs/fleet-server:/etc/ssl/elastic-agent:ro"
+      - "./fleet-server-healthcheck.sh:/healthcheck.sh:ro"
     ports:
       - "127.0.0.1:8220:8220"
       {{ if eq $apm_enabled "true" }}

diff --git a/internal/stack/_static/fleet-server-healthcheck.sh b/internal/stack/_static/fleet-server-healthcheck.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+healthcheck() {
+    curl --cacert /etc/ssl/elastic-agent/ca-cert.pem -f https://localhost:8220/api/status | grep -i healthy 2>&1 >/dev/null
+}
+
+# Fleet Server can restart after announcing to be healthy, agents connecting during this restart will
+# fail to enroll. Expect 3 healthy healthchecks before considering it healthy.
+expected=3
+for i in $(seq $expected); do
+	healthcheck
+	sleep 1
+done
+
+exit 0
diff --git a/.../stack/_static/kibana_healthcheck.sh.tmpl → .../stack/_static/kibana-healthcheck.sh.tmpl b/.../stack/_static/kibana_healthcheck.sh.tmpl → .../stack/_static/kibana-healthcheck.sh.tmpl
diff --git a/internal/stack/boot.go b/internal/stack/boot.go
@@ -94,10 +94,14 @@ func BootUp(ctx context.Context, options Options) error {
 		// As a workaround, try to give another chance to docker-compose if only
 		// elastic-agent failed.
 		if onlyElasticAgentFailed(ctx, options) && !errors.Is(err, context.Canceled) {
-			sleepTime := 10 * time.Second
+			sleepTime := 2 * time.Second
 			fmt.Printf("Elastic Agent failed to start, trying again in %s.\n", sleepTime)
-			time.Sleep(sleepTime)
-			err = dockerComposeUp(ctx, options)
+			select {
+			case <-time.After(sleepTime):
+				err = dockerComposeUp(ctx, options)
+			case <-ctx.Done():
+				err = ctx.Err()
+			}
 		}
 		if err != nil {
 			return fmt.Errorf("running docker-compose failed: %w", err)

diff --git a/internal/stack/resources.go b/internal/stack/resources.go
@@ -39,6 +39,9 @@ const (
 	// KibanaHealthcheckFile is the kibana healthcheck.
 	KibanaHealthcheckFile = "kibana_healthcheck.sh"
 
+	// FleetServerHealthcheckFile is the Fleet Server healthcheck.
+	FleetServerHealthcheckFile = "fleet-server-healthcheck.sh"
+
 	// PackageRegistryConfigFile is the config file for the Elastic Package registry
 	PackageRegistryConfigFile = "package-registry.yml"
 
@@ -107,7 +110,11 @@ var (
 		},
 		&resource.File{
 			Path:    KibanaHealthcheckFile,
-			Content: staticSource.Template("_static/kibana_healthcheck.sh.tmpl"),
+			Content: staticSource.Template("_static/kibana-healthcheck.sh.tmpl"),
+		},
+		&resource.File{
+			Path:    FleetServerHealthcheckFile,
+			Content: staticSource.File("_static/fleet-server-healthcheck.sh"),
 		},
 		&resource.File{
 			Path:    PackageRegistryConfigFile,