feat(fvt): add healthcheck, depends_on and --wait

Update the docker-compose.yml to include healthchecks for toxiproxy and kafka and make the FV startup wait for them to become healthy. Signed-off-by: Dominic Evans <dominic.evans@uk.ibm.com>
IBM · Aug 18, 2023 · 26792a3 · 26792a3
1 parent d2dba29
commit 26792a3
Show file tree

Hide file tree

Showing 3 changed files with 124 additions and 10 deletions.
diff --git a/.github/workflows/fvt.yml b/.github/workflows/fvt.yml
@@ -39,4 +39,9 @@ jobs:
       with:
         go-version: ${{ matrix.go-version }}
     - name: Test (Functional)
-      run: make test_functional
+      run: |
+        curl -sSL "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose
+        sudo install -m755 /tmp/docker-compose "$(dirname $(which docker-compose))"
+        docker version --format 'Docker Engine version v{{.Server.Version}}'
+        docker-compose version
+        make test_functional
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -41,6 +41,26 @@ services:
       args:
         KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
         SCALA_VERSION: ${SCALA_VERSION:-2.13}
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '--bootstrap-server',
+          'kafka-1:9091',
+          '--broker',
+          '1',
+          '--describe',
+        ]
+      interval: 15s
+      timeout: 15s
+      retries: 10
+      start_period: 360s
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+      - toxiproxy
     restart: always
     environment:
       KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
@@ -66,6 +86,26 @@ services:
       args:
         KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
         SCALA_VERSION: ${SCALA_VERSION:-2.13}
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '--bootstrap-server',
+          'kafka-2:9091',
+          '--broker',
+          '2',
+          '--describe',
+        ]
+      interval: 15s
+      timeout: 15s
+      retries: 10
+      start_period: 360s
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+      - toxiproxy
     restart: always
     environment:
       KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
@@ -91,6 +131,26 @@ services:
       args:
         KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
         SCALA_VERSION: ${SCALA_VERSION:-2.13}
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '--bootstrap-server',
+          'kafka-3:9091',
+          '--broker',
+          '3',
+          '--describe',
+        ]
+      interval: 15s
+      timeout: 15s
+      retries: 10
+      start_period: 360s
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+      - toxiproxy
     restart: always
     environment:
       KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
@@ -116,6 +176,26 @@ services:
       args:
         KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
         SCALA_VERSION: ${SCALA_VERSION:-2.13}
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '--bootstrap-server',
+          'kafka-4:9091',
+          '--broker',
+          '4',
+          '--describe',
+        ]
+      interval: 15s
+      timeout: 15s
+      retries: 10
+      start_period: 360s
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+      - toxiproxy
     restart: always
     environment:
       KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
@@ -141,6 +221,26 @@ services:
       args:
         KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
         SCALA_VERSION: ${SCALA_VERSION:-2.13}
+    healthcheck:
+      test:
+        [
+          'CMD',
+          '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
+          '--bootstrap-server',
+          'kafka-5:9091',
+          '--broker',
+          '5',
+          '--describe',
+        ]
+      interval: 15s
+      timeout: 15s
+      retries: 10
+      start_period: 360s
+    depends_on:
+      - zookeeper-1
+      - zookeeper-2
+      - zookeeper-3
+      - toxiproxy
     restart: always
     environment:
       KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
@@ -160,13 +260,19 @@ services:
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
   toxiproxy:
     image: 'ghcr.io/shopify/toxiproxy:2.4.0'
+    healthcheck:
+      test: ['CMD', '/toxiproxy-cli', 'l']
+      interval: 15s
+      timeout: 15s
+      retries: 3
+      start_period: 30s
     ports:
-    # The tests themselves actually start the proxies on these ports
-    - '29091:29091'
-    - '29092:29092'
-    - '29093:29093'
-    - '29094:29094'
-    - '29095:29095'
+      # The tests themselves actually start the proxies on these ports
+      - '29091:29091'
+      - '29092:29092'
+      - '29093:29093'
+      - '29094:29094'
+      - '29095:29095'
 
-    # This is the toxiproxy API port
-    - '8474:8474'
+      # This is the toxiproxy API port
+      - '8474:8474'
diff --git a/functional_test.go b/functional_test.go
@@ -157,7 +157,10 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err
 		env.KafkaVersion = "3.5.1"
 	}
 
-	c := exec.Command("docker-compose", "up", "-d")
+	// docker-compose v2.17.0 or newer required for `--wait-timeout` support
+	c := exec.Command(
+		"docker-compose", "up", "-d", "--quiet-pull", "--timestamps", "--wait", "--wait-timeout", "600",
+	)
 	c.Stdout = os.Stdout
 	c.Stderr = os.Stderr
 	c.Env = append(os.Environ(), fmt.Sprintf("KAFKA_VERSION=%s", env.KafkaVersion))