ci: Exercise the right discipline to restart docker containers (#632)

One does not simply restart a docker container. But when you do,
SpacetimeDB also restarts.

.github/workflows/ci.yml

@@ -16,8 +16,10 @@ jobs:
       - name: Start containers
         run: docker compose up -d
       - name: Run smoketests
-        # TODO: Fix flaky tests https://github.com/clockworklabs/SpacetimeDB/issues/630
         run: test/run-smoke-tests.sh --parallel -x bitcraftmini-pretest zz_docker-restart-module zz_docker-restart-repeating-reducer zz_docker-restart-sql
+      # These cannot run in parallel, even though the script tries to handle it
+      - name: Run restarting smoketests
+        run: test/run-smoke-tests.sh zz_docker-restart-module zz_docker-restart-repeating-reducer zz_docker-restart-sql
       - name: Stop containers
         if: always()
         run: docker compose down

docker-compose.yml

@@ -28,6 +28,8 @@ services:
       # Tracy
       - "8086:8086"
     entrypoint: cargo watch -i flamegraphs -i log.conf --why -C crates/standalone -x 'run start'
+    healthcheck:
+      test: curl -f http://localhost/database/ping || exit 1
     privileged: true
     environment:
       SPACETIMEDB_FLAMEGRAPH_PATH: ../../../../flamegraphs/flamegraph.folded

test/lib.include

@@ -74,9 +74,45 @@ fsed() {
 }
 
 restart_docker() {
-	docker-compose stop node
-	docker-compose start node
-	sleep 10
+	# Behold!
+	#
+	# You thought stop/start restarts? How wrong. Restart restarts.
+	docker compose restart
+	# The suspense!
+	#
+	# Wait until compose believes the health probe succeeds.
+	#
+	# The container may decide to recompile, or grab a coffee at crates.io, or
+	# whatever. In any case, restart doesn't mean the server is up yet.
+	docker compose up --no-recreate --detach --wait-timeout 60
+	# Belts and suspenders!
+	#
+	# The health probe runs inside the container, but that doesn't mean we can
+	# reach it from outside. Ping until we get through.
+	ping
+}
+
+ping() {
+	local retries=5
+	local success=0
+	while true
+	do
+		curl -sf http://127.0.0.1:3000/database/ping && { success=1; break; } || echo "Server down"
+		retries=$((retries - 1))
+		if [ $retries -gt 0 ]
+		then
+			sleep 5
+		else
+			break
+		fi
+	done
+	if [ $success -lt 1 ]
+	then
+		echo "Server at 127.0.0.1:3000 not responding"
+		exit 127
+	else
+		echo "Server up after $((5 - retries)) retries"
+	fi
 }
 
 # vim: noexpandtab tabstop=4 shiftwidth=4