Use incremental backoff instead of hardcoding fixed sleep times

With fixed/long sleep times, tests run slower when there is no need to
wait for that long. When the machine is slow (or under heavy load), we
often to need to wait for longer before things are running and the fixed
sleep times are not sufficient.

dotnet-monitor-works/test.sh

@@ -10,16 +10,13 @@ dotnet tool update -g dotnet-monitor --version "${VERSION_SPLIT[0]}.*-*"
 export PATH="$HOME/.dotnet/tools:$PATH"
 
 dotnet-monitor collect --no-auth &
-sleep 5
-indexhttps=$(wget --no-check-certificate -O https.html https://127.0.0.1:52323/info)
+../run-until-success-with-backoff wget --no-check-certificate -O https.html https://127.0.0.1:52323/info
 
 https=$(cat https.html)
 
 if [[ $https == *"version"* ]]; then
-   sleep 5
    echo "collect - OK"
 else
-   sleep 5
    pkill dotnet-monitor
    rm "https.html"
    echo "collect - FAIL"
@@ -45,4 +42,4 @@ else
    echo "generatekey - FAIL"
    pkill dotnet-monitor
    exit 1
-fi
+fi

libuv-kestrel-sample-app-2x/test.sh

@@ -9,9 +9,7 @@ dotnet build
 dotnet bin/Debug/netcoreapp*/libuv-kestrel-sample-app-2x.dll &
 root_pid=$!
 
-sleep 5
-
-curl "http://localhost:5000"
+../run-until-success-with-backoff curl "http://localhost:5000"
 
 kill -s SIGTERM "${root_pid}"
 sleep 1

publish-aspnet-selfcontained/test.sh

@@ -28,11 +28,10 @@ ASPNETCORE_URLS="$url" "./$output_dir/web" &
 run_pid=$!
 trap "kill $run_pid && wait $run_pid" EXIT
 
-sleep 5
 
-if ! curl "$url"; then
+if ! ../run-until-success-with-backoff curl "$url" ; then
   echo 'FAIL: ASP.NET app failed to respond'
   exit 2
 fi
 
-echo "PASS"
+echo "PASS"

run-until-success-with-backoff

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -euo pipefail
+
+echo "Running with backoff:" "$@"
+
+max_retries=10
+
+iterations=0
+total_slept=0 # seconds
+
+sleep $((iterations + 1))
+((total_slept += iterations + 1))
+
+until "$@"; do
+    ((iterations += 1))
+    if (( total_slept > 10 )); then
+        echo "$@" "still failing after more than ${total_slept} seconds"
+    fi
+    if (( iterations == max_retries )); then
+        echo "$@" "still failing after $max_retries retries"
+        exit 1
+    fi
+    sleep $iterations
+    ((total_slept += iterations))
+done