Improve test battery poll functions

Dedicated functions for the usage patterns.
Positive poll logic.

flakytests/restart/39-auto-restart-no-suitable-host.t

@@ -50,8 +50,7 @@ ${BASE_GLOBALRC}
 "
 
 cylc run "${SUITE_NAME}"
-poll ! test -f "${SUITE_RUN_DIR}/.service/contact"
-sleep 1
+poll_suite_running
 
 create_test_globalrc '' "
 ${BASE_GLOBALRC}
@@ -68,8 +67,6 @@ log_scan "${TEST_NAME_BASE}-no-auto-restart" "${FILE}" 20 1 \
     'Suite cannot automatically restart because:' \
     'No alternative host to restart suite on.'
 
-cylc stop "${SUITE_NAME}" --kill
-poll test -f "${SUITE_RUN_DIR}/.service/contact"
-sleep 1
+cylc stop --kill --max-polls=10 --interval=2 "${SUITE_NAME}"
 purge_suite "${SUITE_NAME}"
 exit

flakytests/restart/40-auto-restart-force-stop.t

@@ -46,8 +46,7 @@ ${BASE_GLOBALRC}
 "
 
 cylc run "${SUITE_NAME}" --hold
-poll ! test -f "${SUITE_RUN_DIR}/.service/contact"
-sleep 1
+poll_suite_running
 
 create_test_globalrc '' "
 ${BASE_GLOBALRC}
@@ -63,8 +62,6 @@ log_scan "${TEST_NAME_BASE}-no-auto-restart" "${FILE}" 20 1 \
     'When another suite host becomes available the suite can' \
     'Suite shutting down - REQUEST(NOW)'
 
-cylc stop "${SUITE_NAME}" --kill 2>/dev/null # in-case test fails
-poll test -f "${SUITE_RUN_DIR}/.service/contact"
-sleep 1
+cylc stop --kill --max-polls=10 --interval=2 "${SUITE_NAME}" 2>'/dev/null'
 purge_suite "${SUITE_NAME}"
 exit

flakytests/restart/46-stop-clock-time.t

@@ -83,7 +83,7 @@ for i in {01..10}; do
     ST_FILE="${SUITE_RUN_DIR}/log/job/1/t_i${i}/01/job.status"
     if [[ -e "${ST_FILE}" ]]; then
         JOB_ID="$(awk -F= '$1 == "CYLC_BATCH_SYS_JOB_ID" {print $2}' "${ST_FILE}")"
-        poll ps "${JOB_ID}" 1>'/dev/null' 2>&1
+        poll_pid_done "${JOB_ID}"
     fi
 done
 

tests/authentication/05-full-control.t

@@ -34,8 +34,7 @@ cylc run "${SUITE_NAME}"
 unset CYLC_CONF_PATH
 
 # Wait for first task 'foo' to fail.
-poll '!' test -f "${SUITE_RUN_DIR}/log/job/1/foo/01/job.status"
-poll '!' grep -q 'CYLC_JOB_EXIT' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.status"
+poll_grep 'CYLC_JOB_EXIT' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.status"
 cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \
     --interval=1 --max-polls=10 || exit 1
 

tests/authentication/08-shared-fs.t

@@ -48,12 +48,9 @@ SUITE_PID="$!"
 SUITE_LOG="${SUITE_RUN_DIR}/log/suite/log"
 # Note: double poll existence of suite log on suite host and then localhost to
 # avoid any issues with unstable mounting of the shared file system.
-poll "! ssh -oBatchMode=yes -n '${CYLC_TEST_HOST}' \"test -e '${SUITE_LOG}'\""
-poll "! test -e '${SUITE_LOG}'"
-poll "! grep -qF '[t1.19700101T0000Z] -submitted => running' '${SUITE_LOG}'" \
-    2>'/dev/null'
-poll "! grep -qF '[t1.19700101T0000Z] -running => failed' '${SUITE_LOG}'" \
-    2>'/dev/null'
+poll ssh -oBatchMode=yes -n '${CYLC_TEST_HOST}' test -e "${SUITE_LOG}"
+poll_grep_suite_log -F '[t1.19700101T0000Z] -submitted => running'
+poll_grep_suite_log -F '[t1.19700101T0000Z] -running => failed'
 
 run_ok "${TEST_NAME_BASE}-broadcast" \
     cylc broadcast -n 't1' -s '[environment]CYLC_TEST_VAR_FOO=foo' "${SUITE_NAME}"

tests/authentication/10-remote-suite-passphrase-cache.t

@@ -41,7 +41,7 @@ cylc run --debug --no-detach --host="${CYLC_TEST_HOST}" --reference-test "${SUIT
 SUITE_PID=$!
 
 # Wait until the task failed
-poll '!' grep -q 't1.19700101T0000Z.*failed' 'err' 2>'/dev/null'
+poll_grep_suite_log 't1.19700101T0000Z.*failed'
 
 run_ok "${TEST_NAME_BASE}-broadcast" \
     cylc broadcast --host="${CYLC_TEST_HOST}" "${SUITE_NAME}" \

tests/authentication/11-suite2-stop-suite1.t

@@ -42,7 +42,7 @@ cat >"${SUITE2_RUND}/suite.rc" <<__SUITERC__
 __SUITERC__
 cylc register "${NAME2}" "${SUITE2_RUND}"
 cylc run --no-detach "${NAME1}" 1>'1.out' 2>&1 &
-poll '!' test -e "${SUITE1_RUND}/.service/contact"
+SUITE_RUN_DIR="${SUITE1_RUND}" poll_suite_running
 run_ok "${TEST_NAME_BASE}" cylc run --no-detach "${NAME2}"
 cylc shutdown "${NAME1}" --max-polls=20 --interval=1 1>'/dev/null' 2>&1 || true
 purge_suite "${NAME1}"

tests/cylc-insert/12-cycle-500-tasks.t

@@ -22,12 +22,11 @@ install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
 
 run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
 run_ok "${TEST_NAME_BASE}" cylc run --hold "${SUITE_NAME}"
-LOG="${SUITE_RUN_DIR}/log/suite/log"
-poll "! grep -q -F 'Held on start-up' '${LOG}' 2>'/dev/null'"
+poll_grep_suite_log -F 'Held on start-up'
 run_ok "${TEST_NAME_BASE}-insert" cylc insert "${SUITE_NAME}" '2008/*'
-poll "! grep -q -F 'Command succeeded: insert_tasks' '${LOG}' 2>'/dev/null'"
+poll_grep_suite_log -F 'Command succeeded: insert_tasks'
 cylc stop --max-polls=10 --interval=6 "${SUITE_NAME}" 2>'/dev/null'
-cut -d' ' -f 2- "${LOG}" >'trimmed-log'
+cut -d' ' -f 2- "${SUITE_RUN_DIR}/log/suite/log" >'trimmed-log'
 {
     for I in {001..500}; do
         echo "INFO - [v_i${I}.2008] -submit-num=00, inserted"

tests/cylc-submit/00-bg.t

@@ -86,27 +86,27 @@ if [[ -n "${SSH}" ]]; then
     SUITE_DIR="${SUITE_DIR#"${HOME}/"}"
     ST_FILE="${SUITE_DIR}/log/job/1/foo/01/job.status"
     # shellcheck disable=SC2086
-    poll ! $SSH "grep -q 'CYLC_BATCH_SYS_JOB_ID=' \"${ST_FILE}\"" 2>/dev/null
+    poll $SSH "grep -q 'CYLC_BATCH_SYS_JOB_ID=' \"${ST_FILE}\"" 2>'/dev/null'
     # shellcheck disable=SC2086
     JOB_ID=$($SSH "cat \"${ST_FILE}\"" \
         | awk -F= '$1 == "CYLC_BATCH_SYS_JOB_ID" {print $2}')
 else
     ST_FILE="${SUITE_DIR}/log/job/1/foo/01/job.status"
-    poll ! grep -q 'CYLC_BATCH_SYS_JOB_ID=' "${ST_FILE}" 2>/dev/null
+    poll_grep 'CYLC_BATCH_SYS_JOB_ID=' "${ST_FILE}"
     JOB_ID=$(awk -F= '$1 == "CYLC_BATCH_SYS_JOB_ID" {print $2}' "${ST_FILE}")
 fi
 contains_ok "${TEST_NAME_BASE}.stdout" <<<"[foo.1] Job ID: ${JOB_ID}"
 cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null'
 #-------------------------------------------------------------------------------
 if [[ -n "${SSH}" ]]; then
     # shellcheck disable=SC2086
-    poll ! $SSH "grep -q 'CYLC_JOB_INIT_TIME=' \"${ST_FILE}\"" 2>/dev/null
+    poll $SSH "grep -q 'CYLC_JOB_INIT_TIME=' \"${ST_FILE}\"" 2>'/dev/null'
     # shellcheck disable=SC2086
-    poll ! $SSH "grep -q 'CYLC_JOB_EXIT=' \"${ST_FILE}\"" 2>/dev/null
+    poll $SSH "grep -q 'CYLC_JOB_EXIT=' \"${ST_FILE}\"" 2>'/dev/null'
     purge_suite_remote "${CYLC_TEST_HOST}" "${SUITE_NAME}"
 else
-    poll ! grep -q 'CYLC_JOB_INIT_TIME=' "${ST_FILE}" 2>/dev/null
-    poll ! grep -q 'CYLC_JOB_EXIT=' "${ST_FILE}" 2>/dev/null
+    poll_grep 'CYLC_JOB_INIT_TIME=' "${ST_FILE}"
+    poll_grep 'CYLC_JOB_EXIT=' "${ST_FILE}"
 fi
 purge_suite "${SUITE_NAME}"
 exit

tests/cylc-submit/11-multi.t

@@ -53,7 +53,7 @@ for TASK_ID in 'food.2020' 'fool.2020' 'foot.2020' 'bar.2021'; do
     ST_FILE="${SUITE_RUN_DIR}/log/job/${POINT}/${NAME}/01/job.status"
     JOB_ID="$(awk -F= '$1 == "CYLC_BATCH_SYS_JOB_ID" {print $2}' "${ST_FILE}")"
     echo "[${TASK_ID}] Job ID: ${JOB_ID}"
-    poll ps "${JOB_ID}" 1>'/dev/null' 2>&1
+    poll_pid_done "${JOB_ID}"
 done >'expected.out'
 contains_ok "${TEST_NAME_BASE}.stdout" 'expected.out'
 cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null'

tests/graph-equivalence/00-oneline.t

@@ -33,8 +33,7 @@ suite_run_ok "${TEST_NAME}" \
 #-------------------------------------------------------------------------------
 TEST_NAME="${TEST_NAME_BASE}-check-a"
 cylc run "${SUITE_NAME}" --hold 1>'out' 2>&1
-poll "! test -e '${SUITE_RUN_DIR}/log/suite/log'"
-poll "! grep -q 'Holding all waiting or queued tasks now' '${SUITE_RUN_DIR}/log/suite/log'"
+poll_grep_suite_log 'Holding all waiting or queued tasks now'
 cylc show "${SUITE_NAME}" 'a.1' | sed -n "/prerequisites/,/outputs/p" > 'a-prereqs'
 cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/a-ref" 'a-prereqs'
 #-------------------------------------------------------------------------------
@@ -46,5 +45,6 @@ TEST_NAME="${TEST_NAME_BASE}-check-c"
 cylc show "${SUITE_NAME}" 'c.1' | sed -n "/prerequisites/,/outputs/p" > 'c-prereqs'
 cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/c-ref" 'c-prereqs'
 #-------------------------------------------------------------------------------
-cylc shutdown "${SUITE_NAME}" --now -f
+cylc shutdown --max-polls=10 --interval=2 --now -f "${SUITE_NAME}"
 purge_suite "${SUITE_NAME}"
+exit

tests/graph-equivalence/01-twolines.t

@@ -33,8 +33,7 @@ suite_run_ok "${TEST_NAME}" \
 #-------------------------------------------------------------------------------
 TEST_NAME="${TEST_NAME_BASE}-check-a"
 cylc run "${SUITE_NAME}" --hold 1>'out' 2>&1
-poll "! test -e '${SUITE_RUN_DIR}/log/suite/log'"
-poll "! grep -q 'Holding all waiting or queued tasks now' '${SUITE_RUN_DIR}/log/suite/log'"
+poll_grep_suite_log 'Holding all waiting or queued tasks now'
 cylc show "${SUITE_NAME}" 'a.1' | sed -n "/prerequisites/,/outputs/p" > 'a-prereqs'
 cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/a-ref" 'a-prereqs'
 #-------------------------------------------------------------------------------

tests/graph-equivalence/02-splitline.t

@@ -34,8 +34,7 @@ suite_run_ok "${TEST_NAME}" \
 #-------------------------------------------------------------------------------
 TEST_NAME="${TEST_NAME_BASE}-check-a"
 cylc run "${SUITE_NAME}" --hold 1>'out' 2>&1
-poll "! test -e '${SUITE_RUN_DIR}/log/suite/log'"
-poll "! grep -q 'Holding all waiting or queued tasks now' '${SUITE_RUN_DIR}/log/suite/log'"
+poll_grep_suite_log 'Holding all waiting or queued tasks now'
 cylc show "${SUITE_NAME}" 'a.1' | sed -n "/prerequisites/,/outputs/p" > 'a-prereqs'
 cmp_ok "${TEST_SOURCE_DIR}/splitline_refs/a-ref" 'a-prereqs'
 #-------------------------------------------------------------------------------

tests/graph-equivalence/03-multiline_and1.t

@@ -34,8 +34,7 @@ suite_run_ok "${TEST_NAME}" \
 #-------------------------------------------------------------------------------
 TEST_NAME="${TEST_NAME_BASE}-check-c"
 cylc run "${SUITE_NAME}" --hold 1>'out' 2>&1
-poll "! test -e '${SUITE_RUN_DIR}/log/suite/log'"
-poll "! grep -q 'Holding all waiting or queued tasks now' '${SUITE_RUN_DIR}/log/suite/log'"
+poll_grep_suite_log 'Holding all waiting or queued tasks now'
 cylc show "${SUITE_NAME}" 'c.1' | sed -n "/prerequisites/,/outputs/p" > 'c-prereqs'
 contains_ok "${TEST_SOURCE_DIR}/multiline_and_refs/c-ref" 'c-prereqs'
 cylc shutdown "${SUITE_NAME}" --now -f

tests/graph-equivalence/04-multiline_and2.t

@@ -35,10 +35,9 @@ suite_run_ok "${TEST_NAME}" \
 #-------------------------------------------------------------------------------
 TEST_NAME=${TEST_NAME_BASE}-check-c
 cylc run "${SUITE_NAME}" --hold 1>'out' 2>&1
-poll "! test -e '${SUITE_RUN_DIR}/log/suite/log'"
-poll "! grep -q 'Holding all waiting or queued tasks now' '${SUITE_RUN_DIR}/log/suite/log'"
+poll_grep_suite_log 'Holding all waiting or queued tasks now'
 cylc show "${SUITE_NAME}" 'c.1' | sed -n "/prerequisites/,/outputs/p" > 'c-prereqs'
 contains_ok "${TEST_SOURCE_DIR}/multiline_and_refs/c-ref-2" 'c-prereqs'
-cylc shutdown "${SUITE_NAME}" --now -f
+cylc shutdown --max-polls=20 --interval=2 --now -f "${SUITE_NAME}"
 #-------------------------------------------------------------------------------
 purge_suite "${SUITE_NAME}"

tests/hold-release/21-client.t

@@ -24,7 +24,7 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
 cylc run --reference-test --hold --debug --no-detach "${SUITE_NAME}" \
     1>"${TEST_NAME_BASE}.out" 2>&1 &
 CYLC_RUN_PID=$!
-poll ! test -f "${SUITE_RUN_DIR}/.service/contact"
+poll_suite_running
 run_ok "${TEST_NAME_BASE}-client" \
     cylc client "${SUITE_NAME}" 'release_suite' -n
 run_ok "${TEST_NAME_BASE}-run" wait "${CYLC_RUN_PID}"

tests/job-file-trap/00-sigusr1.t

@@ -32,8 +32,7 @@ run_tests() {
     # Make sure t1.1.1's status file is in place
     T1_STATUS_FILE=$SUITE_RUN_DIR/log/job/1/t1/01/job.status
 
-    poll '!' test -e "${T1_STATUS_FILE}"
-    poll '!' grep 'CYLC_JOB_PID=' "${T1_STATUS_FILE}"
+    poll_grep 'CYLC_JOB_PID=' "${T1_STATUS_FILE}"
 
     # Kill the job and see what happens
     T1_PID="$(awk -F= '$1=="CYLC_JOB_PID" {print $2}' "${T1_STATUS_FILE}")"

tests/lib/bash/test_header

@@ -90,7 +90,17 @@
 #         Tidy up test directories for SUITE_NAME.
 #     poll COMMAND
 #         Run COMMAND in 1 second intervals for a minute until COMMAND returns
-#         a non-zero value, or exit 1 (abort test) with a timeout message.
+#         a zero value, or exit 1 (abort test) with a timeout message.
+#     poll_grep ...
+#         Shorthand for 'poll grep -s ...'.
+#     poll_grep_suite_log ...
+#         Shorthand for 'poll_grep ... "${SUITE_RUN_DIR}/log/suite/log"'
+#     poll_pid_done PID
+#         Poll until process with PID is done.
+#     poll_suite_running
+#         Shorthand for 'poll test -e "${SUITE_RUN_DIR}/.service/contact"'
+#     poll_suite_stopped
+#         Shorthand for 'poll "!" test -e "${SUITE_RUN_DIR}/.service/contact"'
 #     skip N SKIP_REASON
 #         echo "ok $((++T)) # skip SKIP_REASON" N times.
 #     skip_all SKIP_REASON
@@ -495,19 +505,40 @@ purge_suite() {
 
 poll() {
     local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute
-    local TIMED_OUT=true
-    while (($(date +%s) < TIMEOUT)); do
-        if eval "$@"; then
-            sleep 1
-        else
-            TIMED_OUT=false
-            break
+    while ! "$@"; do
+        sleep 1
+        if (($(date +%s) > TIMEOUT)); then
+            echo "ERROR: poll timed out: $*" >&2
+            exit 1
         fi
     done
-    if $TIMED_OUT; then
-        >&2 echo "ERROR: poll timed out: $*"
-        exit 1
-    fi
+}
+
+poll_grep() {
+    poll grep -s "$@"
+}
+
+poll_grep_suite_log() {
+    poll grep -s "$@" "${SUITE_RUN_DIR}/log/suite/log"
+}
+
+poll_pid_done() {
+    local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute
+    while ps --no-headers "$@" 1>'/dev/null' 2>&1; do
+        sleep 1
+        if (($(date +%s) > TIMEOUT)); then
+            echo "ERROR: poll timed out: ! ps --no-headers $*" >&2
+            exit 1
+        fi
+    done
+}
+
+poll_suite_running() {
+    poll test -e "${SUITE_RUN_DIR}/.service/contact"
+}
+
+poll_suite_stopped() {
+    poll test '!' -e "${SUITE_RUN_DIR}/.service/contact"
 }
 
 skip() {

tests/pre-initial/12-warm-restart.t

@@ -33,7 +33,7 @@ cylc stop --max-polls=10 --interval=2 "${SUITE_NAME}"
 TEST_NAME=${TEST_NAME_BASE}-run-hold-restart
 suite_run_ok "${TEST_NAME}" cylc restart "${SUITE_NAME}"
 # Ensure suite has started
-poll ! test -f "${SUITE_RUN_DIR}/.service/contact"
+poll_suite_running
 #-------------------------------------------------------------------------------
 # Check pre-reqs
 TEST_NAME=${TEST_NAME_BASE}-check-prereq

tests/reload/22-remove-task-cycling.t

@@ -25,7 +25,7 @@ set_test_number 3
 # A suite designed to orphan a single copy of a task 'bar' on self-reload,
 # or stall and abort if the orphaned task triggers the #3306 bug.
 
-init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__'
+init_suite "${TEST_NAME_BASE}" <<__SUITE_RC__
 [cylc]
    [[events]]
       inactivity = PT25S
@@ -45,38 +45,24 @@ init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__'
    [[foo]]
       script = """
 # Use poll function from test_header.
-poll() {
-    local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute
-    local TIMED_OUT=true
-    while (($(date +%s) < TIMEOUT)); do
-        if eval "$@"; then
-            sleep 1
-        else
-            TIMED_OUT=false
-            break
-        fi
-    done
-    if $TIMED_OUT; then
-        >&2 echo "ERROR: poll timed out: $*"
-        exit 1
-    fi
-}
+$(declare -f poll)
+$(declare -f poll_grep)
 
 # Remove bar and tell the server to reload.
 if (( CYLC_TASK_CYCLE_POINT == CYLC_SUITE_INITIAL_CYCLE_POINT )); then
-   sed -i 's/^.*remove*$//g' $CYLC_SUITE_DEF_PATH/suite.rc
-   cylc reload $CYLC_SUITE_NAME
-   poll "! grep 'Reload complete' $CYLC_SUITE_RUN_DIR/log/suite/log"
+   sed -i 's/^.*remove*$//g' "\${CYLC_SUITE_DEF_PATH}/suite.rc"
+   cylc reload "\${CYLC_SUITE_NAME}"
+   poll_grep -F 'Reload complete' "\${CYLC_SUITE_RUN_DIR}/log/suite/log"
    # kill the long-running orphaned bar task.
-   kill $(cat $CYLC_SUITE_RUN_DIR/work/1/bar/pid)
+   kill "\$(cat "\${CYLC_SUITE_RUN_DIR}/work/1/bar/pid")"
 fi
 """
    [[bar]]
       script = """
 # Long sleep to ensure that bar does not finish before the reload.
 # Store long sleep PID to enable kill after the reload.
 sleep 1000 &
-echo $! > pid
+echo \$! > pid
 wait"""
 __SUITE_RC__
 

tests/restart/23-hold-retry.t

@@ -17,7 +17,7 @@
 #-------------------------------------------------------------------------------
 # Test restart with held (retrying) task
 . "$(dirname "$0")/test_header"
-set_test_number 6
+set_test_number 5
 install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
 
 run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
@@ -28,12 +28,14 @@ sqlite3 "${SUITE_RUN_DIR}/log/db" \
 cmp_ok 'task-pool.out' <<__OUT__
 1|t1|1|retrying|1
 __OUT__
-suite_run_ok "${TEST_NAME_BASE}-restart" cylc restart "${SUITE_NAME}" --debug
-ls "${SUITE_RUN_DIR}/log/suite/" 1>'/dev/null' 2>&1
-poll "! grep -qF 'INFO - + t1.1 retrying (held)' '${SUITE_RUN_DIR}/log/suite/log'"
+cylc restart "${SUITE_NAME}" --debug --no-detach 1>'out' 2>&1 &
+SUITE_PID=$!
+poll_grep_suite_log -F 'INFO - + t1.1 retrying (held)'
 run_ok "${TEST_NAME_BASE}-release" cylc release "${SUITE_NAME}"
-poll "! grep -qF 'INFO - DONE' '${SUITE_RUN_DIR}/log/suite/log'"
-poll "test -e '${SUITE_RUN_DIR}/.service/contact'"
+poll_grep_suite_log -F 'INFO - DONE'
+if ! wait "${SUITE_PID}"; then
+    cat 'out' >&2
+fi
 sqlite3 "${SUITE_RUN_DIR}/log/db" \
     'SELECT * FROM task_pool ORDER BY cycle, name' >'task-pool.out'
 cmp_ok 'task-pool.out' <<__OUT__