diff --git a/cylc/flow/tests/test_wallclock.py b/cylc/flow/tests/test_wallclock.py
new file mode 100644
index 00000000000..5e2b4c35d72
--- /dev/null
+++ b/cylc/flow/tests/test_wallclock.py
@@ -0,0 +1,50 @@
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+import pytest
+
+from metomi.isodatetime.data import CALENDAR
+from cylc.flow.wallclock import get_unix_time_from_time_string
+
+
+@pytest.mark.parametrize(
+ 'time_str,time_sec',
+ [
+ ('2016-09-08T09:09:00+01', 1473322140),
+ ('2016-09-08T08:09:00Z', 1473322140),
+ ('2016-09-07T20:09:00-12', 1473322140),
+ ]
+)
+def test_get_unix_time_from_time_string(time_str, time_sec):
+ assert get_unix_time_from_time_string(time_str) == time_sec
+
+
+@pytest.mark.parametrize(
+ 'time_str,time_sec',
+ [
+ ('2016-09-08T09:09:00+01', 1473322140),
+ ('2016-09-08T08:09:00Z', 1473322140),
+ ('2016-09-07T20:09:00-12', 1473322140),
+ ('2016-08-31T18:09:00+01', 1472663340),
+ ]
+)
+def test_get_unix_time_from_time_string_360(time_str, time_sec):
+ mode = CALENDAR.mode
+ CALENDAR.set_mode(CALENDAR.MODE_360)
+ try:
+ assert get_unix_time_from_time_string(time_str) == time_sec
+ finally:
+ CALENDAR.set_mode(mode)
diff --git a/flakytests/cylc-kill/02-submitted.t b/flakytests/cylc-kill/02-submitted.t
index c063c790b5d..fdd0667ce2d 100755
--- a/flakytests/cylc-kill/02-submitted.t
+++ b/flakytests/cylc-kill/02-submitted.t
@@ -18,10 +18,5 @@
# Test kill running jobs only
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/flakytests/cylc-poll/03-poll-all.t b/flakytests/cylc-poll/03-poll-all.t
index 12eaf5472a6..8ea3cb41b52 100755
--- a/flakytests/cylc-poll/03-poll-all.t
+++ b/flakytests/cylc-poll/03-poll-all.t
@@ -17,14 +17,6 @@
#-------------------------------------------------------------------------------
# Test that when polling all a failed task sets the task state correctly
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/flakytests/cylc-poll/03-poll-all/suite.rc b/flakytests/cylc-poll/03-poll-all/suite.rc
index 2a9c452fc2a..02ad82e84fc 100644
--- a/flakytests/cylc-poll/03-poll-all/suite.rc
+++ b/flakytests/cylc-poll/03-poll-all/suite.rc
@@ -46,7 +46,7 @@ pat2="[run_kill.${CYLC_TASK_CYCLE_POINT}] -suiciding"
log="${CYLC_SUITE_LOG_DIR}/log"
while (($(grep -c -F -e "${pat1}" -e "${pat2}" "${log}") != 2))
do
- sleep 2
+ sleep 1
done
st_file="${CYLC_SUITE_RUN_DIR}/log/job/${CYLC_TASK_CYCLE_POINT}/submit_hold/NN/job.status"
diff --git a/flakytests/cylc-reset/02-output-1.t b/flakytests/cylc-reset/02-output-1.t
index 938d3f0828e..19f66f70823 100755
--- a/flakytests/cylc-reset/02-output-1.t
+++ b/flakytests/cylc-reset/02-output-1.t
@@ -17,12 +17,6 @@
#-------------------------------------------------------------------------------
# Test "cylc reset --output='TRIGGER|MESSAGE' 'SUITE' 'TASK.ID'".
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/flakytests/execution-time-limit/04-poll.t b/flakytests/execution-time-limit/04-poll.t
index e86d5d9d1d1..2635ed6101b 100755
--- a/flakytests/execution-time-limit/04-poll.t
+++ b/flakytests/execution-time-limit/04-poll.t
@@ -17,14 +17,6 @@
#-------------------------------------------------------------------------------
# Test execution time limit polling.
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" \
- cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/flakytests/hold-release/14-hold-kill.t b/flakytests/hold-release/14-hold-kill.t
index e7aee1297e1..3003c9ecdea 100755
--- a/flakytests/hold-release/14-hold-kill.t
+++ b/flakytests/hold-release/14-hold-kill.t
@@ -17,12 +17,6 @@
#-------------------------------------------------------------------------------
# Test: hold a task to prevent it from retrying when killing its job
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}-run" cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/flakytests/hold-release/20-reset-waiting-output.t b/flakytests/hold-release/20-reset-waiting-output.t
index 419fd402224..f71d2a93bd8 100755
--- a/flakytests/hold-release/20-reset-waiting-output.t
+++ b/flakytests/hold-release/20-reset-waiting-output.t
@@ -19,10 +19,5 @@
# https://github.com/cylc/cylc-flow/issues/2115
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/flakytests/integer-cycling/00-satellite.t b/flakytests/integer-cycling/00-satellite.t
index 0ecf3b2bba9..56308d649bb 100644
--- a/flakytests/integer-cycling/00-satellite.t
+++ b/flakytests/integer-cycling/00-satellite.t
@@ -18,14 +18,6 @@
# Run the integer cycling satellite example suite.
# Non-specific, tests a bunch of stuff.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/flakytests/modes/03-dummy-env.t b/flakytests/modes/03-dummy-env.t
index a2bbe0abd32..91c82043d72 100644
--- a/flakytests/modes/03-dummy-env.t
+++ b/flakytests/modes/03-dummy-env.t
@@ -15,22 +15,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#-------------------------------------------------------------------------------
-
# Test that user environment is disabled along with env-script in dummy mode.
# And that remote host is disabled in dummy local mode.
-
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" \
- cylc run -m 'dummy-local' --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/broadcast/02-inherit.t b/tests/broadcast/02-inherit.t
index 37e68ff1f37..a65a9168c82 100755
--- a/tests/broadcast/02-inherit.t
+++ b/tests/broadcast/02-inherit.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test broadcasts, with overriding inheritance.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/broadcast/03-expire.t b/tests/broadcast/03-expire.t
index e22d0cdee0e..355aab01887 100755
--- a/tests/broadcast/03-expire.t
+++ b/tests/broadcast/03-expire.t
@@ -18,11 +18,5 @@
# Test broadcast --expire=CYCLE_POINT
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/broadcast/04-empty.t b/tests/broadcast/04-empty.t
index da30c333f29..6b95a8329ac 100755
--- a/tests/broadcast/04-empty.t
+++ b/tests/broadcast/04-empty.t
@@ -18,11 +18,5 @@
# Test broadcast of an empty string
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/broadcast/07-float-setting.t b/tests/broadcast/07-float-setting.t
deleted file mode 100755
index 53a26a7f6a2..00000000000
--- a/tests/broadcast/07-float-setting.t
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#-------------------------------------------------------------------------------
-# Test broadcasts a float setting
-. "$(dirname "$0")/test_header"
-
-set_test_number 3
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-LOG="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/log/suite/log"
-
-grep_ok 'WARNING - \[timeout\.20100808T0000Z\] -execution timeout after PT1S' \
- "${LOG}"
-
-purge_suite "${SUITE_NAME}"
-exit
diff --git a/tests/cyclepoint/01-iso_timeparser.t b/tests/broadcast/07-timeout.t
old mode 100644
new mode 100755
similarity index 76%
rename from tests/cyclepoint/01-iso_timeparser.t
rename to tests/broadcast/07-timeout.t
index fea5b2134d3..3ab702c569c
--- a/tests/cyclepoint/01-iso_timeparser.t
+++ b/tests/broadcast/07-timeout.t
@@ -15,9 +15,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#-------------------------------------------------------------------------------
-# Run the time_parser.py tests
+# Test broadcasts a timeout setting
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
-set_test_number 1
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}" python3 -m 'cylc.flow.time_parser'
+set_test_number 2
+reftest
+exit
diff --git a/tests/broadcast/07-float-setting/reference.log b/tests/broadcast/07-timeout/reference.log
similarity index 100%
rename from tests/broadcast/07-float-setting/reference.log
rename to tests/broadcast/07-timeout/reference.log
diff --git a/tests/broadcast/07-float-setting/suite.rc b/tests/broadcast/07-timeout/suite.rc
similarity index 71%
rename from tests/broadcast/07-float-setting/suite.rc
rename to tests/broadcast/07-timeout/suite.rc
index 19a500d63b2..ce446730984 100644
--- a/tests/broadcast/07-float-setting/suite.rc
+++ b/tests/broadcast/07-timeout/suite.rc
@@ -6,7 +6,7 @@
[scheduling]
initial cycle point = 20100808T0000Z
- final cycle point = 20100809T0000Z
+ final cycle point = 20100808T0000Z
[[graph]]
R1 = send_broadcast => timeout
@@ -14,9 +14,14 @@
[[send_broadcast]]
script = """
cylc broadcast -n timeout --point=20100808T0000Z --set='[events]execution timeout=PT1S' $CYLC_SUITE_NAME
- sleep 10
"""
[[timeout]]
- script = sleep 10
+ script = """
+while ! grep -qF "[${CYLC_TASK_ID}] -execution timeout after PT1S" \
+ "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
+"""
[[[events]]]
execution timeout = PT1M
diff --git a/tests/broadcast/08-space.t b/tests/broadcast/08-space.t
index 0dd64f863f5..63aa4b748a2 100755
--- a/tests/broadcast/08-space.t
+++ b/tests/broadcast/08-space.t
@@ -18,11 +18,5 @@
# Test broadcast -s '[foo] bar=baz' syntax. cylc/cylc-flow#1680
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/broadcast/09-remote.t b/tests/broadcast/09-remote.t
index a60a2a10ace..7e9dcb6cd6e 100755
--- a/tests/broadcast/09-remote.t
+++ b/tests/broadcast/09-remote.t
@@ -20,12 +20,6 @@ export CYLC_TEST_IS_GENERIC=false
. "$(dirname "$0")/test_header"
set_test_remote_host
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
+reftest
purge_suite_remote "${CYLC_TEST_HOST}" "${SUITE_NAME}"
-purge_suite "${SUITE_NAME}"
exit
diff --git a/tests/broadcast/10-file-1.t b/tests/broadcast/10-file-1.t
index e60eba86a95..f5bec52ff05 100755
--- a/tests/broadcast/10-file-1.t
+++ b/tests/broadcast/10-file-1.t
@@ -18,11 +18,5 @@
# Test broadcast settings in a file
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/broadcast/10-file-1/suite.rc b/tests/broadcast/10-file-1/suite.rc
index 77b8de50b2e..a65098a66de 100644
--- a/tests/broadcast/10-file-1/suite.rc
+++ b/tests/broadcast/10-file-1/suite.rc
@@ -7,7 +7,6 @@
[[t1]]
script = """
cylc broadcast -n 't2' -F "${CYLC_SUITE_DEF_PATH}/broadcast.rc" "${CYLC_SUITE_NAME}"
-sleep 1
"""
[[t2]]
script = false
diff --git a/tests/broadcast/11-file-2.t b/tests/broadcast/11-file-2.t
index c5ddbc2d1ba..117c3ad345a 100755
--- a/tests/broadcast/11-file-2.t
+++ b/tests/broadcast/11-file-2.t
@@ -18,11 +18,5 @@
# Test broadcast settings in 2 files
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/broadcast/11-file-2/suite.rc b/tests/broadcast/11-file-2/suite.rc
index f4c47f4ab13..b949e1cb8a8 100644
--- a/tests/broadcast/11-file-2/suite.rc
+++ b/tests/broadcast/11-file-2/suite.rc
@@ -10,7 +10,6 @@ cylc broadcast -n 't2' \
-F "${CYLC_SUITE_DEF_PATH}/broadcast-1.rc" \
-F "${CYLC_SUITE_DEF_PATH}/broadcast-2.rc" \
"${CYLC_SUITE_NAME}"
-sleep 1
"""
[[t2]]
script = false
diff --git a/tests/broadcast/12-file-stdin.t b/tests/broadcast/12-file-stdin.t
index c758daaf558..8b8c7782918 100755
--- a/tests/broadcast/12-file-stdin.t
+++ b/tests/broadcast/12-file-stdin.t
@@ -18,11 +18,5 @@
# Test broadcast settings from STDIN
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/broadcast/12-file-stdin/suite.rc b/tests/broadcast/12-file-stdin/suite.rc
index efecb9da92b..9515b41295c 100644
--- a/tests/broadcast/12-file-stdin/suite.rc
+++ b/tests/broadcast/12-file-stdin/suite.rc
@@ -8,7 +8,6 @@
script = """
cylc broadcast -n 't2' -F - "${CYLC_SUITE_NAME}" \
<"${CYLC_SUITE_DEF_PATH}/broadcast.rc"
-sleep 1
"""
[[t2]]
script = false
diff --git a/tests/broadcast/13-file-cancel.t b/tests/broadcast/13-file-cancel.t
index 956d2974ee3..a87ff826903 100755
--- a/tests/broadcast/13-file-cancel.t
+++ b/tests/broadcast/13-file-cancel.t
@@ -18,11 +18,5 @@
# Test broadcast cancel with settings in a file
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/broadcast/13-file-cancel/suite.rc b/tests/broadcast/13-file-cancel/suite.rc
index a4afec13161..97344533513 100644
--- a/tests/broadcast/13-file-cancel/suite.rc
+++ b/tests/broadcast/13-file-cancel/suite.rc
@@ -7,9 +7,7 @@
[[t1]]
script = """
cylc broadcast -n 't2' -F "${CYLC_SUITE_DEF_PATH}/broadcast-1.rc" "${CYLC_SUITE_NAME}"
-sleep 1
cylc broadcast -n 't2' -G "${CYLC_SUITE_DEF_PATH}/broadcast-2.rc" "${CYLC_SUITE_NAME}"
-sleep 1
"""
[[t2]]
script = false
diff --git a/tests/cyclers/21-360_calendar.t b/tests/cyclers/21-360_calendar.t
index 3312ed06440..451458a61d1 100755
--- a/tests/cyclers/21-360_calendar.t
+++ b/tests/cyclers/21-360_calendar.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test intercycle dependencies.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" 360
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cyclers/360/graph.plain.ref b/tests/cyclers/21-360_calendar/graph.plain.ref
similarity index 100%
rename from tests/cyclers/360/graph.plain.ref
rename to tests/cyclers/21-360_calendar/graph.plain.ref
diff --git a/tests/cyclers/360/reference.log b/tests/cyclers/21-360_calendar/reference.log
similarity index 100%
rename from tests/cyclers/360/reference.log
rename to tests/cyclers/21-360_calendar/reference.log
diff --git a/tests/cyclers/360/suite.rc b/tests/cyclers/21-360_calendar/suite.rc
similarity index 100%
rename from tests/cyclers/360/suite.rc
rename to tests/cyclers/21-360_calendar/suite.rc
diff --git a/tests/cyclers/26-0000_rollunder.t b/tests/cyclers/26-0000_rollunder.t
index 2cf9be625f6..67a3095faf9 100755
--- a/tests/cyclers/26-0000_rollunder.t
+++ b/tests/cyclers/26-0000_rollunder.t
@@ -17,7 +17,6 @@
#-------------------------------------------------------------------------------
# Test intercycle dependencies.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" 0000_rollunder
diff --git a/tests/cyclers/36-icp_fcp_notation.t b/tests/cyclers/36-icp_fcp_notation.t
index 23f558db069..56b0edfabb4 100755
--- a/tests/cyclers/36-icp_fcp_notation.t
+++ b/tests/cyclers/36-icp_fcp_notation.t
@@ -18,11 +18,5 @@
# Test initial and final cycle point special notation (^, $)
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/cyclers/38-unittests.t b/tests/cyclers/38-unittests.t
deleted file mode 100755
index f00dcf41d63..00000000000
--- a/tests/cyclers/38-unittests.t
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#-------------------------------------------------------------------------------
-# Test intercycle dependencies.
-. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
-set_test_number 3
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}-cycling-iso8601" python3 -m 'cylc.flow.cycling.iso8601'
-run_ok "${TEST_NAME_BASE}-cycling-integer" python3 -m 'cylc.flow.cycling.integer'
-run_ok "${TEST_NAME_BASE}-cycling" python3 -c 'import cylc.flow.cycling'
diff --git a/tests/cyclers/49-365_calendar.t b/tests/cyclers/49-365_calendar.t
index 9fcafdeaf89..451458a61d1 100644
--- a/tests/cyclers/49-365_calendar.t
+++ b/tests/cyclers/49-365_calendar.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test intercycle dependencies.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" 365
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cyclers/365/graph.plain.ref b/tests/cyclers/49-365_calendar/graph.plain.ref
similarity index 100%
rename from tests/cyclers/365/graph.plain.ref
rename to tests/cyclers/49-365_calendar/graph.plain.ref
diff --git a/tests/cyclers/365/reference.log b/tests/cyclers/49-365_calendar/reference.log
similarity index 100%
rename from tests/cyclers/365/reference.log
rename to tests/cyclers/49-365_calendar/reference.log
diff --git a/tests/cyclers/365/suite.rc b/tests/cyclers/49-365_calendar/suite.rc
similarity index 100%
rename from tests/cyclers/365/suite.rc
rename to tests/cyclers/49-365_calendar/suite.rc
diff --git a/tests/cyclers/50-366_calendar.t b/tests/cyclers/50-366_calendar.t
index 4bdda4599d5..451458a61d1 100644
--- a/tests/cyclers/50-366_calendar.t
+++ b/tests/cyclers/50-366_calendar.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test intercycle dependencies.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" 366
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cyclers/366/graph.plain.ref b/tests/cyclers/50-366_calendar/graph.plain.ref
similarity index 100%
rename from tests/cyclers/366/graph.plain.ref
rename to tests/cyclers/50-366_calendar/graph.plain.ref
diff --git a/tests/cyclers/366/reference.log b/tests/cyclers/50-366_calendar/reference.log
similarity index 100%
rename from tests/cyclers/366/reference.log
rename to tests/cyclers/50-366_calendar/reference.log
diff --git a/tests/cyclers/366/suite.rc b/tests/cyclers/50-366_calendar/suite.rc
similarity index 100%
rename from tests/cyclers/366/suite.rc
rename to tests/cyclers/50-366_calendar/suite.rc
diff --git a/tests/cyclers/integer1/suite.rc b/tests/cyclers/integer1/suite.rc
index 40734b79a44..1ef0af9ab43 100644
--- a/tests/cyclers/integer1/suite.rc
+++ b/tests/cyclers/integer1/suite.rc
@@ -27,13 +27,13 @@
R1/^ = nang
R/+P1/P6!14 = nong
[runtime]
+ [[root]]
+ script = true
[[foo]]
script = """
-sleep 5
-cylc task message "the cheese is ready"
-sleep 5
-touch typing
- """
+wait
+cylc message -- "${CYLC_SUITE_NAME}" "${CYLC_TASK_JOB}" "the cheese is ready"
+"""
[[[outputs]]]
out1 = "the cheese is ready"
diff --git a/tests/cyclers/r1_initial_immortal/suite.rc b/tests/cyclers/r1_initial_immortal/suite.rc
index f3ed4d82ea5..74ac6fdd78c 100644
--- a/tests/cyclers/r1_initial_immortal/suite.rc
+++ b/tests/cyclers/r1_initial_immortal/suite.rc
@@ -10,7 +10,7 @@
[[root]]
script = true
[[stop]]
- script = "cylc shutdown $CYLC_SUITE_NAME 20140107"
+ script = cylc shutdown "${CYLC_SUITE_NAME}" '20140107'
[visualization]
initial cycle point = 20140101
final cycle point = 20140110
diff --git a/tests/cylc-cat-log/00-local.t b/tests/cylc-cat-log/00-local.t
index dac647635f0..66db1bf2ab7 100755
--- a/tests/cylc-cat-log/00-local.t
+++ b/tests/cylc-cat-log/00-local.t
@@ -24,11 +24,7 @@ install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
#-------------------------------------------------------------------------------
-# Run detached.
-suite_run_ok "${TEST_NAME_BASE}-run" cylc run "${SUITE_NAME}"
-sleep 5
-# Wait for the suite to finish.
-cylc stop --max-polls=10 --interval=2 "${SUITE_NAME}" 2>'/dev/null'
+suite_run_ok "${TEST_NAME_BASE}-run" cylc run --no-detach "${SUITE_NAME}"
#-------------------------------------------------------------------------------
TEST_NAME=${TEST_NAME_BASE}-suite-log-log
run_ok "${TEST_NAME}" cylc cat-log "${SUITE_NAME}"
diff --git a/tests/cylc-cat-log/00-local/suite.rc b/tests/cylc-cat-log/00-local/suite.rc
index f95883c8f85..41973e11e60 100644
--- a/tests/cylc-cat-log/00-local/suite.rc
+++ b/tests/cylc-cat-log/00-local/suite.rc
@@ -1,7 +1,8 @@
[cylc]
[[events]]
- abort on timeout = True
- timeout = PT20S
+ abort on stalled = True
+ abort on inactivity = True
+ inactivity = PT3M
[scheduling]
[[graph]]
R1 = a-task
diff --git a/tests/cylc-cat-log/02-remote-custom-runtime-viewer-pbs.t b/tests/cylc-cat-log/02-remote-custom-runtime-viewer-pbs.t
index 00b3cabc54a..ff97dffe988 100755
--- a/tests/cylc-cat-log/02-remote-custom-runtime-viewer-pbs.t
+++ b/tests/cylc-cat-log/02-remote-custom-runtime-viewer-pbs.t
@@ -35,8 +35,6 @@ CYLC_TEST_DIRECTIVES="$( \
export CYLC_TEST_HOST CYLC_TEST_DIRECTIVES
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
create_test_globalrc "" "
[hosts]
[[${CYLC_TEST_HOST}]]
@@ -44,10 +42,6 @@ create_test_globalrc "" "
[[[[pbs]]]]
err viewer = ${ERR_VIEWER}
out viewer = ${OUT_VIEWER}"
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
+reftest
purge_suite_remote "${CYLC_TEST_HOST}" "${SUITE_NAME}"
-purge_suite "${SUITE_NAME}"
exit
diff --git a/tests/cylc-cat-log/06-log-rotation.t b/tests/cylc-cat-log/06-log-rotation.t
index 5b402a96514..19e2421f305 100755
--- a/tests/cylc-cat-log/06-log-rotation.t
+++ b/tests/cylc-cat-log/06-log-rotation.t
@@ -24,18 +24,12 @@ init_suite "${TEST_NAME_BASE}" '/dev/null'
LOG_DIR="$(dirname "$(cylc cat-log -m p "${SUITE_NAME}")")"
mkdir -p "${LOG_DIR}"
# Note: .0 .1 .2: back compatibility to old log rotation system
-# (short sleeps to get different file mtimes)
-touch "${LOG_DIR}/log.20000103T00Z"
-sleep 1
-touch "${LOG_DIR}/log.20000102T00Z"
-sleep 1
-touch "${LOG_DIR}/log.20000101T00Z"
-sleep 1
-touch "${LOG_DIR}/log.0"
-sleep 1
-touch "${LOG_DIR}/log.1"
-sleep 1
-touch "${LOG_DIR}/log.2"
+touch -t '201001011200.00' "${LOG_DIR}/log.20000103T00Z"
+touch -t '201001011200.01' "${LOG_DIR}/log.20000102T00Z"
+touch -t '201001011200.02' "${LOG_DIR}/log.20000101T00Z"
+touch -t '201001011200.03' "${LOG_DIR}/log.0"
+touch -t '201001011200.04' "${LOG_DIR}/log.1"
+touch -t '201001011200.05' "${LOG_DIR}/log.2"
# Test log rotation.
for I in {0..5}; do
diff --git a/tests/cylc-cat-log/09-cat-running.t b/tests/cylc-cat-log/09-cat-running.t
index 0d513864d0d..2e28c707494 100755
--- a/tests/cylc-cat-log/09-cat-running.t
+++ b/tests/cylc-cat-log/09-cat-running.t
@@ -19,13 +19,6 @@
. "$(dirname "$0")/test_header"
set_test_remote
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --debug --no-detach "${SUITE_NAME}"
-
+reftest
purge_suite_remote "${CYLC_TEST_OWNER}@${CYLC_TEST_HOST}" "${SUITE_NAME}"
-purge_suite "${SUITE_NAME}"
exit
diff --git a/tests/cylc-cat-log/09-cat-running/suite.rc b/tests/cylc-cat-log/09-cat-running/suite.rc
index 5f2b1843102..8072adcc17b 100644
--- a/tests/cylc-cat-log/09-cat-running/suite.rc
+++ b/tests/cylc-cat-log/09-cat-running/suite.rc
@@ -1,18 +1,20 @@
#!Jinja2
[cylc]
[[events]]
- abort on timeout = True
- timeout = PT90S
+ abort on stalled = True
+ abort on inactivity = True
+ inactivity = PT3M
[scheduling]
[[graph]]
R1 = local-task:echo & remote-task:echo => cat-log
[runtime]
[[ECHO]]
script = """
+wait
echo rubbish
echo garbage >&2
cylc message 'echo done'
-sleep 60"""
+"""
[[[outputs]]]
echo = "echo done"
[[local-task]]
@@ -24,7 +26,7 @@ sleep 60"""
owner={{environ["CYLC_TEST_OWNER"]}}
[[cat-log]]
script = """
-sleep 10 # wait for messages for other tasks
+wait
for TASK in local-task.1 remote-task.1; do
cylc cat-log --debug -f o "${CYLC_SUITE_NAME}" $TASK | grep 'rubbish'
cylc cat-log --debug -f e "${CYLC_SUITE_NAME}" $TASK | grep 'garbage'
diff --git a/tests/cylc-get-config/01-no-final/suite.rc b/tests/cylc-get-config/01-no-final/suite.rc
index 872c8c8391a..79f1495369a 100644
--- a/tests/cylc-get-config/01-no-final/suite.rc
+++ b/tests/cylc-get-config/01-no-final/suite.rc
@@ -8,5 +8,5 @@
T06 = foo
[runtime]
[[foo]]
- script = sleep 5
+ script = true
diff --git a/tests/cylc-get-config/05-param-vars/suite.rc b/tests/cylc-get-config/05-param-vars/suite.rc
index 48da80273ac..58520e9a181 100644
--- a/tests/cylc-get-config/05-param-vars/suite.rc
+++ b/tests/cylc-get-config/05-param-vars/suite.rc
@@ -7,7 +7,7 @@
R1 = foo
[runtime]
[[root]]
- script = sleep 1
+ script = true
[[[environment]]]
PARAM1 = 'something'
[[foo]]
diff --git a/tests/cylc-get-cylc-version/00-basic.t b/tests/cylc-get-cylc-version/00-basic.t
index 9a5826cec35..0acc23e7958 100644
--- a/tests/cylc-get-cylc-version/00-basic.t
+++ b/tests/cylc-get-cylc-version/00-basic.t
@@ -16,14 +16,7 @@
# along with this program. If not, see .
#-------------------------------------------------------------------------------
# Basic get-cylc-version test.
-
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/cylc-insert/00-insert.t b/tests/cylc-insert/00-insert.t
index cf2f983acfc..1377b7e088e 100755
--- a/tests/cylc-insert/00-insert.t
+++ b/tests/cylc-insert/00-insert.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test cylc insert command
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run -v -v --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-insert/00-insert/suite.rc b/tests/cylc-insert/00-insert/suite.rc
index a513ce564f4..be7abdbde09 100644
--- a/tests/cylc-insert/00-insert/suite.rc
+++ b/tests/cylc-insert/00-insert/suite.rc
@@ -20,7 +20,7 @@ duration of the suite; and the other is set to stop after two cycles."""
[runtime]
[[root]]
- script = "sleep 1" # quick
+ script = true
[[prep]]
script = """
cylc insert $CYLC_SUITE_NAME bar.20140101T00
diff --git a/tests/cylc-insert/01-insert-bad-cycle-point/suite.rc b/tests/cylc-insert/01-insert-bad-cycle-point/suite.rc
index 4ff583e54dd..6a2faa0bbb6 100644
--- a/tests/cylc-insert/01-insert-bad-cycle-point/suite.rc
+++ b/tests/cylc-insert/01-insert-bad-cycle-point/suite.rc
@@ -3,10 +3,12 @@
[cylc]
cycle point time zone = +01
+ [[events]]
+ abort on stalled = True
[scheduling]
initial cycle point = 20140101T00
- final cycle point = 20140101T00 # 3 cycles
+ final cycle point = 20140101T00
[[special tasks]]
exclude at start-up = foo
[[graph]]
@@ -17,7 +19,7 @@
[runtime]
[[root]]
- script = "sleep 1" # quick
+ script = true
[[prep]]
# Insert the task with a bad cycle point
script = cylc insert $CYLC_SUITE_NAME foo.teatime
diff --git a/tests/cylc-insert/02-insert-bad-stop-cycle-point/suite.rc b/tests/cylc-insert/02-insert-bad-stop-cycle-point/suite.rc
index 3416239160d..9c6e78beeb6 100644
--- a/tests/cylc-insert/02-insert-bad-stop-cycle-point/suite.rc
+++ b/tests/cylc-insert/02-insert-bad-stop-cycle-point/suite.rc
@@ -6,7 +6,7 @@
[scheduling]
initial cycle point = 20140101T00
- final cycle point = 20140101T00 # 3 cycles
+ final cycle point = 20140101T00
[[special tasks]]
exclude at start-up = foo
[[graph]]
@@ -17,7 +17,7 @@
[runtime]
[[root]]
- script = "sleep 1" # quick
+ script = true
[[prep]]
# Insert the task with a bad cycle point
script = cylc insert --stop-point=soon $CYLC_SUITE_NAME foo.20140101T00
diff --git a/tests/cylc-insert/04-insert-family.t b/tests/cylc-insert/04-insert-family.t
index 238c2ed32d2..ff3f03c06fb 100755
--- a/tests/cylc-insert/04-insert-family.t
+++ b/tests/cylc-insert/04-insert-family.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test "cylc insert" on insertion of a task family (and family remove too).
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run -v --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-insert/05-insert-compat.t b/tests/cylc-insert/05-insert-compat.t
index 74774d8d31f..3c50530ce3b 100755
--- a/tests/cylc-insert/05-insert-compat.t
+++ b/tests/cylc-insert/05-insert-compat.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test cylc insert command, back compat
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run -v -v --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-insert/05-insert-compat/suite.rc b/tests/cylc-insert/05-insert-compat/suite.rc
index 79662b11fac..b5c95d1fab1 100644
--- a/tests/cylc-insert/05-insert-compat/suite.rc
+++ b/tests/cylc-insert/05-insert-compat/suite.rc
@@ -20,7 +20,7 @@ duration of the suite; and the other is set to stop after two cycles."""
[runtime]
[[root]]
- script = "sleep 1" # quick
+ script = true
[[prep]]
script = """
cylc insert $CYLC_SUITE_NAME 'bar.20140101T00'
diff --git a/tests/cylc-insert/06-insert-bad-cycle-point-compat/suite.rc b/tests/cylc-insert/06-insert-bad-cycle-point-compat/suite.rc
index e8a9129658d..4f74e0934f9 100644
--- a/tests/cylc-insert/06-insert-bad-cycle-point-compat/suite.rc
+++ b/tests/cylc-insert/06-insert-bad-cycle-point-compat/suite.rc
@@ -3,6 +3,8 @@
[cylc]
cycle point time zone = +01
+ [[events]]
+ abort on stalled = True
[scheduling]
initial cycle point = 20140101T00
@@ -17,7 +19,7 @@
[runtime]
[[root]]
- script = "sleep 1" # quick
+ script = true
[[prep]]
# Insert the task with a bad cycle point
script = cylc insert "$CYLC_SUITE_NAME" 'foo.teatime'
diff --git a/tests/cylc-insert/07-insert-bad-stop-cycle-point/suite.rc b/tests/cylc-insert/07-insert-bad-stop-cycle-point/suite.rc
index 237fb585785..d7974843ef1 100644
--- a/tests/cylc-insert/07-insert-bad-stop-cycle-point/suite.rc
+++ b/tests/cylc-insert/07-insert-bad-stop-cycle-point/suite.rc
@@ -3,10 +3,12 @@
[cylc]
cycle point time zone = +01
+ [[events]]
+ abort on stalled = True
[scheduling]
initial cycle point = 20140101T00
- final cycle point = 20140101T00 # 3 cycles
+ final cycle point = 20140101T00
[[special tasks]]
exclude at start-up = foo
[[graph]]
@@ -17,7 +19,7 @@
[runtime]
[[root]]
- script = "sleep 1" # quick
+ script = true
[[prep]]
# Insert the task with a bad cycle point
script = """
diff --git a/tests/cylc-insert/08-insert-family-compat.t b/tests/cylc-insert/08-insert-family-compat.t
index 238c2ed32d2..ff3f03c06fb 100755
--- a/tests/cylc-insert/08-insert-family-compat.t
+++ b/tests/cylc-insert/08-insert-family-compat.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test "cylc insert" on insertion of a task family (and family remove too).
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run -v --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-insert/09-insert-no-cycle-point/suite.rc b/tests/cylc-insert/09-insert-no-cycle-point/suite.rc
index 4548059c723..b1e6bea6847 100644
--- a/tests/cylc-insert/09-insert-no-cycle-point/suite.rc
+++ b/tests/cylc-insert/09-insert-no-cycle-point/suite.rc
@@ -17,7 +17,7 @@
[runtime]
[[root]]
- script = "sleep 1" # quick
+ script = true
[[prep]]
# Insert the task with a bad cycle point
script = cylc insert "${CYLC_SUITE_NAME}" 'foo'
diff --git a/tests/cylc-insert/11-wildcard.t b/tests/cylc-insert/11-wildcard.t
index f5c21be6a25..9c09d5920fe 100755
--- a/tests/cylc-insert/11-wildcard.t
+++ b/tests/cylc-insert/11-wildcard.t
@@ -18,12 +18,5 @@
# Test cylc insert command, with wildcard in a task name string
. "$(dirname "$0")/test_header"
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/cylc-ping/00-simple.t b/tests/cylc-ping/00-simple.t
index fa1a44eb0dd..f2f21e3864f 100644
--- a/tests/cylc-ping/00-simple.t
+++ b/tests/cylc-ping/00-simple.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test cylc ping behaviour
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" simple
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-ping/simple/reference.log b/tests/cylc-ping/00-simple/reference.log
similarity index 100%
rename from tests/cylc-ping/simple/reference.log
rename to tests/cylc-ping/00-simple/reference.log
diff --git a/tests/cylc-ping/simple/suite.rc b/tests/cylc-ping/00-simple/suite.rc
similarity index 100%
rename from tests/cylc-ping/simple/suite.rc
rename to tests/cylc-ping/00-simple/suite.rc
diff --git a/tests/cylc-poll/00-basic.t b/tests/cylc-poll/00-basic.t
index ee601034384..038bb9f6cc3 100755
--- a/tests/cylc-poll/00-basic.t
+++ b/tests/cylc-poll/00-basic.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test cat-check against suite database
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-poll/00-basic/suite.rc b/tests/cylc-poll/00-basic/suite.rc
index 6ff76649686..3955067065c 100644
--- a/tests/cylc-poll/00-basic/suite.rc
+++ b/tests/cylc-poll/00-basic/suite.rc
@@ -3,6 +3,11 @@
R1 = a:start => b
[runtime]
[[a]]
- script = sleep 20
+ script = """
+while ! grep -qF 'CYLC_JOB_EXIT' "${CYLC_SUITE_RUN_DIR}/log/job/1/b/NN/job.status"
+do
+ sleep 1
+done
+"""
[[b]]
script = cylc poll "$CYLC_SUITE_NAME" 'a.1'
diff --git a/tests/cylc-poll/01-task-failed.t b/tests/cylc-poll/01-task-failed.t
index 8cef2644cc2..bf5ab19f4f1 100755
--- a/tests/cylc-poll/01-task-failed.t
+++ b/tests/cylc-poll/01-task-failed.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test that polling a failed task sets the task state correctly
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-poll/02-task-submit-failed.t b/tests/cylc-poll/02-task-submit-failed.t
index 450a891bfd0..17584a28b77 100755
--- a/tests/cylc-poll/02-task-submit-failed.t
+++ b/tests/cylc-poll/02-task-submit-failed.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test that polling a submit-failed task sets the task state correctly
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-poll/02-task-submit-failed/suite.rc b/tests/cylc-poll/02-task-submit-failed/suite.rc
index 983405d6727..e8e31ddafba 100644
--- a/tests/cylc-poll/02-task-submit-failed/suite.rc
+++ b/tests/cylc-poll/02-task-submit-failed/suite.rc
@@ -14,12 +14,12 @@
batch system = at
batch submit command template = at noon tomorrow
[[poll_foo]]
- script = sleep 5; cylc poll "$CYLC_SUITE_NAME" 'foo'
+ script = wait; cylc poll "$CYLC_SUITE_NAME" 'foo'
[[stop]]
script = cylc stop $CYLC_SUITE_NAME
[[kill_foo_submit]]
script = """
- sleep 5
+ wait
ID=$(sed -n "s/^CYLC_BATCH_SYS_JOB_ID=//p" \
$CYLC_SUITE_RUN_DIR/log/job/1/foo/01/job.status)
atrm $ID
diff --git a/tests/cylc-poll/05-poll-multi-messages.t b/tests/cylc-poll/05-poll-multi-messages.t
index 413c1ae1a77..9cb2391c1e9 100755
--- a/tests/cylc-poll/05-poll-multi-messages.t
+++ b/tests/cylc-poll/05-poll-multi-messages.t
@@ -18,13 +18,5 @@
# Test poll multiple messages
. "$(dirname "$0")/test_header"
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/cylc-poll/05-poll-multi-messages/suite.rc b/tests/cylc-poll/05-poll-multi-messages/suite.rc
index 247dea3d06a..615310eae73 100644
--- a/tests/cylc-poll/05-poll-multi-messages/suite.rc
+++ b/tests/cylc-poll/05-poll-multi-messages/suite.rc
@@ -18,7 +18,13 @@ wait
echo "CYLC_MESSAGE=$(date +%FT%H:%M:%SZ)|INFO|hello1"
echo "CYLC_MESSAGE=$(date +%FT%H:%M:%SZ)|INFO|hello2"
} >>"${CYLC_TASK_LOG_ROOT}.status"
-sleep 30
+LOG="${CYLC_SUITE_LOG_DIR}/log"
+while ! grep -qF '[speaker1.1] status=running: (polled)hello1' "${LOG}"; do
+ sleep 1
+done
+while ! grep -qF '[speaker1.1] status=running: (polled)hello2' "${LOG}"; do
+ sleep 1
+done
"""
[[[outputs]]]
hello1 = "hello1"
@@ -32,7 +38,10 @@ wait
{
echo "CYLC_MESSAGE=$(date +%FT%H:%M:%SZ)|INFO|greet"
} >>"${CYLC_TASK_LOG_ROOT}.status"
-sleep 30
+LOG="${CYLC_SUITE_LOG_DIR}/log"
+while ! grep -qF '[speaker2.1] status=running: (polled)greet' "${LOG}"; do
+ sleep 1
+done
"""
[[[outputs]]]
greet = "greet"
diff --git a/tests/cylc-poll/06-loadleveler.t b/tests/cylc-poll/06-loadleveler.t
index 6fcf186f8dc..61649d26882 100755
--- a/tests/cylc-poll/06-loadleveler.t
+++ b/tests/cylc-poll/06-loadleveler.t
@@ -31,16 +31,8 @@ then
fi
export CYLC_TEST_BATCH_TASK_HOST CYLC_TEST_BATCH_SITE_DIRECTIVES
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
+reftest
if [[ "${CYLC_TEST_BATCH_TASK_HOST}" != 'localhost' ]]; then
purge_suite_remote "${CYLC_TEST_BATCH_TASK_HOST}" "${SUITE_NAME}"
fi
-purge_suite "${SUITE_NAME}"
+exit
diff --git a/tests/cylc-poll/10-basic-compat.t b/tests/cylc-poll/10-basic-compat.t
index ee601034384..038bb9f6cc3 100755
--- a/tests/cylc-poll/10-basic-compat.t
+++ b/tests/cylc-poll/10-basic-compat.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test cat-check against suite database
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-poll/11-event-time/suite.rc b/tests/cylc-poll/11-event-time/suite.rc
index 7b6df644cad..c588a33d691 100644
--- a/tests/cylc-poll/11-event-time/suite.rc
+++ b/tests/cylc-poll/11-event-time/suite.rc
@@ -17,4 +17,4 @@ __STATUS__
exit
"""
[[w2]]
- script=sleep 10; cylc poll "${CYLC_SUITE_NAME}" 'w1.1'
+ script=wait; cylc poll "${CYLC_SUITE_NAME}" 'w1.1'
diff --git a/tests/cylc-remove/00-simple.t b/tests/cylc-remove/00-simple.t
index dc7661b99dd..84a4783406d 100755
--- a/tests/cylc-remove/00-simple.t
+++ b/tests/cylc-remove/00-simple.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test cylc remove
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-remove/01-simple-comat.t b/tests/cylc-remove/01-simple-comat.t
index 54bb921bcdb..4920323f206 100755
--- a/tests/cylc-remove/01-simple-comat.t
+++ b/tests/cylc-remove/01-simple-comat.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test cylc remove, backward compat
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-remove/02-cycling.t b/tests/cylc-remove/02-cycling.t
index 1647d4e3428..7ac879c196f 100755
--- a/tests/cylc-remove/02-cycling.t
+++ b/tests/cylc-remove/02-cycling.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test cylc remove in a cycling suite (spawn before remove, and not).
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-reset/00-compat.t b/tests/cylc-reset/00-compat.t
index 30a363faf11..950c6f2fd43 100755
--- a/tests/cylc-reset/00-compat.t
+++ b/tests/cylc-reset/00-compat.t
@@ -17,12 +17,6 @@
#-------------------------------------------------------------------------------
# Test "cylc reset --state='succeeded SUITE MATCH POINT".
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/cylc-reset/01-filter-failed.t b/tests/cylc-reset/01-filter-failed.t
index ac4f6f464af..f9d89d098ed 100755
--- a/tests/cylc-reset/01-filter-failed.t
+++ b/tests/cylc-reset/01-filter-failed.t
@@ -17,12 +17,6 @@
#-------------------------------------------------------------------------------
# Test "cylc reset --state='succeeded' SUITE '*:failed'".
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/cylc-show/05-complex/suite.rc b/tests/cylc-show/05-complex/suite.rc
index 7a266069f1e..ff07398ac33 100644
--- a/tests/cylc-show/05-complex/suite.rc
+++ b/tests/cylc-show/05-complex/suite.rc
@@ -8,5 +8,5 @@
T00 = a & b & (c | (d & e)) & f[-P1D] => f
[runtime]
[[root]]
- script = sleep 1
+ script = true
[[a, b, c, d, e, f]]
diff --git a/tests/cylc-show/clock-triggered-alt-tz/suite.rc b/tests/cylc-show/clock-triggered-alt-tz/suite.rc
index cb13cf1f1af..b682ae993bd 100644
--- a/tests/cylc-show/clock-triggered-alt-tz/suite.rc
+++ b/tests/cylc-show/clock-triggered-alt-tz/suite.rc
@@ -14,7 +14,7 @@
"""
[runtime]
[[foo]]
- script = sleep 10
+ script = true
[[bar,baz]]
script = true
[[show]]
diff --git a/tests/cylc-show/clock-triggered-non-utc-mode/suite.rc b/tests/cylc-show/clock-triggered-non-utc-mode/suite.rc
index f18ad2bc25f..5cfafae2c11 100644
--- a/tests/cylc-show/clock-triggered-non-utc-mode/suite.rc
+++ b/tests/cylc-show/clock-triggered-non-utc-mode/suite.rc
@@ -13,7 +13,7 @@
"""
[runtime]
[[foo]]
- script = sleep 10
+ script = true
[[bar,baz]]
script = true
[[show]]
diff --git a/tests/cylc-show/clock-triggered/suite.rc b/tests/cylc-show/clock-triggered/suite.rc
index 7867f98b9c6..a8ac9615824 100644
--- a/tests/cylc-show/clock-triggered/suite.rc
+++ b/tests/cylc-show/clock-triggered/suite.rc
@@ -14,7 +14,7 @@
"""
[runtime]
[[foo]]
- script = sleep 10
+ script = true
[[bar,baz]]
script = true
[[show]]
diff --git a/tests/cylc-trigger/00-compat.t b/tests/cylc-trigger/00-compat.t
index d680daecc37..20ca401c30e 100755
--- a/tests/cylc-trigger/00-compat.t
+++ b/tests/cylc-trigger/00-compat.t
@@ -1,7 +1,7 @@
#!/bin/bash
# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
-#
+#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -15,17 +15,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#-------------------------------------------------------------------------------
-# Test intercycle dependencies.
+# Test basic usage of "cylc trigger"
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/cylc-trigger/00-compat/suite.rc b/tests/cylc-trigger/00-compat/suite.rc
index d0bd7590193..ec4662a345c 100644
--- a/tests/cylc-trigger/00-compat/suite.rc
+++ b/tests/cylc-trigger/00-compat/suite.rc
@@ -1,11 +1,8 @@
[scheduling]
[[graph]]
- R1 = "foo => bar"
+ R1 = foo => bar
[runtime]
[[foo]]
- script = """
- cylc trigger $CYLC_SUITE_NAME bar 1
- sleep 10
- cylc stop $CYLC_SUITE_NAME
- sleep 10
- """
+ script = cylc trigger "${CYLC_SUITE_NAME}" 'bar.1'
+ [[bar]]
+ script = true
diff --git a/tests/cylc-trigger/01-queued/suite.rc b/tests/cylc-trigger/01-queued/suite.rc
index 6cb4d02267e..70910090183 100644
--- a/tests/cylc-trigger/01-queued/suite.rc
+++ b/tests/cylc-trigger/01-queued/suite.rc
@@ -10,11 +10,9 @@
[[foo]]
inherit = METASYNTACTIC
script = """
- sleep 10
+ wait
cylc trigger $CYLC_SUITE_NAME bar 1
- sleep 10
- cylc stop $CYLC_SUITE_NAME
- sleep 10
"""
[[bar]]
inherit = METASYNTACTIC
+ script = true
diff --git a/tests/cylc-trigger/02-filter-failed.t b/tests/cylc-trigger/02-filter-failed.t
index c28bc722cc2..29ba6f09173 100755
--- a/tests/cylc-trigger/02-filter-failed.t
+++ b/tests/cylc-trigger/02-filter-failed.t
@@ -17,12 +17,6 @@
#-------------------------------------------------------------------------------
# Test "cylc trigger SUITE '*:failed'".
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/cylc-trigger/06-reset-ready.t b/tests/cylc-trigger/06-reset-ready.t
index 160186c6883..cc47a2bf03a 100755
--- a/tests/cylc-trigger/06-reset-ready.t
+++ b/tests/cylc-trigger/06-reset-ready.t
@@ -18,12 +18,6 @@
# Test "cylc trigger" a task should reset its output to incomplete.
# https://github.com/cylc/cylc-flow/issues/1852
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/cylc-trigger/06-reset-ready/suite.rc b/tests/cylc-trigger/06-reset-ready/suite.rc
index 4b7eba4fc99..1d4156bc85a 100644
--- a/tests/cylc-trigger/06-reset-ready/suite.rc
+++ b/tests/cylc-trigger/06-reset-ready/suite.rc
@@ -11,10 +11,8 @@ t2:failed => t3
[[t1]]
script = """
if ((${CYLC_TASK_SUBMIT_NUMBER} >= 2)); then
- cylc reset --state='waiting' "${CYLC_SUITE_NAME}" 't2.1'
- sleep 5
cylc broadcast "${CYLC_SUITE_NAME}" -s '[environment]CYLC_TEST_VAR_X=x'
- sleep 5
+ cylc reset --state='waiting' "${CYLC_SUITE_NAME}" 't2.1'
fi
"""
[[t2]]
diff --git a/tests/cylc.wallclock/00-single.t b/tests/cylc.wallclock/00-single.t
deleted file mode 100755
index c4775681e2a..00000000000
--- a/tests/cylc.wallclock/00-single.t
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/bash
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#-------------------------------------------------------------------------------
-# Unit test parts of lib/isodatetime/wallclock.py.
-. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
-set_test_number 7
-
-# Arguments: TEST_NAME TIME_STRING EXPECTED_UNIX_TIME CALENDAR_IS_360
-function test_get_unix_time_from_time_string () {
- run_ok "$1" python3 <<__PYTHON__
-from cylc.flow.wallclock import get_unix_time_from_time_string
-
-if $4:
- from metomi.isodatetime.data import CALENDAR
- CALENDAR.set_mode(CALENDAR.MODE_360)
-assert(get_unix_time_from_time_string('$2') == $3)
-__PYTHON__
-}
-#-------------------------------------------------------------------------------
-TEST_NAME=${TEST_NAME_BASE}-get_unix_time_from_time_string-greg-1
-test_get_unix_time_from_time_string "${TEST_NAME}" '2016-09-08T09:09:00+01' 1473322140 False
-#-------------------------------------------------------------------------------
-TEST_NAME=${TEST_NAME_BASE}-get_unix_time_from_time_string-greg-2
-test_get_unix_time_from_time_string "${TEST_NAME}" '2016-09-08T08:09:00Z' 1473322140 False
-#-------------------------------------------------------------------------------
-TEST_NAME=${TEST_NAME_BASE}-get_unix_time_from_time_string-greg-3
-test_get_unix_time_from_time_string "${TEST_NAME}" '2016-09-07T20:09:00-12' 1473322140 False
-#-------------------------------------------------------------------------------
-TEST_NAME=${TEST_NAME_BASE}-get_unix_time_from_time_string-360-1
-test_get_unix_time_from_time_string "${TEST_NAME}" '2016-09-08T09:09:00+01' 1473322140 True
-#-------------------------------------------------------------------------------
-TEST_NAME=${TEST_NAME_BASE}-get_unix_time_from_time_string-360-2
-test_get_unix_time_from_time_string "${TEST_NAME}" '2016-09-08T08:09:00Z' 1473322140 True
-#-------------------------------------------------------------------------------
-TEST_NAME=${TEST_NAME_BASE}-get_unix_time_from_time_string-360-3
-test_get_unix_time_from_time_string "${TEST_NAME}" '2016-09-07T20:09:00-12' 1473322140 True
-#-------------------------------------------------------------------------------
-TEST_NAME=${TEST_NAME_BASE}-get_unix_time_from_time_string-360-31-1
-test_get_unix_time_from_time_string "${TEST_NAME}" '2016-08-31T18:09:00+01' 1472663340 True
-exit
diff --git a/tests/database/04-lock-recover/suite.rc b/tests/database/04-lock-recover/suite.rc
index 8baf3ab7ad5..65434bff594 100644
--- a/tests/database/04-lock-recover/suite.rc
+++ b/tests/database/04-lock-recover/suite.rc
@@ -8,7 +8,7 @@ TRUES:succeed-all => done
"""
[runtime]
[[locker]]
- script = sleep 5; timeout 60 cylc-db-lock
+ script = wait; timeout 60 cylc-db-lock
[[[outputs]]]
lock = "I have locked the public database file"
[[TRUES]]
diff --git a/tests/directives/00-loadleveler.t b/tests/directives/00-loadleveler.t
index 7ff2bd8e688..5135542f77e 100644
--- a/tests/directives/00-loadleveler.t
+++ b/tests/directives/00-loadleveler.t
@@ -37,12 +37,6 @@ fi
export CYLC_TEST_BATCH_TASK_HOST CYLC_TEST_BATCH_SITE_DIRECTIVES
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${BATCH_SYS_NAME}"
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
+reftest "${TEST_NAME_BASE}" "${BATCH_SYS_NAME}"
purge_suite_remote "${CYLC_TEST_BATCH_TASK_HOST}" "${SUITE_NAME}"
-purge_suite "${SUITE_NAME}"
exit
diff --git a/tests/directives/01-at.t b/tests/directives/01-at.t
index 83da787cc7e..95fcaa11a43 100644
--- a/tests/directives/01-at.t
+++ b/tests/directives/01-at.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test at submission
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" at
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/directives/at/reference.log b/tests/directives/01-at/reference.log
similarity index 100%
rename from tests/directives/at/reference.log
rename to tests/directives/01-at/reference.log
diff --git a/tests/directives/at/suite.rc b/tests/directives/01-at/suite.rc
similarity index 100%
rename from tests/directives/at/suite.rc
rename to tests/directives/01-at/suite.rc
diff --git a/tests/events/23-suite-stalled-handler.t b/tests/events/23-suite-stalled-handler.t
index e3d663d5d97..56f5a901d01 100755
--- a/tests/events/23-suite-stalled-handler.t
+++ b/tests/events/23-suite-stalled-handler.t
@@ -18,12 +18,5 @@
# Test suite event handler, flexible interface with stalled handler
. "$(dirname "$0")/test_header"
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-run_ok "${TEST_NAME_BASE}-validate" \
- cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/events/35-task-event-handler-importance/suite.rc b/tests/events/35-task-event-handler-importance/suite.rc
index 23e0e312b1b..e69621eb0a0 100644
--- a/tests/events/35-task-event-handler-importance/suite.rc
+++ b/tests/events/35-task-event-handler-importance/suite.rc
@@ -5,7 +5,7 @@
R1 = t1:fail => dummy => !t1
[runtime]
[[dummy]]
- script = sleep 5
+ script = true
[[t1]]
script = false
[[[meta]]]
diff --git a/tests/events/38-task-event-handler-custom/suite.rc b/tests/events/38-task-event-handler-custom/suite.rc
index 516651ca0d7..8a07e8805df 100644
--- a/tests/events/38-task-event-handler-custom/suite.rc
+++ b/tests/events/38-task-event-handler-custom/suite.rc
@@ -4,23 +4,20 @@
foo:b => baz"""
[runtime]
[[root]]
- script = sleep 1
+ script = true
[[[events]]]
custom handler = echo !!CUSTOM!! %(name)s.%(point)s %(fish)s %(message)s
[[[meta]]]
fish = trout
[[foo]]
script = """
-sleep 2
+wait
# Output message for triggering, and custom event handler.
cylc message -p CUSTOM "Data ready for barring"
-sleep 2
# Generic message, not for triggering or custom event handler.
cylc message "Aren't the hydrangeas nice?"
-sleep 2
# Output message for triggering, not custom event handler.
cylc message "Data ready for bazzing"
-sleep 2
"""
[[[outputs]]]
a = "Data ready for barring"
diff --git a/tests/ext-trigger/00-satellite.t b/tests/ext-trigger/00-satellite.t
index cc89c22769f..21b289ca751 100644
--- a/tests/ext-trigger/00-satellite.t
+++ b/tests/ext-trigger/00-satellite.t
@@ -17,14 +17,6 @@
#-------------------------------------------------------------------------------
# Validate and run the external trigger test suite
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/hold-release/00-suite.t b/tests/hold-release/00-suite.t
index e9063107bac..6cff5dfbec9 100755
--- a/tests/hold-release/00-suite.t
+++ b/tests/hold-release/00-suite.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test suite hold and release, with cycling and async tasks present.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" suite
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-val"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/hold-release/suite/reference.log b/tests/hold-release/00-suite/reference.log
similarity index 100%
rename from tests/hold-release/suite/reference.log
rename to tests/hold-release/00-suite/reference.log
diff --git a/tests/hold-release/suite/suite.rc b/tests/hold-release/00-suite/suite.rc
similarity index 75%
rename from tests/hold-release/suite/suite.rc
rename to tests/hold-release/00-suite/suite.rc
index 25ce938a23b..50147a0d51c 100644
--- a/tests/hold-release/suite/suite.rc
+++ b/tests/hold-release/00-suite/suite.rc
@@ -20,10 +20,14 @@
[runtime]
[[holdrelease]]
script = """
-sleep 5
-cylc hold $CYLC_SUITE_NAME
-sleep 5
-cylc release $CYLC_SUITE_NAME
-sleep 5"""
+wait
+cylc hold "${CYLC_SUITE_NAME}"
+while ! grep -qF 'INFO - Command succeeded: hold_suite()' \
+ "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
+cylc release "${CYLC_SUITE_NAME}"
+"""
[[foo,bar]]
script = true
diff --git a/tests/hold-release/01-beyond-stop.t b/tests/hold-release/01-beyond-stop.t
index 1cb57dc5da1..2a557177bf1 100755
--- a/tests/hold-release/01-beyond-stop.t
+++ b/tests/hold-release/01-beyond-stop.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test suite hold and release, with tasks present held beyond stop point.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" suite
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-val"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/hold-release/beyond-stop/reference.log b/tests/hold-release/01-beyond-stop/reference.log
similarity index 100%
rename from tests/hold-release/beyond-stop/reference.log
rename to tests/hold-release/01-beyond-stop/reference.log
diff --git a/tests/hold-release/beyond-stop/suite.rc b/tests/hold-release/01-beyond-stop/suite.rc
similarity index 96%
rename from tests/hold-release/beyond-stop/suite.rc
rename to tests/hold-release/01-beyond-stop/suite.rc
index 6ae9d287bb1..aa333b77411 100644
--- a/tests/hold-release/beyond-stop/suite.rc
+++ b/tests/hold-release/01-beyond-stop/suite.rc
@@ -25,4 +25,4 @@ cylc release $CYLC_SUITE_NAME"""
[[foo]]
script = true
[[bar]]
- script = sleep 10
+ script = true
diff --git a/tests/hold-release/02-hold-on-spawn.t b/tests/hold-release/02-hold-on-spawn.t
index 9ddc763b548..4320f276925 100755
--- a/tests/hold-release/02-hold-on-spawn.t
+++ b/tests/hold-release/02-hold-on-spawn.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test suite hold and limited task release, making sure spawned tasks are held
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" hold-on-spawn
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-val"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/hold-release/hold-on-spawn/reference.log b/tests/hold-release/02-hold-on-spawn/reference.log
similarity index 100%
rename from tests/hold-release/hold-on-spawn/reference.log
rename to tests/hold-release/02-hold-on-spawn/reference.log
diff --git a/tests/hold-release/hold-on-spawn/suite.rc b/tests/hold-release/02-hold-on-spawn/suite.rc
similarity index 69%
rename from tests/hold-release/hold-on-spawn/suite.rc
rename to tests/hold-release/02-hold-on-spawn/suite.rc
index f78b3b979f3..3d11a3c6ccb 100644
--- a/tests/hold-release/hold-on-spawn/suite.rc
+++ b/tests/hold-release/02-hold-on-spawn/suite.rc
@@ -14,13 +14,17 @@
[runtime]
[[holdrelease]]
script = """
-sleep 5
-cylc hold $CYLC_SUITE_NAME
-sleep 5
+wait
+cylc hold "${CYLC_SUITE_NAME}"
+while ! grep -qF 'INFO - Command succeeded: hold_suite()' \
+ "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
# Release all tasks in the first cycle point.
-cylc release "$CYLC_SUITE_NAME" '*.20141009T0000Z'
+cylc release "${CYLC_SUITE_NAME}" '*.20141009T0000Z'
"""
[[foo,bar]]
script = true
[[stop]]
- script = cylc stop $CYLC_SUITE_NAME
+ script = cylc stop "${CYLC_SUITE_NAME}"
diff --git a/tests/hold-release/11-retrying.t b/tests/hold-release/11-retrying.t
index 51660639a90..a09e2bb07f1 100755
--- a/tests/hold-release/11-retrying.t
+++ b/tests/hold-release/11-retrying.t
@@ -17,13 +17,6 @@
#-------------------------------------------------------------------------------
# Test hold and release of a retrying task.
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/hold-release/12-hold-then-retry.t b/tests/hold-release/12-hold-then-retry.t
index 9420f22c7e2..af78f51da6e 100755
--- a/tests/hold-release/12-hold-then-retry.t
+++ b/tests/hold-release/12-hold-then-retry.t
@@ -17,13 +17,6 @@
#-------------------------------------------------------------------------------
# Test suite hold => retry and submit-retry => suite release
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/hold-release/17-hold-after-point.t b/tests/hold-release/17-hold-after-point.t
index 7459fe070ac..6486e6b11e7 100755
--- a/tests/hold-release/17-hold-after-point.t
+++ b/tests/hold-release/17-hold-after-point.t
@@ -17,16 +17,6 @@
#-------------------------------------------------------------------------------
# Test defining a hold after point in a suite.rc file
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" hold-after-point
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-val"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test \
- --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/hold-release/hold-after-point/reference.log b/tests/hold-release/17-hold-after-point/reference.log
similarity index 100%
rename from tests/hold-release/hold-after-point/reference.log
rename to tests/hold-release/17-hold-after-point/reference.log
diff --git a/tests/hold-release/hold-after-point/suite.rc b/tests/hold-release/17-hold-after-point/suite.rc
similarity index 70%
rename from tests/hold-release/hold-after-point/suite.rc
rename to tests/hold-release/17-hold-after-point/suite.rc
index 58b827a5f15..1fa81a23041 100644
--- a/tests/hold-release/hold-after-point/suite.rc
+++ b/tests/hold-release/17-hold-after-point/suite.rc
@@ -15,9 +15,13 @@
T00 = foo[-P1D] => foo
[runtime]
[[stopper]]
- script = sleep 30; cylc stop $CYLC_SUITE_NAME
+ script = """
+while ! grep -qF 'holding (beyond suite hold point) 20100102T00Z' \
+ "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
+cylc stop "${CYLC_SUITE_NAME}"
+"""
[[foo]]
script = true
-[runtime]
- [[foo]]
- script = sleep 5
diff --git a/tests/hold-release/18-hold-cycle-globs.t b/tests/hold-release/18-hold-cycle-globs.t
index 1a3cceeb5ed..8bcda925001 100755
--- a/tests/hold-release/18-hold-cycle-globs.t
+++ b/tests/hold-release/18-hold-cycle-globs.t
@@ -18,10 +18,5 @@
# Test hold cycle point glob
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/hold-release/19-no-reset-prereq-on-waiting.t b/tests/hold-release/19-no-reset-prereq-on-waiting.t
index bcba2b51cc3..03c7aefdd59 100755
--- a/tests/hold-release/19-no-reset-prereq-on-waiting.t
+++ b/tests/hold-release/19-no-reset-prereq-on-waiting.t
@@ -18,10 +18,5 @@
# Test on release of a waiting task, don't reset its prerequisites
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/hold-release/19-no-reset-prereq-on-waiting/suite.rc b/tests/hold-release/19-no-reset-prereq-on-waiting/suite.rc
index 33757251482..865cd473583 100644
--- a/tests/hold-release/19-no-reset-prereq-on-waiting/suite.rc
+++ b/tests/hold-release/19-no-reset-prereq-on-waiting/suite.rc
@@ -17,6 +17,13 @@ t1[-P1Y] => t1
[[t1]]
script = true
[[holder]]
- script = cylc hold "${CYLC_SUITE_NAME}" 't1.2016'; sleep 1
+ script = """
+cylc hold "${CYLC_SUITE_NAME}" 't1.2016'
+while ! grep -qF '[t1.20160101T0000Z] -waiting => waiting (held)' \
+ "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
+"""
[[releaser]]
- script = sleep 1; cylc release "${CYLC_SUITE_NAME}" 't1.2016'
+ script = wait; cylc release "${CYLC_SUITE_NAME}" 't1.2016'
diff --git a/tests/hold-release/hold-family/suite.rc b/tests/hold-release/hold-family/suite.rc
index 96ba127e67b..333d613f882 100644
--- a/tests/hold-release/hold-family/suite.rc
+++ b/tests/hold-release/hold-family/suite.rc
@@ -15,9 +15,13 @@ point of the suite."""
[runtime]
[[holdrelease]]
script = """
-sleep 5
+wait
cylc hold "$CYLC_SUITE_NAME" '{{HOLD_MATCH}}.20141009T0000Z'
-sleep 5
+while ! grep -qF "[foo.20141009T0000Z] -waiting => waiting (held)" \
+ "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
"""
[[STUFF]]
[[STOP]]
@@ -26,4 +30,4 @@ sleep 5
script = true
[[stop]]
inherit = STOP
- script = sleep 5; cylc stop $CYLC_SUITE_NAME
+ script = cylc stop "${CYLC_SUITE_NAME}"
diff --git a/tests/hold-release/hold-task/suite.rc b/tests/hold-release/hold-task/suite.rc
index 8c7af7753c0..d2f94397d44 100644
--- a/tests/hold-release/hold-task/suite.rc
+++ b/tests/hold-release/hold-task/suite.rc
@@ -15,11 +15,14 @@ point of the suite."""
[runtime]
[[holdrelease]]
script = """
-sleep 5
cylc hold "$CYLC_SUITE_NAME" '{{HOLD_MATCH}}.20141009T0000Z'
-sleep 5
+while ! grep -qF "[foo.20141009T0000Z] -waiting => waiting (held)" \
+ "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
"""
[[foo]]
script = true
[[stop]]
- script = sleep 5; cylc stop $CYLC_SUITE_NAME
+ script = cylc stop "${CYLC_SUITE_NAME}"
diff --git a/tests/hold-release/release-family/suite.rc b/tests/hold-release/release-family/suite.rc
index 59012eccc28..5bcfc72c34a 100644
--- a/tests/hold-release/release-family/suite.rc
+++ b/tests/hold-release/release-family/suite.rc
@@ -15,9 +15,13 @@ first cycle point of the suite.."""
[runtime]
[[holdrelease]]
script = """
-sleep 5
-cylc hold $CYLC_SUITE_NAME
-sleep 5
+wait
+cylc hold "${CYLC_SUITE_NAME}"
+while ! grep -qF 'INFO - Command succeeded: hold_suite()' \
+ "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
# Release a family in the first cycle point.
cylc release "$CYLC_SUITE_NAME" '{{RELEASE_MATCH}}.20141009T0000Z'
"""
@@ -28,7 +32,7 @@ cylc release "$CYLC_SUITE_NAME" '{{RELEASE_MATCH}}.20141009T0000Z'
script = true
[[stop]]
inherit = STUFF
- script = cylc stop $CYLC_SUITE_NAME
+ script = cylc stop "${CYLC_SUITE_NAME}"
[[unrelated]]
inherit = UNRELATED
script = true
diff --git a/tests/hold-release/release-task/suite.rc b/tests/hold-release/release-task/suite.rc
index 3085f6e4351..984a87f52c9 100644
--- a/tests/hold-release/release-task/suite.rc
+++ b/tests/hold-release/release-task/suite.rc
@@ -15,13 +15,17 @@ first cycle point of the suite."""
[runtime]
[[holdrelease]]
script = """
-sleep 5
-cylc hold $CYLC_SUITE_NAME
-sleep 5
+wait
+cylc hold "${CYLC_SUITE_NAME}"
+while ! grep -qF 'INFO - Command succeeded: hold_suite()' \
+ "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
# Release tasks in the first cycle point.
cylc release "$CYLC_SUITE_NAME" '{{RELEASE_MATCH}}.20141009T0000Z'
"""
[[foo,bar]]
script = true
[[stop]]
- script = cylc stop $CYLC_SUITE_NAME
+ script = cylc stop "${CYLC_SUITE_NAME}"
diff --git a/tests/host-select/01-timeout/reference.log b/tests/host-select/01-timeout/reference.log
deleted file mode 100644
index 3600f20c03e..00000000000
--- a/tests/host-select/01-timeout/reference.log
+++ /dev/null
@@ -1,3 +0,0 @@
-2015-02-02T15:14:04Z INFO - Initial point: 1
-2015-02-02T15:14:04Z INFO - Final point: 1
-2015-02-02T15:14:04Z INFO - [foo.1] -triggered off []
diff --git a/tests/host-select/01-timeout/suite.rc b/tests/host-select/01-timeout/suite.rc
deleted file mode 100644
index 3b257883d5e..00000000000
--- a/tests/host-select/01-timeout/suite.rc
+++ /dev/null
@@ -1,15 +0,0 @@
-#!Jinja2
-[meta]
- title = Test task host selection with a bad command
-
-[cylc]
- [[reference test]]
- expected task failures = foo.1
-[scheduling]
- [[graph]]
- R1 = "foo:submit-fail => !foo"
-[runtime]
- [[foo]]
- script = true
- [[[remote]]]
- host = $(sleep 2; echo 'localhost')
diff --git a/tests/intercycle/00-past.t b/tests/intercycle/00-past.t
index cba2dff3d66..451458a61d1 100644
--- a/tests/intercycle/00-past.t
+++ b/tests/intercycle/00-past.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test intercycle dependencies.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" past
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/intercycle/past/reference.log b/tests/intercycle/00-past/reference.log
similarity index 100%
rename from tests/intercycle/past/reference.log
rename to tests/intercycle/00-past/reference.log
diff --git a/tests/intercycle/past/suite.rc b/tests/intercycle/00-past/suite.rc
similarity index 100%
rename from tests/intercycle/past/suite.rc
rename to tests/intercycle/00-past/suite.rc
diff --git a/tests/intercycle/01-future.t b/tests/intercycle/01-future.t
index d8699ace663..dee52e6ac38 100644
--- a/tests/intercycle/01-future.t
+++ b/tests/intercycle/01-future.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test future cycle dependencies.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" future
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/intercycle/future/reference.log b/tests/intercycle/01-future/reference.log
similarity index 100%
rename from tests/intercycle/future/reference.log
rename to tests/intercycle/01-future/reference.log
diff --git a/tests/intercycle/future/suite.rc b/tests/intercycle/01-future/suite.rc
similarity index 100%
rename from tests/intercycle/future/suite.rc
rename to tests/intercycle/01-future/suite.rc
diff --git a/tests/intercycle/03-datetime-abs.t b/tests/intercycle/03-datetime-abs.t
index 13b978b89a6..d30dd67bfda 100644
--- a/tests/intercycle/03-datetime-abs.t
+++ b/tests/intercycle/03-datetime-abs.t
@@ -18,13 +18,6 @@
# Test dependency on absolute datetime point
# https://github.com/cylc/cylc-flow/issues/1951
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/intercycle/05-datetime-abs-3.t b/tests/intercycle/05-datetime-abs-3.t
index 13b978b89a6..d30dd67bfda 100644
--- a/tests/intercycle/05-datetime-abs-3.t
+++ b/tests/intercycle/05-datetime-abs-3.t
@@ -18,13 +18,6 @@
# Test dependency on absolute datetime point
# https://github.com/cylc/cylc-flow/issues/1951
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/jinja2/commandline-set/suite.rc b/tests/jinja2/commandline-set/suite.rc
index 893ec89748f..afa9d20db2a 100644
--- a/tests/jinja2/commandline-set/suite.rc
+++ b/tests/jinja2/commandline-set/suite.rc
@@ -15,4 +15,4 @@ to pass the test run their values must be 'foo' and '2', respectively."""
P{{STEP}}Y = {{TASKNAME}}
[runtime]
[[ {{TASKNAME}} ]]
- script = "sleep 1"
+ script = true
diff --git a/tests/job-file-trap/02-pipefail/suite.rc b/tests/job-file-trap/02-pipefail/suite.rc
index 31036a7b42e..3545be91693 100644
--- a/tests/job-file-trap/02-pipefail/suite.rc
+++ b/tests/job-file-trap/02-pipefail/suite.rc
@@ -1,7 +1,7 @@
#!jinja2
[cylc]
[[events]]
- abort if any task fails = True
+ abort on stalled = True
[[reference test]]
expected task failures = t1.1
diff --git a/tests/job-kill/02-loadleveler.t b/tests/job-kill/02-loadleveler.t
index 935843b65ca..414c116b88f 100755
--- a/tests/job-kill/02-loadleveler.t
+++ b/tests/job-kill/02-loadleveler.t
@@ -31,17 +31,8 @@ then
fi
export CYLC_TEST_BATCH_TASK_HOST CYLC_TEST_BATCH_SITE_DIRECTIVES
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
+reftest
if [[ $CYLC_TEST_BATCH_TASK_HOST != 'localhost' ]]; then
purge_suite_remote "${CYLC_TEST_BATCH_TASK_HOST}" "${SUITE_NAME}"
fi
-purge_suite "${SUITE_NAME}"
exit
diff --git a/tests/job-submission/00-user.t b/tests/job-submission/00-user.t
index 6a1bc87bc63..de6b16cc226 100755
--- a/tests/job-submission/00-user.t
+++ b/tests/job-submission/00-user.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test user-defined batch system handlers can be used.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/job-submission/08-activity-log-host/suite.rc b/tests/job-submission/08-activity-log-host/suite.rc
index 9253d0e3ba7..551db056a3f 100644
--- a/tests/job-submission/08-activity-log-host/suite.rc
+++ b/tests/job-submission/08-activity-log-host/suite.rc
@@ -17,4 +17,4 @@
[[killer]]
script = cylc kill "${CYLC_SUITE_NAME}" sleeper.1999
[[releaser]]
- script = sleep 5; cylc release "${CYLC_SUITE_NAME}" sleeper.1999
+ script = wait; cylc release "${CYLC_SUITE_NAME}" sleeper.1999
diff --git a/tests/job-submission/15-garbage-host-command-2.t b/tests/job-submission/15-garbage-host-command-2.t
index 2c27d78a44b..474dbbc65a3 100755
--- a/tests/job-submission/15-garbage-host-command-2.t
+++ b/tests/job-submission/15-garbage-host-command-2.t
@@ -17,12 +17,6 @@
#-------------------------------------------------------------------------------
# Test recovery of a failed host select command for a group of tasks.
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/job-submission/17-remote-localtime.t b/tests/job-submission/17-remote-localtime.t
index c299fa88bfd..8f9c1bb6f79 100755
--- a/tests/job-submission/17-remote-localtime.t
+++ b/tests/job-submission/17-remote-localtime.t
@@ -18,18 +18,10 @@
# Test that suite does not set remote job TZ when in local time.
export CYLC_TEST_IS_GENERIC=false
. "$(dirname "$0")/test_header"
-
set_test_remote_host
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
+reftest
if [[ "${CYLC_TEST_HOST}" != 'localhost' ]]; then
purge_suite_remote "${CYLC_TEST_HOST}" "${SUITE_NAME}"
fi
-purge_suite "${SUITE_NAME}"
exit
diff --git a/tests/jobscript/13-hostname.t b/tests/jobscript/13-hostname.t
index 79e76782b4a..7eb2a05bebd 100755
--- a/tests/jobscript/13-hostname.t
+++ b/tests/jobscript/13-hostname.t
@@ -18,10 +18,5 @@
# Test HOSTNAME does not get modified by job script.
. "$(dirname "${0}")/test_header"
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}-run" cylc run "${SUITE_NAME}" --reference-test --debug --no-detach
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/jobscript/15-semicolon.t b/tests/jobscript/15-semicolon.t
index 3e7e63b72ad..e2c3dc7e7fc 100755
--- a/tests/jobscript/15-semicolon.t
+++ b/tests/jobscript/15-semicolon.t
@@ -19,10 +19,5 @@
# should trigger.
. "$(dirname "${0}")/test_header"
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}-run" cylc run "${SUITE_NAME}" --reference-test --debug --no-detach
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/jobscript/16-midfail.t b/tests/jobscript/16-midfail.t
index 4371a6a63c1..e554cc5de0f 100755
--- a/tests/jobscript/16-midfail.t
+++ b/tests/jobscript/16-midfail.t
@@ -18,10 +18,5 @@
# Test error trapping of internal commands in job script. GitHub #2218
. "$(dirname "${0}")/test_header"
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}-run" cylc run "${SUITE_NAME}" --reference-test --debug --no-detach
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/lib/bash/test_header b/tests/lib/bash/test_header
index c48d2ee2e59..28ea803e667 100644
--- a/tests/lib/bash/test_header
+++ b/tests/lib/bash/test_header
@@ -97,6 +97,10 @@
# echo "1..0 # SKIP SKIP_REASON" and exit.
# ssh_install_cylc HOST
# Install cylc on a remote host.
+# reftest [TEST_NAME [SOURCE]]
+# Install a reference suite using `install_suite`, run a validation
+# test on the suite and run the reference suite with `suite_run_ok`.
+# Run `purge_suite` on success. Expect 2 OK tests.
# create_test_globalrc [PRE [POST]]
# Create a new global config file $PWD/etc from site global-tests.rc
# with PRE and POST pre- and ap-pended (PRE for top level items with no
@@ -398,8 +402,8 @@ init_suite() {
}
install_suite() {
- local TEST_NAME="$1"
- local TEST_SOURCE_BASE="$2"
+ local TEST_NAME="${1:-${TEST_NAME_BASE}}"
+ local TEST_SOURCE_BASE="${2:-${TEST_NAME}}"
SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}"
SUITE_RUN_DIR="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}"
mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
@@ -580,6 +584,15 @@ mock_smtpd_kill() { # Logic borrowed from Rose
fi
}
+reftest() {
+ local TEST_NAME="${1:-${TEST_NAME_BASE}}"
+ install_suite "$@"
+ run_ok "${TEST_NAME}-validate" cylc validate "${SUITE_NAME}"
+ suite_run_ok "${TEST_NAME}-run" \
+ cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
+ purge_suite "${SUITE_NAME}"
+}
+
create_test_globalrc() {
# (Documented in file header).
local PRE=
diff --git a/tests/message-triggers/00-basic.t b/tests/message-triggers/00-basic.t
index c4c5d71133a..a20f3d49b63 100644
--- a/tests/message-triggers/00-basic.t
+++ b/tests/message-triggers/00-basic.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test new-style simplified message triggers (see GitHub #1761)
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/message-triggers/00-basic/suite.rc b/tests/message-triggers/00-basic/suite.rc
index 1e5b8993b74..7bb8567ecdf 100644
--- a/tests/message-triggers/00-basic/suite.rc
+++ b/tests/message-triggers/00-basic/suite.rc
@@ -15,11 +15,10 @@
[runtime]
[[foo]]
script = """
-sleep 2
+wait
cylc message -- "${CYLC_SUITE_NAME} "${CYLC_TASK_JOB} "file 1 done"
-sleep 2
cylc message -- "${CYLC_SUITE_NAME} "${CYLC_TASK_JOB} "file 2 done"
-sleep 2"""
+"""
[[[outputs]]]
out1 = "file 1 done"
out2 = "file 2 done"
diff --git a/tests/modes/00-simulation.t b/tests/modes/00-simulation.t
index 3577dcdd626..6dc5280fc6b 100644
--- a/tests/modes/00-simulation.t
+++ b/tests/modes/00-simulation.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test simulation mode
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" simulation
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/modes/simulation/reference.log b/tests/modes/00-simulation/reference.log
similarity index 100%
rename from tests/modes/simulation/reference.log
rename to tests/modes/00-simulation/reference.log
diff --git a/tests/modes/simulation/suite.rc b/tests/modes/00-simulation/suite.rc
similarity index 100%
rename from tests/modes/simulation/suite.rc
rename to tests/modes/00-simulation/suite.rc
diff --git a/tests/modes/01-dummy.t b/tests/modes/01-dummy.t
index 87b1a0d98fc..1e2e2af0040 100644
--- a/tests/modes/01-dummy.t
+++ b/tests/modes/01-dummy.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test dummy mode
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" dummy
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/modes/dummy/reference.log b/tests/modes/01-dummy/reference.log
similarity index 100%
rename from tests/modes/dummy/reference.log
rename to tests/modes/01-dummy/reference.log
diff --git a/tests/modes/dummy/suite.rc b/tests/modes/01-dummy/suite.rc
similarity index 100%
rename from tests/modes/dummy/suite.rc
rename to tests/modes/01-dummy/suite.rc
diff --git a/tests/modes/02-dummy-message-outputs.t b/tests/modes/02-dummy-message-outputs.t
index 3a20b56f96d..8d2fb619a5b 100644
--- a/tests/modes/02-dummy-message-outputs.t
+++ b/tests/modes/02-dummy-message-outputs.t
@@ -20,7 +20,7 @@
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" 'dummy-message-outputs'
+install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
suite_run_ok "${TEST_NAME_BASE}-run" \
cylc run --no-detach --mode=dummy --reference-test "${SUITE_NAME}"
diff --git a/tests/modes/dummy-message-outputs/reference.log b/tests/modes/02-dummy-message-outputs/reference.log
similarity index 100%
rename from tests/modes/dummy-message-outputs/reference.log
rename to tests/modes/02-dummy-message-outputs/reference.log
diff --git a/tests/modes/dummy-message-outputs/suite.rc b/tests/modes/02-dummy-message-outputs/suite.rc
similarity index 100%
rename from tests/modes/dummy-message-outputs/suite.rc
rename to tests/modes/02-dummy-message-outputs/suite.rc
diff --git a/tests/offset/00-final-simple.t b/tests/offset/00-final-simple.t
index 551b76db5fe..62d93aa5a9f 100644
--- a/tests/offset/00-final-simple.t
+++ b/tests/offset/00-final-simple.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test specifying final cycle as an offset
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/offset/01-final-next.t b/tests/offset/01-final-next.t
index 364cdde2ca0..4df90ffc9eb 100644
--- a/tests/offset/01-final-next.t
+++ b/tests/offset/01-final-next.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test specifying final cycle as the next instance of T06
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/offset/02-final-chain.t b/tests/offset/02-final-chain.t
index 3889902a0d7..88ce727dd29 100644
--- a/tests/offset/02-final-chain.t
+++ b/tests/offset/02-final-chain.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test specifying final cycle as an offset chain
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/offset/03-final-next-chain.t b/tests/offset/03-final-next-chain.t
index 43fccc13ef8..db7ff757d12 100644
--- a/tests/offset/03-final-next-chain.t
+++ b/tests/offset/03-final-next-chain.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test specifying final cycle as an offset to the next occurrence of T06
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/offset/04-cycle-offset-chain.t b/tests/offset/04-cycle-offset-chain.t
index 674107ffc96..27a12a609fa 100644
--- a/tests/offset/04-cycle-offset-chain.t
+++ b/tests/offset/04-cycle-offset-chain.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test specifying an offset as a chain of periods
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/offset/05-long-final-chain.t b/tests/offset/05-long-final-chain.t
index d51ba313e9d..e5fb0d31db3 100644
--- a/tests/offset/05-long-final-chain.t
+++ b/tests/offset/05-long-final-chain.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test a 3 item offset
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/param_expand/03-env-tmpl.t b/tests/param_expand/03-env-tmpl.t
index 1f46730f850..da87405b931 100755
--- a/tests/param_expand/03-env-tmpl.t
+++ b/tests/param_expand/03-env-tmpl.t
@@ -18,11 +18,5 @@
# Test parameter environment templates
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/periodicals/00-daily.t b/tests/periodicals/00-daily.t
index 6a2a1ea187f..3bcced37e63 100644
--- a/tests/periodicals/00-daily.t
+++ b/tests/periodicals/00-daily.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test Daily cycling
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" Daily
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest "${TEST_NAME_BASE}" 'Daily'
+exit
diff --git a/tests/periodicals/03-monthly.t b/tests/periodicals/03-monthly.t
index ddae4d50306..1fde12508a6 100644
--- a/tests/periodicals/03-monthly.t
+++ b/tests/periodicals/03-monthly.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test Monthly cycling
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" Monthly
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest "${TEST_NAME_BASE}" 'Monthly'
+exit
diff --git a/tests/periodicals/06-yearly.t b/tests/periodicals/06-yearly.t
index 2b04b8b2f9f..377a9750f38 100644
--- a/tests/periodicals/06-yearly.t
+++ b/tests/periodicals/06-yearly.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test Yearly cycling
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" Yearly
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest "${TEST_NAME_BASE}" 'Yearly'
+exit
diff --git a/tests/periodicals/Daily/suite.rc b/tests/periodicals/Daily/suite.rc
index 8d5ce223218..5d86a486b0b 100644
--- a/tests/periodicals/Daily/suite.rc
+++ b/tests/periodicals/Daily/suite.rc
@@ -9,7 +9,7 @@
20140110T06/P3D = "a"
[runtime]
[[a]]
- script = "sleep 1"
+ script = true
[visualization]
initial cycle point = 20140107
diff --git a/tests/periodicals/Monthly-reorder/suite.rc b/tests/periodicals/Monthly-reorder/suite.rc
index f505e8679ba..290aa490314 100644
--- a/tests/periodicals/Monthly-reorder/suite.rc
+++ b/tests/periodicals/Monthly-reorder/suite.rc
@@ -13,4 +13,4 @@
R/01T/P1M = "dummy => monthly" # marker2
[runtime]
[[root]]
- script = "sleep 1"
+ script = true
diff --git a/tests/periodicals/Monthly/suite.rc b/tests/periodicals/Monthly/suite.rc
index 0bc6a174c79..4d11322022d 100644
--- a/tests/periodicals/Monthly/suite.rc
+++ b/tests/periodicals/Monthly/suite.rc
@@ -8,4 +8,4 @@
2010-01/P3M = "a"
[runtime]
[[a]]
- script = "sleep 1"
+ script = true
diff --git a/tests/periodicals/Yearly/suite.rc b/tests/periodicals/Yearly/suite.rc
index 004a1ec5cc5..743439f2564 100644
--- a/tests/periodicals/Yearly/suite.rc
+++ b/tests/periodicals/Yearly/suite.rc
@@ -8,4 +8,4 @@
2010/P3Y = "a"
[runtime]
[[a]]
- script = "sleep 1"
+ script = true
diff --git a/tests/pre-initial/00-simple.t b/tests/pre-initial/00-simple.t
index d85a0f4524c..25fe7b34dfe 100644
--- a/tests/pre-initial/00-simple.t
+++ b/tests/pre-initial/00-simple.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test simple pre initial cycling dependencies removal
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" simple
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/pre-initial/simple/reference.log b/tests/pre-initial/00-simple/reference.log
similarity index 100%
rename from tests/pre-initial/simple/reference.log
rename to tests/pre-initial/00-simple/reference.log
diff --git a/tests/pre-initial/simple/suite.rc b/tests/pre-initial/00-simple/suite.rc
similarity index 86%
rename from tests/pre-initial/simple/suite.rc
rename to tests/pre-initial/00-simple/suite.rc
index 65d3b98517a..605e921ee51 100644
--- a/tests/pre-initial/simple/suite.rc
+++ b/tests/pre-initial/00-simple/suite.rc
@@ -7,4 +7,4 @@
T00 = "a[-PT24H] => a"
[runtime]
[[a]]
- script = "sleep 1"
+ script = true
diff --git a/tests/pre-initial/01-basic.t b/tests/pre-initial/01-basic.t
index 54023a76a51..f45a74adcd4 100644
--- a/tests/pre-initial/01-basic.t
+++ b/tests/pre-initial/01-basic.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test simplification of basic conditionals in pre-initial cycling
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" basic-conditional
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/pre-initial/basic-conditional/reference.log b/tests/pre-initial/01-basic/reference.log
similarity index 100%
rename from tests/pre-initial/basic-conditional/reference.log
rename to tests/pre-initial/01-basic/reference.log
diff --git a/tests/pre-initial/basic-conditional/suite.rc b/tests/pre-initial/01-basic/suite.rc
similarity index 87%
rename from tests/pre-initial/basic-conditional/suite.rc
rename to tests/pre-initial/01-basic/suite.rc
index 406a1bd571d..fa5c01c37c5 100644
--- a/tests/pre-initial/basic-conditional/suite.rc
+++ b/tests/pre-initial/01-basic/suite.rc
@@ -7,4 +7,4 @@
T00 = "a[-P1D] & b => a"
[runtime]
[[a, b]]
- script = "sleep 1"
+ script = true
diff --git a/tests/pre-initial/02-advanced.t b/tests/pre-initial/02-advanced.t
index a17d84aa3ac..1c9da851735 100644
--- a/tests/pre-initial/02-advanced.t
+++ b/tests/pre-initial/02-advanced.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test nested conditional simplification for pre-initial cycling.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" advanced-conditional
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/pre-initial/advanced-conditional/reference.log b/tests/pre-initial/02-advanced/reference.log
similarity index 100%
rename from tests/pre-initial/advanced-conditional/reference.log
rename to tests/pre-initial/02-advanced/reference.log
diff --git a/tests/pre-initial/advanced-conditional/suite.rc b/tests/pre-initial/02-advanced/suite.rc
similarity index 88%
rename from tests/pre-initial/advanced-conditional/suite.rc
rename to tests/pre-initial/02-advanced/suite.rc
index e58a20d0e6f..24c02113d0f 100644
--- a/tests/pre-initial/advanced-conditional/suite.rc
+++ b/tests/pre-initial/02-advanced/suite.rc
@@ -7,4 +7,4 @@
PT6H = "(a[-PT6H] & b) & c[-PT6H] => a & c"
[runtime]
[[a, b, c]]
- script = "sleep 1"
+ script = true
diff --git a/tests/pre-initial/03-drop.t b/tests/pre-initial/03-drop.t
index 74ef23742c7..14843baf809 100644
--- a/tests/pre-initial/03-drop.t
+++ b/tests/pre-initial/03-drop.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test the case of dropping a conditional based on pre-initial cycling
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" drop-conditional
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/pre-initial/drop-conditional/reference.log b/tests/pre-initial/03-drop/reference.log
similarity index 100%
rename from tests/pre-initial/drop-conditional/reference.log
rename to tests/pre-initial/03-drop/reference.log
diff --git a/tests/pre-initial/drop-conditional/suite.rc b/tests/pre-initial/03-drop/suite.rc
similarity index 87%
rename from tests/pre-initial/drop-conditional/suite.rc
rename to tests/pre-initial/03-drop/suite.rc
index 17f2e837ded..71f9393bb34 100644
--- a/tests/pre-initial/drop-conditional/suite.rc
+++ b/tests/pre-initial/03-drop/suite.rc
@@ -7,4 +7,4 @@
PT6H = "a[-PT6H] & b[-PT6H] => a => b"
[runtime]
[[a, b]]
- script = "sleep 1"
+ script = true
diff --git a/tests/pre-initial/06-over-bracketed.t b/tests/pre-initial/06-over-bracketed.t
index 22cc0901c84..1c9da851735 100644
--- a/tests/pre-initial/06-over-bracketed.t
+++ b/tests/pre-initial/06-over-bracketed.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test nested conditional simplification for pre-initial cycling.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" bracketed-conditional
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/pre-initial/bracketed-conditional/reference.log b/tests/pre-initial/06-over-bracketed/reference.log
similarity index 100%
rename from tests/pre-initial/bracketed-conditional/reference.log
rename to tests/pre-initial/06-over-bracketed/reference.log
diff --git a/tests/pre-initial/bracketed-conditional/suite.rc b/tests/pre-initial/06-over-bracketed/suite.rc
similarity index 92%
rename from tests/pre-initial/bracketed-conditional/suite.rc
rename to tests/pre-initial/06-over-bracketed/suite.rc
index 90cd0053976..a6a85fbeedb 100644
--- a/tests/pre-initial/bracketed-conditional/suite.rc
+++ b/tests/pre-initial/06-over-bracketed/suite.rc
@@ -10,4 +10,4 @@
"""
[runtime]
[[root]]
- script = sleep 1
+ script = true
diff --git a/tests/pre-initial/07-simple-messaging.t b/tests/pre-initial/07-simple-messaging.t
index 6f2cb18255f..f383b4f9cc6 100644
--- a/tests/pre-initial/07-simple-messaging.t
+++ b/tests/pre-initial/07-simple-messaging.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test removal of pre-initial cycle message dependencies.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" simple-messaging
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/pre-initial/simple-messaging/reference.log b/tests/pre-initial/07-simple-messaging/reference.log
similarity index 100%
rename from tests/pre-initial/simple-messaging/reference.log
rename to tests/pre-initial/07-simple-messaging/reference.log
diff --git a/tests/pre-initial/simple-messaging/suite.rc b/tests/pre-initial/07-simple-messaging/suite.rc
similarity index 94%
rename from tests/pre-initial/simple-messaging/suite.rc
rename to tests/pre-initial/07-simple-messaging/suite.rc
index dfcc16e54c5..77754fdc4ad 100644
--- a/tests/pre-initial/simple-messaging/suite.rc
+++ b/tests/pre-initial/07-simple-messaging/suite.rc
@@ -8,8 +8,8 @@
[runtime]
[[foo]]
script = """
-sleep 1
+wait
cylc message "restart files ready"
-sleep 1"""
+"""
[[[outputs]]]
restart1 = "restart files ready"
diff --git a/tests/pre-initial/08-conditional-messaging.t b/tests/pre-initial/08-conditional-messaging.t
index b13f6b31878..e755e01558a 100644
--- a/tests/pre-initial/08-conditional-messaging.t
+++ b/tests/pre-initial/08-conditional-messaging.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test removal of pre-initial cycle messages from conditionals.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" conditional-messaging
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/pre-initial/conditional-messaging/reference.log b/tests/pre-initial/08-conditional-messaging/reference.log
similarity index 100%
rename from tests/pre-initial/conditional-messaging/reference.log
rename to tests/pre-initial/08-conditional-messaging/reference.log
diff --git a/tests/pre-initial/conditional-messaging/suite.rc b/tests/pre-initial/08-conditional-messaging/suite.rc
similarity index 96%
rename from tests/pre-initial/conditional-messaging/suite.rc
rename to tests/pre-initial/08-conditional-messaging/suite.rc
index 23c2adaa0bc..9e6a4229801 100644
--- a/tests/pre-initial/conditional-messaging/suite.rc
+++ b/tests/pre-initial/08-conditional-messaging/suite.rc
@@ -14,9 +14,9 @@
[runtime]
[[foo]]
script = """
-sleep 1
+wait
cylc message "restart files ready"
-sleep 1"""
+"""
[[[outputs]]]
restart1 = "restart files ready"
[[bar]]
diff --git a/tests/recurrence-min/00-basic.t b/tests/recurrence-min/00-basic.t
index 75e3b767d85..324a2621a7e 100755
--- a/tests/recurrence-min/00-basic.t
+++ b/tests/recurrence-min/00-basic.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test running with R/min(T00,...) syntax
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-val"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/recurrence-min/01-offset-initial.t b/tests/recurrence-min/01-offset-initial.t
index 785bf05d967..95add1eaa53 100755
--- a/tests/recurrence-min/01-offset-initial.t
+++ b/tests/recurrence-min/01-offset-initial.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test running with R/min(+PTNH,...) syntax
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-val"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/recurrence-min/02-offset-truncated.t b/tests/recurrence-min/02-offset-truncated.t
index 785bf05d967..95add1eaa53 100755
--- a/tests/recurrence-min/02-offset-truncated.t
+++ b/tests/recurrence-min/02-offset-truncated.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test running with R/min(+PTNH,...) syntax
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-val"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/recurrence-min/03-neg-offset-truncated.t b/tests/recurrence-min/03-neg-offset-truncated.t
index 785bf05d967..95add1eaa53 100755
--- a/tests/recurrence-min/03-neg-offset-truncated.t
+++ b/tests/recurrence-min/03-neg-offset-truncated.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test running with R/min(+PTNH,...) syntax
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-val"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/00-simple.t b/tests/reload/00-simple.t
index c732008117e..81b002df97f 100644
--- a/tests/reload/00-simple.t
+++ b/tests/reload/00-simple.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test reloading a simple suite
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" simple
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/simple/reference.log b/tests/reload/00-simple/reference.log
similarity index 100%
rename from tests/reload/simple/reference.log
rename to tests/reload/00-simple/reference.log
diff --git a/tests/reload/simple/suite.rc b/tests/reload/00-simple/suite.rc
similarity index 100%
rename from tests/reload/simple/suite.rc
rename to tests/reload/00-simple/suite.rc
diff --git a/tests/reload/01-startup.t b/tests/reload/01-startup.t
index 8369f9074c8..3d195c81209 100644
--- a/tests/reload/01-startup.t
+++ b/tests/reload/01-startup.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test reloading a suite containing a startup task.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" startup
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/startup/reference.log b/tests/reload/01-startup/reference.log
similarity index 100%
rename from tests/reload/startup/reference.log
rename to tests/reload/01-startup/reference.log
diff --git a/tests/reload/startup/suite.rc b/tests/reload/01-startup/suite.rc
similarity index 61%
rename from tests/reload/startup/suite.rc
rename to tests/reload/01-startup/suite.rc
index 5e2438b8eff..c4d4f187e31 100644
--- a/tests/reload/startup/suite.rc
+++ b/tests/reload/01-startup/suite.rc
@@ -9,6 +9,11 @@
T18 = "c"
[runtime]
[[a,c,start]]
- script = "sleep 1"
+ script = true
[[b]]
- script = "cylc reload $CYLC_SUITE_NAME -f; sleep 5"
+ script = """
+cylc reload "${CYLC_SUITE_NAME}" -f
+while ! grep -qF 'Reload completed' "${CYLC_SUITE_LOG_DIR}/log"; do
+ sleep 1
+done
+"""
diff --git a/tests/reload/02-content.t b/tests/reload/02-content.t
index e2471c83210..348a392bb01 100644
--- a/tests/reload/02-content.t
+++ b/tests/reload/02-content.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test reloads have actually updated environment variables
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" content
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/content/reference.log b/tests/reload/02-content/reference.log
similarity index 100%
rename from tests/reload/content/reference.log
rename to tests/reload/02-content/reference.log
diff --git a/tests/reload/content/suite.rc b/tests/reload/02-content/suite.rc
similarity index 100%
rename from tests/reload/content/suite.rc
rename to tests/reload/02-content/suite.rc
diff --git a/tests/reload/03-queues.t b/tests/reload/03-queues.t
index c696a35365f..dec94ef3908 100644
--- a/tests/reload/03-queues.t
+++ b/tests/reload/03-queues.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test changing queue size via reload doesn't break anything
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" queues
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/queues/reference.log b/tests/reload/03-queues/reference.log
similarity index 100%
rename from tests/reload/queues/reference.log
rename to tests/reload/03-queues/reference.log
diff --git a/tests/reload/queues/suite.rc b/tests/reload/03-queues/suite.rc
similarity index 89%
rename from tests/reload/queues/suite.rc
rename to tests/reload/03-queues/suite.rc
index c545e76a7cb..40fbf7ea826 100644
--- a/tests/reload/queues/suite.rc
+++ b/tests/reload/03-queues/suite.rc
@@ -20,12 +20,15 @@
# change the limit on number of members of q_fam:
perl -pi -e 's/(limit = )5( # marker)/\1 3 \2/' $CYLC_SUITE_DEF_PATH/suite.rc
# reload
-cylc reload -f $CYLC_SUITE_NAME
-sleep 5
+cylc reload -f "${CYLC_SUITE_NAME}"
+while ! grep -qF 'Reload completed' "${CYLC_SUITE_LOG_DIR}/log"
+do
+ sleep 1
+done
"""
[[monitor]]
script = """
- sleep 5
+ wait
while true; do
TASKS=$(cylc suite-state $CYLC_SUITE_NAME -S running | wc -l)
if ((TASKS > 4)); then
diff --git a/tests/reload/04-inheritance.t b/tests/reload/04-inheritance.t
index 17d52590acb..ffb3f6bac98 100644
--- a/tests/reload/04-inheritance.t
+++ b/tests/reload/04-inheritance.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test changing inheritance via a reload
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" inheritance
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/inheritance/reference.log b/tests/reload/04-inheritance/reference.log
similarity index 100%
rename from tests/reload/inheritance/reference.log
rename to tests/reload/04-inheritance/reference.log
diff --git a/tests/reload/inheritance/suite.rc b/tests/reload/04-inheritance/suite.rc
similarity index 100%
rename from tests/reload/inheritance/suite.rc
rename to tests/reload/04-inheritance/suite.rc
diff --git a/tests/reload/05-graphing-simple.t b/tests/reload/05-graphing-simple.t
index f4a1fbf1e48..1f592818d76 100644
--- a/tests/reload/05-graphing-simple.t
+++ b/tests/reload/05-graphing-simple.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test changing basic graphing via a reload
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" graphing-simple
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/graphing-simple/reference.log b/tests/reload/05-graphing-simple/reference.log
similarity index 100%
rename from tests/reload/graphing-simple/reference.log
rename to tests/reload/05-graphing-simple/reference.log
diff --git a/tests/reload/graphing-simple/suite.rc b/tests/reload/05-graphing-simple/suite.rc
similarity index 74%
rename from tests/reload/graphing-simple/suite.rc
rename to tests/reload/05-graphing-simple/suite.rc
index 72009485698..19b8e9d20f7 100644
--- a/tests/reload/graphing-simple/suite.rc
+++ b/tests/reload/05-graphing-simple/suite.rc
@@ -11,8 +11,10 @@
# change the order of foo and bar in the graphing section:
perl -pi -e 's/(R1 = reloader => )bar => foo( # marker)/\1foo => bar\2/' $CYLC_SUITE_DEF_PATH/suite.rc
# reload
-cylc reload -f $CYLC_SUITE_NAME
-sleep 5
+cylc reload -f "${CYLC_SUITE_NAME}"
+while ! grep -qF 'Reload completed' "${CYLC_SUITE_LOG_DIR}/log"; do
+ sleep 1
+done
"""
[[foo, bar]]
- script = sleep 5
+ script = true
diff --git a/tests/reload/06-graphing-fam.t b/tests/reload/06-graphing-fam.t
index 36a60dce796..be92b8c88ab 100644
--- a/tests/reload/06-graphing-fam.t
+++ b/tests/reload/06-graphing-fam.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test changing family triggering via a reload
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" graphing-fam
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/graphing-fam/reference.log b/tests/reload/06-graphing-fam/reference.log
similarity index 100%
rename from tests/reload/graphing-fam/reference.log
rename to tests/reload/06-graphing-fam/reference.log
diff --git a/tests/reload/graphing-fam/suite.rc b/tests/reload/06-graphing-fam/suite.rc
similarity index 82%
rename from tests/reload/graphing-fam/suite.rc
rename to tests/reload/06-graphing-fam/suite.rc
index 4fc6c29a199..e210ab271ab 100644
--- a/tests/reload/graphing-fam/suite.rc
+++ b/tests/reload/06-graphing-fam/suite.rc
@@ -15,11 +15,13 @@
perl -pi -e 's/(reloader => )BAR( # marker1)/\1FOO\2/' $CYLC_SUITE_DEF_PATH/suite.rc
perl -pi -e 's/( )BAR:finish-all => FOO( # marker2)/\1FOO:finish-all => BAR\2/' $CYLC_SUITE_DEF_PATH/suite.rc
# reload
-cylc reload -f $CYLC_SUITE_NAME
-sleep 5
+cylc reload -f "${CYLC_SUITE_NAME}"
+while ! grep -qF 'Reload completed' "${CYLC_SUITE_LOG_DIR}/log"; do
+ sleep 1
+done
"""
[[FOO, BAR]]
- script = sleep 5
+ script = true
[[a,b,c,d]]
inherit = FOO
[[e,f,g,h]]
diff --git a/tests/reload/07-final-cycle.t b/tests/reload/07-final-cycle.t
index 1aca165084c..9d26f649e10 100644
--- a/tests/reload/07-final-cycle.t
+++ b/tests/reload/07-final-cycle.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test changing final cycle point
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" final-cycle
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/final-cycle/reference.log b/tests/reload/07-final-cycle/reference.log
similarity index 100%
rename from tests/reload/final-cycle/reference.log
rename to tests/reload/07-final-cycle/reference.log
diff --git a/tests/reload/final-cycle/suite.rc b/tests/reload/07-final-cycle/suite.rc
similarity index 83%
rename from tests/reload/final-cycle/suite.rc
rename to tests/reload/07-final-cycle/suite.rc
index 66da19cf9c5..9c31f031a17 100644
--- a/tests/reload/final-cycle/suite.rc
+++ b/tests/reload/07-final-cycle/suite.rc
@@ -18,7 +18,9 @@
perl -pi -e 's/(final cycle point = )20100102T00( # marker)/\1 20100101T12\2/' $CYLC_SUITE_DEF_PATH/suite.rc
# reload
cylc reload -f $CYLC_SUITE_NAME
-sleep 5
+while ! grep -qF 'Reload completed' "${CYLC_SUITE_LOG_DIR}/log"; do
+ sleep 1
+done
"""
[[a]]
- script = sleep 1
+ script = true
diff --git a/tests/reload/08-cycle.t b/tests/reload/08-cycle.t
index 9de0f62aa8a..696bf5c9d37 100644
--- a/tests/reload/08-cycle.t
+++ b/tests/reload/08-cycle.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test changing cycle times (and dependencies)
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" cycle-times
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/cycle-times/reference.log b/tests/reload/08-cycle/reference.log
similarity index 100%
rename from tests/reload/cycle-times/reference.log
rename to tests/reload/08-cycle/reference.log
diff --git a/tests/reload/cycle-times/suite.rc b/tests/reload/08-cycle/suite.rc
similarity index 83%
rename from tests/reload/cycle-times/suite.rc
rename to tests/reload/08-cycle/suite.rc
index cb629f062a5..2cc9092b4ac 100644
--- a/tests/reload/cycle-times/suite.rc
+++ b/tests/reload/08-cycle/suite.rc
@@ -18,7 +18,9 @@
sed -i 's/T00,T12 = a\[-PT12H\]/T00,T06,T12,T18 = a[-PT6H]/' "${CYLC_SUITE_DEF_PATH}/suite.rc"
# reload
cylc reload -f "${CYLC_SUITE_NAME}"
-sleep 5
+while ! grep -qF 'Reload completed' "${CYLC_SUITE_LOG_DIR}/log"; do
+ sleep 1
+done
"""
[[a]]
- script = sleep 1
+ script = true
diff --git a/tests/reload/11-retrying.t b/tests/reload/11-retrying.t
index e0e31ee2fc0..d88b3167db2 100644
--- a/tests/reload/11-retrying.t
+++ b/tests/reload/11-retrying.t
@@ -18,11 +18,5 @@
# Test that a reloaded retrying does does retry; ref github #945
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" 'retrying'
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/reload/retrying/reference.log b/tests/reload/11-retrying/reference.log
similarity index 100%
rename from tests/reload/retrying/reference.log
rename to tests/reload/11-retrying/reference.log
diff --git a/tests/reload/retrying/suite.rc b/tests/reload/11-retrying/suite.rc
similarity index 100%
rename from tests/reload/retrying/suite.rc
rename to tests/reload/11-retrying/suite.rc
diff --git a/tests/reload/12-remove-task.t b/tests/reload/12-remove-task.t
index 4c6dda86261..0d1e9dd23df 100644
--- a/tests/reload/12-remove-task.t
+++ b/tests/reload/12-remove-task.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test that removing a task from the graph works OK.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" remove-task
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/remove-task/reference.log b/tests/reload/12-remove-task/reference.log
similarity index 100%
rename from tests/reload/remove-task/reference.log
rename to tests/reload/12-remove-task/reference.log
diff --git a/tests/reload/remove-task/suite.rc b/tests/reload/12-remove-task/suite.rc
similarity index 71%
rename from tests/reload/remove-task/suite.rc
rename to tests/reload/12-remove-task/suite.rc
index 954d767364f..544997d23a6 100644
--- a/tests/reload/remove-task/suite.rc
+++ b/tests/reload/12-remove-task/suite.rc
@@ -10,7 +10,10 @@
script = """
sed -i "s/remove_me =>//g" $CYLC_SUITE_DEF_PATH/suite.rc
cylc reload $CYLC_SUITE_NAME
- sleep 10
+ while ! grep -q 'Reload completed' "${CYLC_SUITE_LOG_DIR}/log"
+ do
+ sleep 1 # make sure reload completes
+ done
"""
[[remove_me]]
script = false
diff --git a/tests/reload/13-add-task.t b/tests/reload/13-add-task.t
index 0e5a1fb6362..31d63c7db9f 100644
--- a/tests/reload/13-add-task.t
+++ b/tests/reload/13-add-task.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test that adding a task to the graph works OK.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" add-task
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/add-task/reference.log b/tests/reload/13-add-task/reference.log
similarity index 100%
rename from tests/reload/add-task/reference.log
rename to tests/reload/13-add-task/reference.log
diff --git a/tests/reload/add-task/suite.rc b/tests/reload/13-add-task/suite.rc
similarity index 100%
rename from tests/reload/add-task/suite.rc
rename to tests/reload/13-add-task/suite.rc
diff --git a/tests/reload/16-remove-add-alter-task.t b/tests/reload/16-remove-add-alter-task.t
index cbe2ff1ee0d..cc452ae62ba 100644
--- a/tests/reload/16-remove-add-alter-task.t
+++ b/tests/reload/16-remove-add-alter-task.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test that removing, adding, then altering a task in the graph works OK.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" remove-add-alter-task
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/reload/remove-add-alter-task/reference.log b/tests/reload/16-remove-add-alter-task/reference.log
similarity index 100%
rename from tests/reload/remove-add-alter-task/reference.log
rename to tests/reload/16-remove-add-alter-task/reference.log
diff --git a/tests/reload/16-remove-add-alter-task/suite.rc b/tests/reload/16-remove-add-alter-task/suite.rc
new file mode 100644
index 00000000000..5eddeae1281
--- /dev/null
+++ b/tests/reload/16-remove-add-alter-task/suite.rc
@@ -0,0 +1,39 @@
+[meta]
+ title = "Test reloading of a task removed then added by a reload."
+# Don't run this suite in-place: it modifies itself.
+
+[cylc]
+ UTC mode = True
+
+[scheduling]
+ [[graph]]
+ R1 = reloader => remove_add_alter_me
+
+[runtime]
+ [[remove_add_alter_me]]
+ script = false
+ [[reloader]]
+ script = """
+do_reload() {
+ cylc reload "${CYLC_SUITE_NAME}"
+ while test "$(grep -cF 'Reload completed' "${CYLC_SUITE_LOG_DIR}/log")" -ne "$1"
+ do
+ sleep 1
+ done
+}
+sed -i "s/\(R1 = reloader\).*/\1/" "${CYLC_SUITE_DEF_PATH}/suite.rc"
+do_reload 1
+sed -i "s/\(R1 = reloader\)/\1 => remove_add_alter_me/" \
+ "${CYLC_SUITE_DEF_PATH}/suite.rc"
+do_reload 2
+cylc insert "${CYLC_SUITE_NAME}" 'remove_add_alter_me.1'
+while ! grep -qF 'remove_add_alter_me' < <(cylc dump -t "${CYLC_SUITE_NAME}")
+do
+ sleep 1
+done
+cat >>"${CYLC_SUITE_DEF_PATH}/suite.rc" <<'__RUNTIME__'
+[[remove_add_alter_me]]
+ script = true
+__RUNTIME__
+do_reload 3
+"""
diff --git a/tests/reload/18-broadcast-insert.t b/tests/reload/18-broadcast-insert.t
index 91b65800119..743827214fb 100755
--- a/tests/reload/18-broadcast-insert.t
+++ b/tests/reload/18-broadcast-insert.t
@@ -17,13 +17,6 @@
#-------------------------------------------------------------------------------
# Test reload with new task + broadcast + insert new task.
. "$(dirname "$0")/test_header"
-
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/reload/remove-add-alter-task/suite.rc b/tests/reload/remove-add-alter-task/suite.rc
deleted file mode 100644
index ce5630681f0..00000000000
--- a/tests/reload/remove-add-alter-task/suite.rc
+++ /dev/null
@@ -1,31 +0,0 @@
-[meta]
- title = "Test reloading of a task removed then added by a reload."
-# Don't run this suite in-place: it modifies itself.
-
-[cylc]
- UTC mode = True
-
-[scheduling]
- [[graph]]
- R1 = reloader => remove_add_alter_me
-
-[runtime]
- [[remove_add_alter_me]]
- script = false
- [[reloader]]
- script = """
-sed -i "12s/\(graph = reloader\).*/\1/" $CYLC_SUITE_DEF_PATH/suite.rc
-cylc reload $CYLC_SUITE_NAME
-sleep 5
-sed -i "12s/\(graph = reloader\)/\1 => remove_add_alter_me/" $CYLC_SUITE_DEF_PATH/suite.rc
-cylc reload $CYLC_SUITE_NAME
-sleep 5
-cylc insert $CYLC_SUITE_NAME remove_add_alter_me.1
-sleep 5
-cat >>$CYLC_SUITE_DEF_PATH/suite.rc <<'__RUNTIME__'
-[[remove_add_alter_me]]
- script = true
-__RUNTIME__
-cylc reload $CYLC_SUITE_NAME
-sleep 5
- """
diff --git a/tests/restart/34-auto-restart-basic.t b/tests/restart/34-auto-restart-basic.t
index 6353f6bd348..ec6747477ce 100644
--- a/tests/restart/34-auto-restart-basic.t
+++ b/tests/restart/34-auto-restart-basic.t
@@ -86,8 +86,7 @@ grep_ok "cylc-restart ${SUITE_NAME} --host=localhost" \
"${TEST_NAME}-contact.stdout"
# stop suite
-cylc stop "${SUITE_NAME}" --kill 2>/dev/null
-sleep 2
+cylc stop "${SUITE_NAME}" --kill --max-polls=10 --interval=2 2>'/dev/null'
purge_suite "${SUITE_NAME}"
exit
diff --git a/tests/restart/failed/suite.rc b/tests/restart/failed/suite.rc
index 743a32271d6..23136cc7727 100644
--- a/tests/restart/failed/suite.rc
+++ b/tests/restart/failed/suite.rc
@@ -3,8 +3,9 @@
[cylc]
UTC mode = True
[[events]]
- abort on timeout = True
- timeout = PT3M
+ abort on stalled = True
+ abort on inactivity = True
+ inactivity = PT3M
[scheduling]
initial cycle point = 20130923T00
final cycle point = 20130923T00
@@ -17,7 +18,6 @@
[runtime]
[[failed_task]]
script = """
- sleep 10
exit 1
"""
[[[meta]]]
diff --git a/tests/restart/lib/suite-runtime-restart.rc b/tests/restart/lib/suite-runtime-restart.rc
index e0c70657ae2..3b5c9a262de 100644
--- a/tests/restart/lib/suite-runtime-restart.rc
+++ b/tests/restart/lib/suite-runtime-restart.rc
@@ -1,6 +1,6 @@
[[OUTPUT]]
script = """
- sleep 5
+ wait
# Handle db locks.
for i in {0..10}; do
ctb-select-task-states \
@@ -11,19 +11,14 @@
"""
[[shutdown]]
inherit = OUTPUT
- post-script = """
- cylc shutdown $CYLC_SUITE_NAME
- sleep 5
- """
+ post-script = cylc shutdown "${CYLC_SUITE_NAME}"
[[[meta]]]
description = "Force a shutdown of the suite"
[[[environment]]]
OUTPUT_SUFFIX=pre-restart
[[output_states]]
inherit = OUTPUT
- pre-script = """
- sleep 5
- """
+ pre-script = wait
[[[meta]]]
description = "Wait for the restart to complete, then output states"
[[[environment]]]
diff --git a/tests/restart/reload/suite.rc b/tests/restart/reload/suite.rc
index b9fc8bccd55..f400ec02dab 100644
--- a/tests/restart/reload/suite.rc
+++ b/tests/restart/reload/suite.rc
@@ -22,7 +22,9 @@ which should run to completion on restarting."""
[[foo]]
script = """
if [[ "$CYLC_TASK_CYCLE_POINT" == "$CYLC_SUITE_INITIAL_CYCLE_POINT" ]]; then
- cylc reload $CYLC_SUITE_NAME
- cylc stop $CYLC_SUITE_NAME
- sleep 5
+ cylc reload "${CYLC_SUITE_NAME}"
+ while ! grep -qF 'Reload completed' "${CYLC_SUITE_LOG_DIR}/log"; do
+ sleep 1
+ done
+ cylc stop "${CYLC_SUITE_NAME}"
fi"""
diff --git a/tests/restart/succeeded/suite.rc b/tests/restart/succeeded/suite.rc
index b363d998e98..e443dff503c 100644
--- a/tests/restart/succeeded/suite.rc
+++ b/tests/restart/succeeded/suite.rc
@@ -16,9 +16,7 @@
"""
[runtime]
[[succeeded_task]]
- script = """
- sleep 1
- """
+ script = true
[[[meta]]]
description = "Succeeded task (runs before restart)"
{% include 'suite-runtime-restart.rc' %}
diff --git a/tests/runahead/no_final/suite.rc b/tests/runahead/no_final/suite.rc
index cf34a54a118..7032d9cc323 100644
--- a/tests/runahead/no_final/suite.rc
+++ b/tests/runahead/no_final/suite.rc
@@ -1,8 +1,9 @@
[cylc]
cycle point time zone = Z
[[events]]
- timeout = PT0.3M
- abort on timeout = True
+ abort on stalled = True
+ abort on inactivity = True
+ inactivity = PT3M
[scheduling]
runahead limit = PT18H
initial cycle point = 20100101T00
@@ -15,6 +16,4 @@
[[bar]]
script = true
[[shutdown]]
- script = """
-sleep 5
-cylc shutdown $CYLC_SUITE_NAME"""
+ script = cylc shutdown "${CYLC_SUITE_NAME}"
diff --git a/tests/shutdown/00-cycle.t b/tests/shutdown/00-cycle.t
index 1b926f99f94..627cd85c6e1 100755
--- a/tests/shutdown/00-cycle.t
+++ b/tests/shutdown/00-cycle.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test shutdown after a particular cycle.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/shutdown/01-task.t b/tests/shutdown/01-task.t
index 4985a37f691..920d9c09fd5 100755
--- a/tests/shutdown/01-task.t
+++ b/tests/shutdown/01-task.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test shutdown after a specific task.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/shutdown/04-kill.t b/tests/shutdown/04-kill.t
index 57388901f30..675f1460f17 100755
--- a/tests/shutdown/04-kill.t
+++ b/tests/shutdown/04-kill.t
@@ -17,14 +17,6 @@
#-------------------------------------------------------------------------------
# Test kill OK after shutdown. Suite will timeout if kill not OK.
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run "${SUITE_NAME}" --debug --no-detach --reference-test
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/shutdown/08-now1/suite.rc b/tests/shutdown/08-now1/suite.rc
index c72a003c801..0e23f023756 100644
--- a/tests/shutdown/08-now1/suite.rc
+++ b/tests/shutdown/08-now1/suite.rc
@@ -10,7 +10,7 @@
[runtime]
[[t1]]
- script = sleep 1 && cylc stop --now "${CYLC_SUITE_NAME}"
+ script = wait; cylc stop --now "${CYLC_SUITE_NAME}"
[[[events]]]
started handler = sleep 10 && echo 'Hello %(id)s %(event)s'
succeeded handler = echo 'Well done %(id)s %(event)s'
diff --git a/tests/spawn-max/00-basic.t b/tests/spawn-max/00-basic.t
index 5b0711d1f78..1c5c538a3a2 100644
--- a/tests/spawn-max/00-basic.t
+++ b/tests/spawn-max/00-basic.t
@@ -14,20 +14,10 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-
#-------------------------------------------------------------------------------
# Test that spawning out to max active cycle points to allows tasks to run out
# of order - see GitHub #1538, #1904.
-
. "$(dirname "$0")/test_header"
set_test_number 2
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/special/00-sequential.t b/tests/special/00-sequential.t
index 48e4a4e901b..6b5c1553dd8 100644
--- a/tests/special/00-sequential.t
+++ b/tests/special/00-sequential.t
@@ -17,17 +17,6 @@
#-------------------------------------------------------------------------------
# Test sequential tasks
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" sequential
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-
-# Suite uses sqlite3 CLI.
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/special/sequential/reference.log b/tests/special/00-sequential/reference.log
similarity index 100%
rename from tests/special/sequential/reference.log
rename to tests/special/00-sequential/reference.log
diff --git a/tests/special/sequential/suite.rc b/tests/special/00-sequential/suite.rc
similarity index 100%
rename from tests/special/sequential/suite.rc
rename to tests/special/00-sequential/suite.rc
diff --git a/tests/special/02-exclude.t b/tests/special/02-exclude.t
index 5bb130c9a5f..eaf37c632e1 100644
--- a/tests/special/02-exclude.t
+++ b/tests/special/02-exclude.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test exclude at start-up is working
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" exclude
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/special/exclude/reference.log b/tests/special/02-exclude/reference.log
similarity index 100%
rename from tests/special/exclude/reference.log
rename to tests/special/02-exclude/reference.log
diff --git a/tests/special/exclude/suite.rc b/tests/special/02-exclude/suite.rc
similarity index 100%
rename from tests/special/exclude/suite.rc
rename to tests/special/02-exclude/suite.rc
diff --git a/tests/special/03-include.t b/tests/special/03-include.t
index 823f673cfc9..3bd5ad1212b 100644
--- a/tests/special/03-include.t
+++ b/tests/special/03-include.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test include at start-up is working
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" include
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/special/include/reference.log b/tests/special/03-include/reference.log
similarity index 100%
rename from tests/special/include/reference.log
rename to tests/special/03-include/reference.log
diff --git a/tests/special/include/suite.rc b/tests/special/03-include/suite.rc
similarity index 100%
rename from tests/special/include/suite.rc
rename to tests/special/03-include/suite.rc
diff --git a/tests/task-name/00-basic.t b/tests/task-name/00-basic.t
index e0799300604..378ca48aa6a 100644
--- a/tests/task-name/00-basic.t
+++ b/tests/task-name/00-basic.t
@@ -16,14 +16,7 @@
# along with this program. If not, see .
#-------------------------------------------------------------------------------
# Basic task names tests.
-
. "$(dirname "$0")/test_header"
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
exit
diff --git a/tests/task-proc-loop/00-count/suite.rc b/tests/task-proc-loop/00-count/suite.rc
index 9979e04becf..16c2c87a98e 100644
--- a/tests/task-proc-loop/00-count/suite.rc
+++ b/tests/task-proc-loop/00-count/suite.rc
@@ -1,7 +1,8 @@
[cylc]
[[events]]
- timeout = PT1M
- abort on timeout = True
+ abort on stalled = True
+ abort on inactivity = True
+ inactivity = PT3M
[scheduling]
[[queues]]
[[[default]]]
@@ -10,4 +11,4 @@
R1 = m1 & m2
[runtime]
[[m1, m2]]
- script = sleep 10
+ script = true
diff --git a/tests/triggering/00-recovery.t b/tests/triggering/00-recovery.t
index a1c74c36186..013392341a6 100644
--- a/tests/triggering/00-recovery.t
+++ b/tests/triggering/00-recovery.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test automated failure recovery example
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" recovery
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/recovery/reference.log b/tests/triggering/00-recovery/reference.log
similarity index 100%
rename from tests/triggering/recovery/reference.log
rename to tests/triggering/00-recovery/reference.log
diff --git a/tests/triggering/recovery/suite.rc b/tests/triggering/00-recovery/suite.rc
similarity index 98%
rename from tests/triggering/recovery/suite.rc
rename to tests/triggering/00-recovery/suite.rc
index b4144f9ef96..97f9feef9c8 100644
--- a/tests/triggering/recovery/suite.rc
+++ b/tests/triggering/00-recovery/suite.rc
@@ -29,7 +29,6 @@
[[model]]
script = """
echo Hello from $CYLC_TASK_ID
- #sleep 10
if [[ $(cylc util cycletime --print-hour) == 12 ]]; then
echo "FAILING NOW!"
false
diff --git a/tests/triggering/01-or-conditional.t b/tests/triggering/01-or-conditional.t
index 16ffb81336c..2812c8a044b 100644
--- a/tests/triggering/01-or-conditional.t
+++ b/tests/triggering/01-or-conditional.t
@@ -1,7 +1,7 @@
#!/bin/bash
# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
-#
+#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test "or" conditionals
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" or-condition
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/01-or-conditional/reference.log b/tests/triggering/01-or-conditional/reference.log
new file mode 100644
index 00000000000..5f71d2c98f0
--- /dev/null
+++ b/tests/triggering/01-or-conditional/reference.log
@@ -0,0 +1,7 @@
+Initial point: 1
+Final point: 1
+[a.1] -triggered off []
+[b.1] -triggered off []
+[c.1] -triggered off ['a.1']
+[d.1] -triggered off ['a.1']
+[e.1] -triggered off ['d.1']
diff --git a/tests/triggering/01-or-conditional/suite.rc b/tests/triggering/01-or-conditional/suite.rc
new file mode 100644
index 00000000000..73715c79122
--- /dev/null
+++ b/tests/triggering/01-or-conditional/suite.rc
@@ -0,0 +1,16 @@
+[cylc]
+ [[reference test]]
+ expected task failures = b.1, c.1
+[scheduling]
+ [[graph]]
+ R1 = """
+ a | b => c & d
+ c | d => e
+ b:failed => !b
+ c:failed => !c
+ """
+[runtime]
+ [[b,c]]
+ script = false
+ [[a,d, e]]
+ script = true
diff --git a/tests/triggering/02-fam-start-all.t b/tests/triggering/02-fam-start-all.t
index d75fbc59e0e..cf21d6d2a29 100644
--- a/tests/triggering/02-fam-start-all.t
+++ b/tests/triggering/02-fam-start-all.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test correct expansion of FAM:start-all
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" fam-start-all
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/fam-start-all/reference.log b/tests/triggering/02-fam-start-all/reference.log
similarity index 100%
rename from tests/triggering/fam-start-all/reference.log
rename to tests/triggering/02-fam-start-all/reference.log
diff --git a/tests/triggering/fam-start-all/suite.rc b/tests/triggering/02-fam-start-all/suite.rc
similarity index 85%
rename from tests/triggering/fam-start-all/suite.rc
rename to tests/triggering/02-fam-start-all/suite.rc
index cfa8a5f2191..25754a0f319 100644
--- a/tests/triggering/fam-start-all/suite.rc
+++ b/tests/triggering/02-fam-start-all/suite.rc
@@ -3,7 +3,7 @@
R1 = """FAM:start-all => foo"""
[runtime]
[[FAM]]
- script = "sleep 5"
+ script = wait
[[a,b,c]]
inherit = FAM
[[foo]]
diff --git a/tests/triggering/03-fam-succeed-all.t b/tests/triggering/03-fam-succeed-all.t
index 09b6e3a016a..7eb13a9423f 100644
--- a/tests/triggering/03-fam-succeed-all.t
+++ b/tests/triggering/03-fam-succeed-all.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test correct expansion of FAM:succeed-all
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" fam-succeed-all
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/fam-succeed-all/reference.log b/tests/triggering/03-fam-succeed-all/reference.log
similarity index 100%
rename from tests/triggering/fam-succeed-all/reference.log
rename to tests/triggering/03-fam-succeed-all/reference.log
diff --git a/tests/triggering/fam-succeed-all/suite.rc b/tests/triggering/03-fam-succeed-all/suite.rc
similarity index 100%
rename from tests/triggering/fam-succeed-all/suite.rc
rename to tests/triggering/03-fam-succeed-all/suite.rc
diff --git a/tests/triggering/04-fam-fail-all.t b/tests/triggering/04-fam-fail-all.t
index e7a06551788..d01acd41aea 100644
--- a/tests/triggering/04-fam-fail-all.t
+++ b/tests/triggering/04-fam-fail-all.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test correct expansion of FAM:fail-all
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" fam-fail-all
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/fam-fail-all/reference.log b/tests/triggering/04-fam-fail-all/reference.log
similarity index 100%
rename from tests/triggering/fam-fail-all/reference.log
rename to tests/triggering/04-fam-fail-all/reference.log
diff --git a/tests/triggering/fam-fail-all/suite.rc b/tests/triggering/04-fam-fail-all/suite.rc
similarity index 100%
rename from tests/triggering/fam-fail-all/suite.rc
rename to tests/triggering/04-fam-fail-all/suite.rc
diff --git a/tests/triggering/05-fam-finish-all.t b/tests/triggering/05-fam-finish-all.t
index 2fb3db476ee..29f802893e9 100644
--- a/tests/triggering/05-fam-finish-all.t
+++ b/tests/triggering/05-fam-finish-all.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test correct expansion of FAM:finish-all
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" fam-finish-all
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/fam-finish-all/reference.log b/tests/triggering/05-fam-finish-all/reference.log
similarity index 100%
rename from tests/triggering/fam-finish-all/reference.log
rename to tests/triggering/05-fam-finish-all/reference.log
diff --git a/tests/triggering/fam-finish-all/suite.rc b/tests/triggering/05-fam-finish-all/suite.rc
similarity index 100%
rename from tests/triggering/fam-finish-all/suite.rc
rename to tests/triggering/05-fam-finish-all/suite.rc
diff --git a/tests/triggering/06-fam-succeed-any.t b/tests/triggering/06-fam-succeed-any.t
index 0e566e74c8a..7b6c4a96ce3 100644
--- a/tests/triggering/06-fam-succeed-any.t
+++ b/tests/triggering/06-fam-succeed-any.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test correct expansion of FAM:succeed-any
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" fam-succeed-any
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/fam-succeed-any/reference.log b/tests/triggering/06-fam-succeed-any/reference.log
similarity index 100%
rename from tests/triggering/fam-succeed-any/reference.log
rename to tests/triggering/06-fam-succeed-any/reference.log
diff --git a/tests/triggering/fam-succeed-any/suite.rc b/tests/triggering/06-fam-succeed-any/suite.rc
similarity index 100%
rename from tests/triggering/fam-succeed-any/suite.rc
rename to tests/triggering/06-fam-succeed-any/suite.rc
diff --git a/tests/triggering/07-fam-fail-any.t b/tests/triggering/07-fam-fail-any.t
index 1e776ad1b2b..694e35e8b03 100644
--- a/tests/triggering/07-fam-fail-any.t
+++ b/tests/triggering/07-fam-fail-any.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test correct expansion of FAM:fail-any
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" fam-fail-any
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/fam-fail-any/reference.log b/tests/triggering/07-fam-fail-any/reference.log
similarity index 100%
rename from tests/triggering/fam-fail-any/reference.log
rename to tests/triggering/07-fam-fail-any/reference.log
diff --git a/tests/triggering/fam-fail-any/suite.rc b/tests/triggering/07-fam-fail-any/suite.rc
similarity index 100%
rename from tests/triggering/fam-fail-any/suite.rc
rename to tests/triggering/07-fam-fail-any/suite.rc
diff --git a/tests/triggering/08-fam-finish-any.t b/tests/triggering/08-fam-finish-any.t
index a3f773388d2..caf50a10814 100644
--- a/tests/triggering/08-fam-finish-any.t
+++ b/tests/triggering/08-fam-finish-any.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test correct expansion of FAM:finish-any
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" fam-finish-any
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/fam-finish-any/reference.log b/tests/triggering/08-fam-finish-any/reference.log
similarity index 100%
rename from tests/triggering/fam-finish-any/reference.log
rename to tests/triggering/08-fam-finish-any/reference.log
diff --git a/tests/triggering/fam-finish-any/suite.rc b/tests/triggering/08-fam-finish-any/suite.rc
similarity index 68%
rename from tests/triggering/fam-finish-any/suite.rc
rename to tests/triggering/08-fam-finish-any/suite.rc
index d5a4fad407c..6d8790a829f 100644
--- a/tests/triggering/fam-finish-any/suite.rc
+++ b/tests/triggering/08-fam-finish-any/suite.rc
@@ -3,11 +3,11 @@
R1 = """FAM:finish-any => foo"""
[runtime]
[[FAM]]
- script = "sleep 10"
+ script = sleep 10
[[a,c]]
inherit = FAM
[[b]]
inherit = FAM
- script = "true"
+ script = true
[[foo]]
- script = "true"
+ script = true
diff --git a/tests/triggering/09-fail.t b/tests/triggering/09-fail.t
index 0c78eff865d..e26becc086a 100644
--- a/tests/triggering/09-fail.t
+++ b/tests/triggering/09-fail.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test fail triggering
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" fail
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/fail/reference.log b/tests/triggering/09-fail/reference.log
similarity index 100%
rename from tests/triggering/fail/reference.log
rename to tests/triggering/09-fail/reference.log
diff --git a/tests/triggering/fail/suite.rc b/tests/triggering/09-fail/suite.rc
similarity index 100%
rename from tests/triggering/fail/suite.rc
rename to tests/triggering/09-fail/suite.rc
diff --git a/tests/triggering/10-finish.t b/tests/triggering/10-finish.t
index cb8d397b122..2581f95845a 100644
--- a/tests/triggering/10-finish.t
+++ b/tests/triggering/10-finish.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test finish triggering
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" finish
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/finish/reference.log b/tests/triggering/10-finish/reference.log
similarity index 100%
rename from tests/triggering/finish/reference.log
rename to tests/triggering/10-finish/reference.log
diff --git a/tests/triggering/finish/suite.rc b/tests/triggering/10-finish/suite.rc
similarity index 100%
rename from tests/triggering/finish/suite.rc
rename to tests/triggering/10-finish/suite.rc
diff --git a/tests/triggering/11-start.t b/tests/triggering/11-start.t
index 61df73ec319..f188bac67b7 100644
--- a/tests/triggering/11-start.t
+++ b/tests/triggering/11-start.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test start triggering
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" start
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/start/reference.log b/tests/triggering/11-start/reference.log
similarity index 100%
rename from tests/triggering/start/reference.log
rename to tests/triggering/11-start/reference.log
diff --git a/tests/triggering/start/suite.rc b/tests/triggering/11-start/suite.rc
similarity index 100%
rename from tests/triggering/start/suite.rc
rename to tests/triggering/11-start/suite.rc
diff --git a/tests/triggering/12-succeed.t b/tests/triggering/12-succeed.t
index c1cc4d207a3..d7c54fc1f5f 100644
--- a/tests/triggering/12-succeed.t
+++ b/tests/triggering/12-succeed.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test succeed triggering
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" succeed
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/succeed/reference.log b/tests/triggering/12-succeed/reference.log
similarity index 100%
rename from tests/triggering/succeed/reference.log
rename to tests/triggering/12-succeed/reference.log
diff --git a/tests/triggering/succeed/suite.rc b/tests/triggering/12-succeed/suite.rc
similarity index 100%
rename from tests/triggering/succeed/suite.rc
rename to tests/triggering/12-succeed/suite.rc
diff --git a/tests/triggering/13-submit.t b/tests/triggering/13-submit.t
index a46d7a06003..8e45eaed1b3 100644
--- a/tests/triggering/13-submit.t
+++ b/tests/triggering/13-submit.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test submit triggering
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" submit
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/submit/reference.log b/tests/triggering/13-submit/reference.log
similarity index 100%
rename from tests/triggering/submit/reference.log
rename to tests/triggering/13-submit/reference.log
diff --git a/tests/triggering/submit/suite.rc b/tests/triggering/13-submit/suite.rc
similarity index 100%
rename from tests/triggering/submit/suite.rc
rename to tests/triggering/13-submit/suite.rc
diff --git a/tests/triggering/14-submit-fail.t b/tests/triggering/14-submit-fail.t
index d3ff5e7251e..e347f4f80f0 100644
--- a/tests/triggering/14-submit-fail.t
+++ b/tests/triggering/14-submit-fail.t
@@ -17,15 +17,6 @@
#-------------------------------------------------------------------------------
# Test submit-fail triggering
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" submit-fail
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/submit-fail/reference.log b/tests/triggering/14-submit-fail/reference.log
similarity index 100%
rename from tests/triggering/submit-fail/reference.log
rename to tests/triggering/14-submit-fail/reference.log
diff --git a/tests/triggering/submit-fail/suite.rc b/tests/triggering/14-submit-fail/suite.rc
similarity index 100%
rename from tests/triggering/submit-fail/suite.rc
rename to tests/triggering/14-submit-fail/suite.rc
diff --git a/tests/triggering/15-suicide.t b/tests/triggering/15-suicide.t
index 3514cb146bd..ef8de7cb439 100644
--- a/tests/triggering/15-suicide.t
+++ b/tests/triggering/15-suicide.t
@@ -19,15 +19,6 @@
# (this is currently just a copy of the tutorial suicide example, but I
# anticipate making it more fiendish at some point).
. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" suicide
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/triggering/suicide/reference.log b/tests/triggering/15-suicide/reference.log
similarity index 100%
rename from tests/triggering/suicide/reference.log
rename to tests/triggering/15-suicide/reference.log
diff --git a/tests/triggering/suicide/suite.rc b/tests/triggering/15-suicide/suite.rc
similarity index 83%
rename from tests/triggering/suicide/suite.rc
rename to tests/triggering/15-suicide/suite.rc
index ae7cc82459a..b3ca75bb42d 100644
--- a/tests/triggering/suicide/suite.rc
+++ b/tests/triggering/15-suicide/suite.rc
@@ -12,9 +12,9 @@
really_goodbye => !goodbye"""
[runtime]
[[hello]]
- script = "sleep 1; echo Hello World!"
+ script = echo Hello World!
[[goodbye]]
script = """
-sleep 1
echo Goodbye ... oops!
-/bin/false # FAIL"""
+false
+"""
diff --git a/tests/triggering/17-suicide-multi/suite.rc b/tests/triggering/17-suicide-multi/suite.rc
index d11e66e3113..d50d7651f3b 100644
--- a/tests/triggering/17-suicide-multi/suite.rc
+++ b/tests/triggering/17-suicide-multi/suite.rc
@@ -1,6 +1,6 @@
[cylc]
[[events]]
- timeout = PT1M
+ abort on stalled = True
[scheduling]
cycling mode = integer
diff --git a/tests/triggering/19-and-suicide/suite.rc b/tests/triggering/19-and-suicide/suite.rc
index b7f2283e1a9..459f4a832bc 100644
--- a/tests/triggering/19-and-suicide/suite.rc
+++ b/tests/triggering/19-and-suicide/suite.rc
@@ -14,7 +14,11 @@
[[t0]]
# https://github.com/cylc/cylc-flow/issues/2655
# "t2.1" should not suicide on "t1.1:failed"
- script = sleep 10
+ script = """
+while ! grep -q '\[t1\.1\].*failed' "${CYLC_SUITE_LOG_DIR}/log"; do
+ sleep 1
+done
+"""
[[t1]]
script = false
[[t2]]
diff --git a/tests/triggering/or-condition/reference.log b/tests/triggering/or-condition/reference.log
deleted file mode 100644
index 85de0929178..00000000000
--- a/tests/triggering/or-condition/reference.log
+++ /dev/null
@@ -1,7 +0,0 @@
-2013/11/05 15:08:51 INFO - Initial point: 1
-2013/11/05 15:08:51 INFO - Final point: 1
-2013/11/05 15:08:51 INFO - [a.1] -triggered off []
-2013/11/05 15:08:51 INFO - [b.1] -triggered off []
-2013/11/05 15:08:55 INFO - [c.1] -triggered off ['a.1']
-2013/11/05 15:08:55 INFO - [d.1] -triggered off ['a.1']
-2013/11/05 15:08:59 INFO - [e.1] -triggered off ['d.1']
diff --git a/tests/triggering/or-condition/suite.rc b/tests/triggering/or-condition/suite.rc
deleted file mode 100644
index 1c88f1754b4..00000000000
--- a/tests/triggering/or-condition/suite.rc
+++ /dev/null
@@ -1,9 +0,0 @@
-[scheduling]
- [[graph]]
- R1 = """a | b => c & d
- c | d => e"""
-[runtime]
- [[b,c]]
- script = sleep 10
- [[a,d, e]]
- script = true