More deterministic tests

Remove many sleeps in favour of detecting events. test_header: new function to install, validate and run reference test.
cylc · Sep 3, 2019 · fa44e30 · fa44e30
1 parent 4c6c05f
commit fa44e30
Show file tree

Hide file tree

Showing 310 changed files with 584 additions and 1,640 deletions.
diff --git a/cylc/flow/tests/test_wallclock.py b/cylc/flow/tests/test_wallclock.py
@@ -0,0 +1,50 @@
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+
+from metomi.isodatetime.data import CALENDAR
+from cylc.flow.wallclock import get_unix_time_from_time_string
+
+
+@pytest.mark.parametrize(
+    'time_str,time_sec',
+    [
+        ('2016-09-08T09:09:00+01', 1473322140),
+        ('2016-09-08T08:09:00Z', 1473322140),
+        ('2016-09-07T20:09:00-12', 1473322140),
+    ]
+)
+def test_get_unix_time_from_time_string(time_str, time_sec):
+    assert get_unix_time_from_time_string(time_str) == time_sec
+
+
+@pytest.mark.parametrize(
+    'time_str,time_sec',
+    [
+        ('2016-09-08T09:09:00+01', 1473322140),
+        ('2016-09-08T08:09:00Z', 1473322140),
+        ('2016-09-07T20:09:00-12', 1473322140),
+        ('2016-08-31T18:09:00+01', 1472663340),
+    ]
+)
+def test_get_unix_time_from_time_string_360(time_str, time_sec):
+    mode = CALENDAR.mode
+    CALENDAR.set_mode(CALENDAR.MODE_360)
+    try:
+        assert get_unix_time_from_time_string(time_str) == time_sec
+    finally:
+        CALENDAR.set_mode(mode)
diff --git a/flakytests/cylc-kill/02-submitted.t b/flakytests/cylc-kill/02-submitted.t
@@ -18,10 +18,5 @@
 # Test kill running jobs only
 . "$(dirname "$0")/test_header"
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/flakytests/cylc-poll/03-poll-all.t b/flakytests/cylc-poll/03-poll-all.t
@@ -17,14 +17,6 @@
 #-------------------------------------------------------------------------------
 # Test that when polling all a failed task sets the task state correctly
 . "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
 set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
-    cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/flakytests/cylc-poll/03-poll-all/suite.rc b/flakytests/cylc-poll/03-poll-all/suite.rc
@@ -46,7 +46,7 @@ pat2="[run_kill.${CYLC_TASK_CYCLE_POINT}] -suiciding"
 log="${CYLC_SUITE_LOG_DIR}/log"
 while (($(grep -c -F -e "${pat1}" -e "${pat2}" "${log}") != 2))
 do
-    sleep 2
+    sleep 1
 done
 
 st_file="${CYLC_SUITE_RUN_DIR}/log/job/${CYLC_TASK_CYCLE_POINT}/submit_hold/NN/job.status"

diff --git a/flakytests/cylc-reset/02-output-1.t b/flakytests/cylc-reset/02-output-1.t
@@ -17,12 +17,6 @@
 #-------------------------------------------------------------------------------
 # Test "cylc reset --output='TRIGGER|MESSAGE' 'SUITE' 'TASK.ID'".
 . "$(dirname "$0")/test_header"
-
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/flakytests/execution-time-limit/04-poll.t b/flakytests/execution-time-limit/04-poll.t
@@ -17,14 +17,6 @@
 #-------------------------------------------------------------------------------
 # Test execution time limit polling.
 . "$(dirname "$0")/test_header"
-
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" \
-    cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
-    cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/flakytests/hold-release/14-hold-kill.t b/flakytests/hold-release/14-hold-kill.t
@@ -17,12 +17,6 @@
 #-------------------------------------------------------------------------------
 # Test: hold a task to prevent it from retrying when killing its job
 . "$(dirname "$0")/test_header"
-
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}-run" cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/flakytests/hold-release/20-reset-waiting-output.t b/flakytests/hold-release/20-reset-waiting-output.t
@@ -19,10 +19,5 @@
 # https://github.com/cylc/cylc-flow/issues/2115
 . "$(dirname "$0")/test_header"
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-run_ok "${TEST_NAME_BASE}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/flakytests/integer-cycling/00-satellite.t b/flakytests/integer-cycling/00-satellite.t
@@ -18,14 +18,6 @@
 # Run the integer cycling satellite example suite.
 # Non-specific, tests a bunch of stuff.
 . "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
 set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
-    cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/flakytests/modes/03-dummy-env.t b/flakytests/modes/03-dummy-env.t
@@ -15,22 +15,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #-------------------------------------------------------------------------------
-
 # Test that user environment is disabled along with env-script in dummy mode.
 # And that remote host is disabled in dummy local mode.
-
 . "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
 set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" \
-    cylc run -m 'dummy-local' --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/tests/broadcast/02-inherit.t b/tests/broadcast/02-inherit.t
@@ -17,15 +17,6 @@
 #-------------------------------------------------------------------------------
 # Test broadcasts, with overriding inheritance.
 . "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
 set_test_number 2
-#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-#-------------------------------------------------------------------------------
-purge_suite "${SUITE_NAME}"
+reftest
+exit
diff --git a/tests/broadcast/03-expire.t b/tests/broadcast/03-expire.t
@@ -18,11 +18,5 @@
 # Test broadcast --expire=CYCLE_POINT
 . "$(dirname "$0")/test_header"
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
-    cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/tests/broadcast/04-empty.t b/tests/broadcast/04-empty.t
@@ -18,11 +18,5 @@
 # Test broadcast of an empty string
 . "$(dirname "$0")/test_header"
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
-    cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/tests/broadcast/07-float-setting.t b/tests/broadcast/07-float-setting.t
diff --git a/tests/cyclepoint/01-iso_timeparser.t → tests/broadcast/07-timeout.t b/tests/cyclepoint/01-iso_timeparser.t → tests/broadcast/07-timeout.t
@@ -15,9 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #-------------------------------------------------------------------------------
-# Run the time_parser.py tests
+# Test broadcasts a timeout setting
 . "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
-set_test_number 1
-#-------------------------------------------------------------------------------
-run_ok "${TEST_NAME_BASE}" python3 -m 'cylc.flow.time_parser'
+set_test_number 2
+reftest
+exit
diff --git a/.../broadcast/07-float-setting/reference.log → tests/broadcast/07-timeout/reference.log b/.../broadcast/07-float-setting/reference.log → tests/broadcast/07-timeout/reference.log
diff --git a/tests/broadcast/07-float-setting/suite.rc → tests/broadcast/07-timeout/suite.rc b/tests/broadcast/07-float-setting/suite.rc → tests/broadcast/07-timeout/suite.rc
@@ -6,17 +6,22 @@
 
 [scheduling]
     initial cycle point = 20100808T0000Z
-    final cycle point   = 20100809T0000Z
+    final cycle point   = 20100808T0000Z
     [[graph]]
         R1 = send_broadcast => timeout
 
 [runtime]
     [[send_broadcast]]
         script = """
             cylc broadcast -n timeout --point=20100808T0000Z --set='[events]execution timeout=PT1S' $CYLC_SUITE_NAME
-            sleep 10
         """
     [[timeout]]
-        script = sleep 10
+        script = """
+while ! grep -qF "[${CYLC_TASK_ID}] -execution timeout after PT1S" \
+    "${CYLC_SUITE_LOG_DIR}/log"
+do
+    sleep 1
+done
+"""
         [[[events]]]
             execution timeout = PT1M
diff --git a/tests/broadcast/08-space.t b/tests/broadcast/08-space.t
@@ -18,11 +18,5 @@
 # Test broadcast -s '[foo]  bar=baz' syntax. cylc/cylc-flow#1680
 . "$(dirname "$0")/test_header"
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
-    cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/tests/broadcast/09-remote.t b/tests/broadcast/09-remote.t
@@ -20,12 +20,6 @@ export CYLC_TEST_IS_GENERIC=false
 . "$(dirname "$0")/test_header"
 set_test_remote_host
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}-run" \
-    cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-
+reftest
 purge_suite_remote "${CYLC_TEST_HOST}" "${SUITE_NAME}"
-purge_suite "${SUITE_NAME}"
 exit
diff --git a/tests/broadcast/10-file-1.t b/tests/broadcast/10-file-1.t
@@ -18,11 +18,5 @@
 # Test broadcast settings in a file
 . "$(dirname "$0")/test_header"
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
-    cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/tests/broadcast/10-file-1/suite.rc b/tests/broadcast/10-file-1/suite.rc
@@ -7,7 +7,6 @@
     [[t1]]
         script = """
 cylc broadcast -n 't2' -F "${CYLC_SUITE_DEF_PATH}/broadcast.rc" "${CYLC_SUITE_NAME}"
-sleep 1
 """
     [[t2]]
         script = false
diff --git a/tests/broadcast/11-file-2.t b/tests/broadcast/11-file-2.t
@@ -18,11 +18,5 @@
 # Test broadcast settings in 2 files
 . "$(dirname "$0")/test_header"
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
-    cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/tests/broadcast/11-file-2/suite.rc b/tests/broadcast/11-file-2/suite.rc
@@ -10,7 +10,6 @@ cylc broadcast -n 't2' \
     -F "${CYLC_SUITE_DEF_PATH}/broadcast-1.rc" \
     -F "${CYLC_SUITE_DEF_PATH}/broadcast-2.rc" \
     "${CYLC_SUITE_NAME}"
-sleep 1
 """
     [[t2]]
         script = false
diff --git a/tests/broadcast/12-file-stdin.t b/tests/broadcast/12-file-stdin.t
@@ -18,11 +18,5 @@
 # Test broadcast settings from STDIN
 . "$(dirname "$0")/test_header"
 set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
-    cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
-purge_suite "${SUITE_NAME}"
+reftest
 exit
diff --git a/tests/broadcast/12-file-stdin/suite.rc b/tests/broadcast/12-file-stdin/suite.rc
@@ -8,7 +8,6 @@
         script = """
 cylc broadcast -n 't2' -F - "${CYLC_SUITE_NAME}" \
     <"${CYLC_SUITE_DEF_PATH}/broadcast.rc"
-sleep 1
 """
     [[t2]]
         script = false