From 51492d08add20f2089fa0ea533ea4666123d154b Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Fri, 10 Dec 2021 11:11:29 +1300 Subject: [PATCH 1/4] Fix job submit delta. --- cylc/flow/task_events_mgr.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py index d21d608597f..f3551c3ac19 100644 --- a/cylc/flow/task_events_mgr.py +++ b/cylc/flow/task_events_mgr.py @@ -1030,20 +1030,22 @@ def _process_message_submitted(self, itask, event_time, submit_num): f"[{summary['submit_method_id']}]" ) + # Register the newly submitted job with the database and datastore. + self._insert_task_job(itask, event_time, self.JOB_SUBMIT_SUCCESS_FLAG) + job_d = get_task_job_id( + itask.point, itask.tdef.name, itask.submit_num) + + itask.set_summary_time('submitted', event_time) + self.data_store_mgr.delta_job_time(job_d, 'submitted', event_time) if itask.tdef.run_mode == 'simulation': - # Simulate job execution at this point. - itask.set_summary_time('submitted', event_time) + # Simulate job started as well. itask.set_summary_time('started', event_time) + self.data_store_mgr.delta_job_time(job_d, 'started', event_time) if itask.state_reset(TASK_STATUS_RUNNING): self.data_store_mgr.delta_task_state(itask) itask.state.outputs.set_completion(TASK_OUTPUT_STARTED, True) self.data_store_mgr.delta_task_output(itask, TASK_OUTPUT_STARTED) - else: - itask.set_summary_time('submitted', event_time) - job_d = get_task_job_id( - itask.point, itask.tdef.name, itask.submit_num) - self.data_store_mgr.delta_job_time(job_d, 'submitted', event_time) self.data_store_mgr.delta_job_state(job_d, TASK_STATUS_SUBMITTED) # Unset started and finished times in case of resubmission. itask.set_summary_time('started') @@ -1065,9 +1067,6 @@ def _process_message_submitted(self, itask, event_time, submit_num): self.data_store_mgr.delta_task_queued(itask) self._reset_job_timers(itask) - # Register the newly submitted job with the database and datastore. - self._insert_task_job(itask, event_time, self.JOB_SUBMIT_SUCCESS_FLAG) - def _insert_task_job( self, itask: 'TaskProxy', From 09d3f2d32cb65b35264b4b8565692b81027cd2f0 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Fri, 10 Dec 2021 12:56:33 +1300 Subject: [PATCH 2/4] Extend a func test. --- tests/functional/graphql/03-is-held-arg.t | 24 ++++++++++++++++++++--- tests/functional/lib/python/diffr.py | 10 ++++------ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/tests/functional/graphql/03-is-held-arg.t b/tests/functional/graphql/03-is-held-arg.t index fc7c8ca9353..7f17ab59756 100755 --- a/tests/functional/graphql/03-is-held-arg.t +++ b/tests/functional/graphql/03-is-held-arg.t @@ -49,8 +49,12 @@ query { workflows { name isHeldTotal - taskProxies(isHeld: false) { + taskProxies(isHeld: true) { id + jobs { + submittedTime + startedTime + } } familyProxies(exids: [\"root\"], isHeld: true) { id @@ -69,14 +73,28 @@ run_ok "${TEST_NAME_BASE}-contact" cylc get-contact "${WORKFLOW_NAME}" # stop workflow cylc stop --max-polls=10 --interval=2 --kill "${WORKFLOW_NAME}" +RESPONSE="${TEST_NAME_BASE}-is-held-arg.stdout" +perl -pi -e 's/("submittedTime":).*$/${1} "blargh",/' "${RESPONSE}" +perl -pi -e 's/("startedTime":).*$/${1} "blargh"/' "${RESPONSE}" + # compare to expectation -cmp_json "${TEST_NAME}-out" "${TEST_NAME_BASE}-is-held-arg.stdout" << __HERE__ +cmp_json "${TEST_NAME}-out" "$RESPONSE" << __HERE__ { "workflows": [ { "name": "${WORKFLOW_NAME}", "isHeldTotal": 1, - "taskProxies": [], + "taskProxies": [ + { + "id": "${USER}${ID_DELIM}${WORKFLOW_NAME}${ID_DELIM}1${ID_DELIM}foo", + "jobs": [ + { + "submittedTime": "blargh", + "startedTime": "blargh" + } + ] + } + ], "familyProxies": [ { "id": "${USER}${ID_DELIM}${WORKFLOW_NAME}${ID_DELIM}1${ID_DELIM}BAZ" diff --git a/tests/functional/lib/python/diffr.py b/tests/functional/lib/python/diffr.py index 179e90a68e7..df2798b8d06 100644 --- a/tests/functional/lib/python/diffr.py +++ b/tests/functional/lib/python/diffr.py @@ -224,18 +224,16 @@ def load_json(file1, file2=None): """ try: this = json.loads(file1) - except json.decoder.JSONDecodeError: - sys.exit('Syntax error in file1') - raise + except json.decoder.JSONDecodeError as exc: + sys.exit(f'Syntax error in file1: {exc}') try: if file2: that = json.loads(file2) else: that = json.load(sys.stdin) - except json.decoder.JSONDecodeError: - sys.exit('Syntax error in file2') - raise + except json.decoder.JSONDecodeError as exc: + sys.exit(f'Syntax error in file2: {exc}') return this, that From 63e3033c7b6da793886aaa663b9a628cdcd6021b Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Fri, 10 Dec 2021 12:58:32 +1300 Subject: [PATCH 3/4] Update change log. --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index cfa3285f499..c666d5575d7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -80,6 +80,9 @@ are now sparse, i.e. they will no longer be fleshed-out with defaults. ### Fixes +[#4553](https://github.com/cylc/cylc-flow/pull/4553) - Add job submit time +to the datastore. + [#4526](https://github.com/cylc/cylc-flow/pull/4526) - Prevent `runN` and `run` being allowed as installation target names. From 8b2a16f1db1426c0046ba79a2a6a85b15b3d528d Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Thu, 16 Dec 2021 16:41:50 +1300 Subject: [PATCH 4/4] Copy task jobs on reload. --- cylc/flow/task_proxy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py index c7c46c632f4..893af5ffdc7 100644 --- a/cylc/flow/task_proxy.py +++ b/cylc/flow/task_proxy.py @@ -262,6 +262,7 @@ def copy_to_reload_successor(self, reload_successor): reload_successor.state.is_runahead = self.state.is_runahead reload_successor.state.is_updated = self.state.is_updated reload_successor.state.prerequisites = self.state.prerequisites + reload_successor.jobs = self.jobs @staticmethod def get_offset_as_seconds(offset):