Skip to content

Commit

Permalink
Better event handler failure and retry messages (#4826)
Browse files Browse the repository at this point in the history
  • Loading branch information
hjoliver authored Apr 20, 2022
1 parent 6fc565a commit 5361c42
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 10 deletions.
11 changes: 6 additions & 5 deletions cylc/flow/task_events_mgr.py
Original file line number Diff line number Diff line change
Expand Up @@ -463,21 +463,22 @@ def process_events(self, schd: 'Scheduler') -> None:
if not timer.is_timeout_set():
if timer.next() is None:
LOG.warning(
f"{point}/{name}/{submit_num:02d} {key1} failed"
f"{point}/{name}/{submit_num:02d}"
f" handler:{key1[0]} for task event:{key1[1]} failed"
)
self.remove_event_timer(id_key)
continue
# Report retries and delayed 1st try
msg = None
if timer.num > 1:
msg = (
f"{key1} failed, "
f"retrying in {timer.delay_timeout_as_str()}"
f"handler:{key1[0]} for task event:{key1[1]} failed,"
f" retrying in {timer.delay_timeout_as_str()}"
)
elif timer.delay:
msg = (
f"{key1} will run after "
f"{timer.delay_timeout_as_str()}"
f"handler:{key1[0]} for task event:{key1[1]} will"
f" run after {timer.delay_timeout_as_str()}"
)
if msg:
LOG.debug(f"{point}/{name}/{submit_num:02d} {msg}")
Expand Down
2 changes: 1 addition & 1 deletion tests/flakyfunctional/events/44-timeout.t
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ contains_ok 'log' <<__END__
ERROR - [(('event-handler-00', 'started'), 1) cmd] sleeper.sh 1/foo
${LOG_INDENT}[(('event-handler-00', 'started'), 1) ret_code] -9
${LOG_INDENT}[(('event-handler-00', 'started'), 1) err] killed on timeout (PT10S)
WARNING - 1/foo/01 ('event-handler-00', 'started') failed
WARNING - 1/foo/01 handler:event-handler-00 for task event:started failed
__END__

cylc workflow-state "${WORKFLOW_NAME}" >'workflow-state.log'
Expand Down
8 changes: 4 additions & 4 deletions tests/functional/events/10-task-event-job-logs-retrieve.t
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,11 @@ cmp_ok 'edited-activities.log' <<'__LOG__'
__LOG__

grep -F 'will run after' "${WORKFLOW_RUN_DIR}/log/workflow/log" \
| cut -d' ' -f 4-10 | sort >"edited-log"
| cut -d' ' -f 4-12 | sort >"edited-log"
cmp_ok 'edited-log' <<'__LOG__'
1/t1/01 ('job-logs-retrieve', 'retry') will run after PT5S
1/t1/02 ('job-logs-retrieve', 'retry') will run after PT5S
1/t1/03 ('job-logs-retrieve', 'succeeded') will run after PT5S
1/t1/01 handler:job-logs-retrieve for task event:retry will run after PT5S
1/t1/02 handler:job-logs-retrieve for task event:retry will run after PT5S
1/t1/03 handler:job-logs-retrieve for task event:succeeded will run after PT5S
__LOG__

purge
Expand Down

0 comments on commit 5361c42

Please sign in to comment.