Skip to content

Commit

Permalink
Result_traceback should not include job stdout
Browse files Browse the repository at this point in the history
If a job fails, we do receptor work results and put that output
into result_traceback.

We should only do this if
1. Receptor unit has failed
2. Runner callback processed 0 events

Otherwise we risk putting too much data into this field.
  • Loading branch information
fosterseth committed Sep 27, 2022
1 parent 9c2185c commit 94410bf
Showing 1 changed file with 13 additions and 3 deletions.
16 changes: 13 additions & 3 deletions awx/main/tasks/receptor.py
Original file line number Diff line number Diff line change
Expand Up @@ -398,9 +398,11 @@ def _run_internal(self, receptor_ctl):
unit_status = receptor_ctl.simple_command(f'work status {self.unit_id}')
detail = unit_status.get('Detail', None)
state_name = unit_status.get('StateName', None)
stdout_size = unit_status.get('StdoutSize', 0)
except Exception:
detail = ''
state_name = ''
stdout_size = 0
logger.exception(f'An error was encountered while getting status for work unit {self.unit_id}')

if 'exceeded quota' in detail:
Expand All @@ -411,15 +413,23 @@ def _run_internal(self, receptor_ctl):
return

try:
resultsock = receptor_ctl.get_work_results(self.unit_id, return_sockfile=True)
lines = resultsock.readlines()
receptor_output = b"".join(lines).decode()
receptor_output = ''
if state_name == 'Failed' and self.task.runner_callback.event_ct == 0:
# if receptor work unit failed and no events were emitted, work results may
# contain useful information about why the job failed. In case stdout is
# massive, only ask for last 1000 bytes
startpos = max(stdout_size - 1000, 0)
resultsock = receptor_ctl.get_work_results(self.unit_id, startpos=startpos, return_sockfile=True)
result.setblocking(False)
lines = resultsock.readlines()
receptor_output = b"".join(lines).decode()
if receptor_output:
self.task.runner_callback.delay_update(result_traceback=receptor_output)
elif detail:
self.task.runner_callback.delay_update(result_traceback=detail)
else:
logger.warning(f'No result details or output from {self.task.instance.log_format}, status:\n{state_name}')

except Exception:
raise RuntimeError(detail)

Expand Down

0 comments on commit 94410bf

Please sign in to comment.