From 51f705aeebb12c96eeb4308b13094cbf473999f2 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Sat, 17 Nov 2018 20:26:32 -0600 Subject: [PATCH 1/6] New job "exit-script" for EXIT, ERR, TERM. Note "cylc kill" for bg jobs use SIGKILL, so the exit-script doesn't run in that case. --- lib/cylc/cfgspec/suite.py | 1 + lib/cylc/job.sh | 16 +++++++++------- lib/cylc/job_file.py | 5 +++-- lib/cylc/task_job_mgr.py | 1 + 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/cylc/cfgspec/suite.py b/lib/cylc/cfgspec/suite.py index 6ad41659054..49e10c3f107 100644 --- a/lib/cylc/cfgspec/suite.py +++ b/lib/cylc/cfgspec/suite.py @@ -165,6 +165,7 @@ 'init-script': [VDR.V_STRING], 'env-script': [VDR.V_STRING], 'err-script': [VDR.V_STRING], + 'exit-script': [VDR.V_STRING], 'pre-script': [VDR.V_STRING], 'script': [VDR.V_STRING], 'post-script': [VDR.V_STRING], diff --git a/lib/cylc/job.sh b/lib/cylc/job.sh index d15e3277a8a..e9502ab1af7 100644 --- a/lib/cylc/job.sh +++ b/lib/cylc/job.sh @@ -144,6 +144,7 @@ cylc__job__main() { wait "${CYLC_TASK_MESSAGE_STARTED_PID}" 2>'/dev/null' || true cylc message -- "${CYLC_SUITE_NAME}" "${CYLC_TASK_JOB}" 'succeeded' || true trap '' ${CYLC_VACATION_SIGNALS:-} ${CYLC_FAIL_SIGNALS} + cylc__job__run_inst_func 'exit_script' exit 0 } @@ -175,7 +176,7 @@ cylc__job__run_inst_func() { # see "cylc help message" for format of messages. # Returns: # exit 1 -cylc__job_finish() { +cylc__job_finish_err() { typeset signal="$1" typeset run_err_script="$2" shift 2 @@ -188,25 +189,26 @@ cylc__job_finish() { if "${run_err_script}"; then cylc__job__run_inst_func 'err_script' "${signal}" >&2 fi + cylc__job__run_inst_func 'exit_script' exit 1 } ############################################################################### -# Wrap cylc__job_finish to abort with a user-defined error message. +# Wrap cylc__job_finish_err to abort with a user-defined error message. cylc__job_abort() { - cylc__job_finish "EXIT" true "CRITICAL: aborted/\"${1}\"" + cylc__job_finish_err "EXIT" true "CRITICAL: aborted/\"${1}\"" } ############################################################################### -# Wrap cylc__job_finish for job preempt/vacation signal trap. +# Wrap cylc__job_finish_err for job preempt/vacation signal trap. cylc__job_vacation() { - cylc__job_finish "${1}" false "WARNING: vacated/${1}" + cylc__job_finish_err "${1}" false "WARNING: vacated/${1}" } ############################################################################### -# Wrap cylc__job_finish for automatic job exit signal trap. +# Wrap cylc__job_finish_err for automatic job exit signal trap. cylc__job_err() { - cylc__job_finish "${1}" true "CRITICAL: failed/${1}" + cylc__job_finish_err "${1}" true "CRITICAL: failed/${1}" } ############################################################################### diff --git a/lib/cylc/job_file.py b/lib/cylc/job_file.py index af09df03a8b..05fcec8ed65 100644 --- a/lib/cylc/job_file.py +++ b/lib/cylc/job_file.py @@ -296,9 +296,10 @@ def _write_global_init_script(cls, handle, job_conf): def _write_script(cls, handle, job_conf): """Write (*-)script in functions. - init-script, env-script, err-script, pre-script, script, post-script + init-script, env-script, err-script, pre-script, script, post-script, + exit-script """ - for prefix in ['init-', 'env-', 'err-', 'pre-', '', 'post-']: + for prefix in ['init-', 'env-', 'err-', 'pre-', '', 'post-', 'exit-']: value = job_conf[prefix + 'script'] if cls._check_script_value(value): handle.write("\n\ncylc__job__inst__%sscript() {" % ( diff --git a/lib/cylc/task_job_mgr.py b/lib/cylc/task_job_mgr.py index 866a301e0b5..dcbe7498eb5 100644 --- a/lib/cylc/task_job_mgr.py +++ b/lib/cylc/task_job_mgr.py @@ -849,6 +849,7 @@ def _prep_submit_task_job_impl(self, suite, itask, rtconfig): 'execution_time_limit': itask.summary[self.KEY_EXECUTE_TIME_LIMIT], 'env-script': rtconfig['env-script'], 'err-script': rtconfig['err-script'], + 'exit-script': rtconfig['exit-script'], 'host': itask.task_host, 'init-script': rtconfig['init-script'], 'job_file_path': job_file_path, From c580c3eaf1f75d9e6f7d98f11f09c007f2b4812c Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Sat, 17 Nov 2018 21:48:37 -0600 Subject: [PATCH 2/6] Add tests for exit-script. --- tests/jobscript/19-exit-script.t | 50 +++++++++++++++++++++++++ tests/jobscript/19-exit-script/suite.rc | 30 +++++++++++++++ tests/lib/bash/test_header | 16 ++++++++ 3 files changed, 96 insertions(+) create mode 100755 tests/jobscript/19-exit-script.t create mode 100644 tests/jobscript/19-exit-script/suite.rc diff --git a/tests/jobscript/19-exit-script.t b/tests/jobscript/19-exit-script.t new file mode 100755 index 00000000000..cc00ec0f233 --- /dev/null +++ b/tests/jobscript/19-exit-script.t @@ -0,0 +1,50 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#------------------------------------------------------------------------------ +# Test exit-script. + +. "$(dirname "${0}")/test_header" +set_test_number 9 + +install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" + +run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" + +# 1) runs on normal successful job exit? +run_ok "${TEST_NAME_BASE}-run" \ + cylc run --debug --no-detach "${SUITE_NAME}" +grep_ok 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out" +grep_fail 'Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err" + +# 2) runs on internal early EXIT? +run_fail "${TEST_NAME_BASE}-run" \ + cylc run --debug --no-detach --set=EXIT=true "${SUITE_NAME}" +grep_ok 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out" +grep_ok 'EXIT Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err" + +# 3) runs on external job TERM? +cylc run --set=NSLEEP=30 "${SUITE_NAME}" +sleep 3 +PID_SET=$(grep CYLC_JOB_PID "${SUITE_RUN_DIR}/log/job/1/foo/01/job.status") +eval $PID_SET +kill -s TERM $CYLC_JOB_PID +sleep 5 +grep_ok 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out" +grep_ok 'TERM Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err" + +purge_suite "${SUITE_NAME}" diff --git a/tests/jobscript/19-exit-script/suite.rc b/tests/jobscript/19-exit-script/suite.rc new file mode 100644 index 00000000000..209a8347352 --- /dev/null +++ b/tests/jobscript/19-exit-script/suite.rc @@ -0,0 +1,30 @@ +#!Jinja2 + +{% set NSLEEP = NSLEEP | default(1) %} +{% set EXIT = EXIT | default("false") %} + +[cylc] + [[events]] + abort on stalled = True +[scheduling] + [[dependencies]] + graph = foo +[runtime] + [[foo]] + exit-script = echo "Cheesy peas!" + err-script = echo "$1 Oops!" + script = """ +echo "HELLO" +if {{EXIT}}; then + exit 0 +fi +# This must not be a single monolothic sleep. If a command is running when +# bash receives a signal that a trap is set for, it waits for the command to +# complete before executing the trap. We want a quick exit, to make it +# obvious that the TERM signal had the intended effect. +for I in $(seq 1 {{NSLEEP}}); do + echo $I + sleep 1 +done +echo "BYE" + """ diff --git a/tests/lib/bash/test_header b/tests/lib/bash/test_header index 618e8d70413..f5bdef8c754 100644 --- a/tests/lib/bash/test_header +++ b/tests/lib/bash/test_header @@ -56,6 +56,8 @@ # (stdin if FILE_CONTROL is "-" or missing). # grep_ok PATTERN FILE # Run "grep -q -e PATTERN FILE". +# grep_fail PATTERN FILE +# Run "grep -q -e PATTERN FILE", expect no match. # count_ok PATTERN FILE COUNT # Test that PATTERN occurs in exactly COUNT lines of FILE. # exists_ok FILE @@ -332,6 +334,20 @@ grep_ok() { fail "${TEST_NAME}" } +grep_fail() { + local BRE="$1" + local FILE="$2" + local TEST_NAME="$(basename "${FILE}")-grep-fail" + if grep -q -e "${BRE}" "${FILE}"; then + mkdir -p "${TEST_LOG_DIR}" + echo "ERROR: Found ${BRE} in ${FILE}" \ + >"${TEST_LOG_DIR}/${TEST_NAME}.stderr" + fail "${TEST_NAME}" + return + fi + ok "${TEST_NAME}" +} + exists_ok() { local FILE=$1 local TEST_NAME="$(basename "${FILE}")-file-exists-ok" From 6a88e257b97b9ce38d020903b01f43199e939882 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Tue, 20 Nov 2018 16:09:57 -0600 Subject: [PATCH 3/6] Add new item to config test. --- tests/cylc-get-config/00-simple/section2.stdout | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/cylc-get-config/00-simple/section2.stdout b/tests/cylc-get-config/00-simple/section2.stdout index aad1b8985c1..6216a7c4912 100644 --- a/tests/cylc-get-config/00-simple/section2.stdout +++ b/tests/cylc-get-config/00-simple/section2.stdout @@ -2,6 +2,7 @@ script = echo "RUN: run-var.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = @@ -82,6 +83,7 @@ script = echo "RUN: run-ops.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = @@ -161,6 +163,7 @@ script = echo "RUN: run-ops.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = @@ -241,6 +244,7 @@ script = echo "RUN: run-ops.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = @@ -321,6 +325,7 @@ script = echo "RUN: run-ops.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = @@ -401,6 +406,7 @@ script = echo "RUN: run-ops.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = @@ -481,6 +487,7 @@ script = echo "RUN: run-var.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = @@ -561,6 +568,7 @@ script = echo "RUN: run-var.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = @@ -640,6 +648,7 @@ script = echo "RUN: run-var.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = @@ -719,6 +728,7 @@ [[SERIAL]] env-script = err-script = + exit-script = script = extra log files = work sub-directory = @@ -799,6 +809,7 @@ [[root]] env-script = err-script = + exit-script = script = extra log files = work sub-directory = @@ -878,6 +889,7 @@ [[PARALLEL]] env-script = err-script = + exit-script = script = extra log files = work sub-directory = @@ -959,6 +971,7 @@ script = echo "RUN: run-var.sh" env-script = err-script = + exit-script = extra log files = work sub-directory = init-script = From a5ab761cdf2ee97ead91e736f1a96533c4da3381 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 21 Nov 2018 15:56:43 +1300 Subject: [PATCH 4/6] Run exist-script on success only. --- lib/cylc/job.sh | 2 +- tests/jobscript/19-exit-script.t | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/cylc/job.sh b/lib/cylc/job.sh index e9502ab1af7..31fa7c1db8e 100644 --- a/lib/cylc/job.sh +++ b/lib/cylc/job.sh @@ -144,6 +144,7 @@ cylc__job__main() { wait "${CYLC_TASK_MESSAGE_STARTED_PID}" 2>'/dev/null' || true cylc message -- "${CYLC_SUITE_NAME}" "${CYLC_TASK_JOB}" 'succeeded' || true trap '' ${CYLC_VACATION_SIGNALS:-} ${CYLC_FAIL_SIGNALS} + # Execute success exit script cylc__job__run_inst_func 'exit_script' exit 0 } @@ -189,7 +190,6 @@ cylc__job_finish_err() { if "${run_err_script}"; then cylc__job__run_inst_func 'err_script' "${signal}" >&2 fi - cylc__job__run_inst_func 'exit_script' exit 1 } diff --git a/tests/jobscript/19-exit-script.t b/tests/jobscript/19-exit-script.t index cc00ec0f233..c74867f4899 100755 --- a/tests/jobscript/19-exit-script.t +++ b/tests/jobscript/19-exit-script.t @@ -25,26 +25,26 @@ install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" -# 1) runs on normal successful job exit? +# 1) Should run on normal successful job exit. run_ok "${TEST_NAME_BASE}-run" \ cylc run --debug --no-detach "${SUITE_NAME}" grep_ok 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out" grep_fail 'Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err" -# 2) runs on internal early EXIT? +# 2) Should not run on internal early EXIT. run_fail "${TEST_NAME_BASE}-run" \ cylc run --debug --no-detach --set=EXIT=true "${SUITE_NAME}" -grep_ok 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out" +grep_fail 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out" grep_ok 'EXIT Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err" -# 3) runs on external job TERM? +# 3) Should not run on external job TERM. cylc run --set=NSLEEP=30 "${SUITE_NAME}" sleep 3 PID_SET=$(grep CYLC_JOB_PID "${SUITE_RUN_DIR}/log/job/1/foo/01/job.status") eval $PID_SET kill -s TERM $CYLC_JOB_PID sleep 5 -grep_ok 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out" +grep_fail 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out" grep_ok 'TERM Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err" purge_suite "${SUITE_NAME}" From e8281dfffe3d12c5a416fb2d1a7c7d0c12fab25d Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 21 Nov 2018 16:13:56 +1300 Subject: [PATCH 5/6] Document exit-script. --- doc/src/cylc-user-guide/suiterc.tex | 76 +++++++++++++++++------------ 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/doc/src/cylc-user-guide/suiterc.tex b/doc/src/cylc-user-guide/suiterc.tex index 7966a5f2491..81e23cabf40 100644 --- a/doc/src/cylc-user-guide/suiterc.tex +++ b/doc/src/cylc-user-guide/suiterc.tex @@ -1165,14 +1165,14 @@ \subsection{[runtime]} \paragraph[init-script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow init-script} -This is invoked by the task job script before the task execution -environment is configured, so it does not have access to any suite or task -environment variables. It can be a single command or multiple lines of -scripting. The original intention was to allow remote tasks to +Custom script invoked by the task job script before the task execution environment +is configured - so it does not have access to any suite or task +environment variables. It can be an external command or script, or inlined +scripting. The original intention for this item was to allow remote tasks to source login scripts to configure their access to cylc, but this should no longer be necessary (see~\ref{HowTasksGetAccessToCylc}). See also -\lstinline=env-script=, \lstinline=err-script=, \lstinline=pre-script=, -\lstinline=script=, and \lstinline=post-script=. +\lstinline=env-script=, \lstinline=err-script=, \lstinline=exit-script=, +\lstinline=pre-script=, \lstinline=script=, and \lstinline=post-script=. \begin{myitemize} \item {\em type:} string @@ -1182,13 +1182,13 @@ \subsection{[runtime]} \paragraph[env-script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow env-script} -This is invoked by the task job script between the cylc-defined environment +Custom script invoked by the task job script between the cylc-defined environment (suite and task identity, etc.) and the user-defined task runtime environment - -i.e.\ it has access to the cylc environment, and the task environment has -access to variables defined by this scripting. It can be a single command or -multiple lines of scripting. See also \lstinline=init-script=, -\lstinline=err-script=, \lstinline=pre-script=, \lstinline=script=, and -\lstinline=post-script=. +so it has access to the cylc environment (and the task environment has +access to variables defined by this scripting). It can be an external command +or script, or inlined scripting. See also \lstinline=init-script=, +\lstinline=err-script=, \lstinline=exit-script=, \lstinline=pre-script=, +\lstinline=script=, and \lstinline=post-script=. \begin{myitemize} \item {\em type:} string @@ -1196,16 +1196,32 @@ \subsection{[runtime]} \item {\em example:} \lstinline@env-script = "echo Hello World"@ \end{myitemize} +\paragraph[exit-script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow exit-script} + +Custom script invoked at the very end of {\em successful} job execution, just +before the job script exits. It should execute very quickly. Companion of +\lstinline=err-script=, which is executed on job failure. It can be an external +command or script, or inlined scripting. See also \lstinline=init-script=, +\lstinline=env-script=, \lstinline=exit-script=, \lstinline=pre-script=, +\lstinline=script=, and \lstinline=post-script=. + +\begin{myitemize} +\item {\em type:} string +\item {\em default:} (none) +\item {\em example:} \lstinline@exit-script = "rm -f $TMP_FILES"@ +\end{myitemize} + \paragraph[err-script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow err-script} -This is any custom script to be invoked at the end of the error trap, (if the -error trap is triggered due to failure of a command in the task job). The +Custom script to be invoked at the end of the error trap, which is triggered +due to failure of a command in the task job script or trapable job kill. The output of this will always be sent to STDERR and \lstinline=$1= is set to the name of the signal caught by the error trap. The script should be fast and use very little system resource to ensure that the error trap can return quickly. -It can be a single command or multiple lines of scripting. See also -\lstinline=init-script=, \lstinline=env-script=, \lstinline=pre-script=, -\lstinline=script=, and \lstinline=post-script=. +Companion of \lstinline=exit-script=, which is executed on job success. +It can be an external command or script, or inlined scripting. See also +\lstinline=init-script=, \lstinline=env-script=, \lstinline=exit-script=, +\lstinline=pre-script=, \lstinline=script=, and \lstinline=post-script=. \begin{myitemize} \item {\em type:} string @@ -1215,10 +1231,11 @@ \subsection{[runtime]} \paragraph[pre-script]{ [runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow pre-script} -This is invoked by the task job script immediately before the \lstinline=script= -item (just below). It can be a single command or multiple lines of scripting. -See also \lstinline=init-script=, \lstinline=env-script=, \lstinline=err-script=, -\lstinline=script=, and \lstinline=post-script=. +Custom script invoked by the task job script immediately before the \lstinline=script= +item (just below). It can be an external command or script, or inlined scripting. +See also \lstinline=init-script=, \lstinline=env-script=, +\lstinline=err-script=, \lstinline=exit-script=, \lstinline=script=, and +\lstinline=post-script=. \begin{myitemize} \item {\em type:} string @@ -1234,10 +1251,10 @@ \subsection{[runtime]} \paragraph[script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow script} \label{ScriptItem} -The is the main user-defined scripting to run when the task is ready. It can be a -single command or multiple lines of scripting. See also \lstinline=init-script=, -\lstinline=env-script=, \lstinline=err-script=, \lstinline=pre-script=, and -\lstinline=post-script=. +The main custom script invoked from the task job script. It can be an +external command or script, or inlined scripting. See also +\lstinline=init-script=, \lstinline=env-script=, \lstinline=err-script=, +\lstinline=exit-script=, \lstinline=pre-script=, and \lstinline=post-script=. \begin{myitemize} \item {\em type:} string @@ -1245,12 +1262,11 @@ \subsection{[runtime]} \end{myitemize} \paragraph[post-script]{ [runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow post-script} - -This is invoked by the task job script immediately after the \lstinline=script= -item (just above). It can be a single command or multiple lines of scripting. -See also +Custom script invoked by the task job script immediately after the +\lstinline=script= item (just above). It can be an external command or script, +or inlined scripting. See also \lstinline=init-script=, \lstinline=env-script=, \lstinline=err-script=, -\lstinline=pre-script=, and \lstinline=script=. +\lstinline=exit-script=, \lstinline=pre-script=, and \lstinline=script=. \begin{myitemize} \item {\em type:} string From 5841d2eade6206d14a1c97551b22cb26da9512dd Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 21 Nov 2018 10:16:52 +1300 Subject: [PATCH 6/6] Address review feedback. --- tests/jobscript/19-exit-script.t | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/jobscript/19-exit-script.t b/tests/jobscript/19-exit-script.t index c74867f4899..30c881c292b 100755 --- a/tests/jobscript/19-exit-script.t +++ b/tests/jobscript/19-exit-script.t @@ -40,8 +40,7 @@ grep_ok 'EXIT Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err" # 3) Should not run on external job TERM. cylc run --set=NSLEEP=30 "${SUITE_NAME}" sleep 3 -PID_SET=$(grep CYLC_JOB_PID "${SUITE_RUN_DIR}/log/job/1/foo/01/job.status") -eval $PID_SET +CYLC_JOB_PID=$(sed -n 's/^CYLC_JOB_PID=//p' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.status") kill -s TERM $CYLC_JOB_PID sleep 5 grep_fail 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out"