job exit script

doc/src/cylc-user-guide/suiterc.tex

@@ -1165,14 +1165,14 @@ \subsection{[runtime]}
 
 \paragraph[init-script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow init-script}
 
-This is invoked by the task job script before the task execution
-environment is configured, so it does not have access to any suite or task
-environment variables. It can be a single command or multiple lines of
-scripting. The original intention was to allow remote tasks to
+Custom script invoked by the task job script before the task execution environment
+is configured - so it does not have access to any suite or task
+environment variables. It can be an external command or script, or inlined
+scripting. The original intention for this item was to allow remote tasks to
 source login scripts to configure their access to cylc, but this should no
 longer be necessary (see~\ref{HowTasksGetAccessToCylc}). See also
-\lstinline=env-script=, \lstinline=err-script=, \lstinline=pre-script=,
-\lstinline=script=, and \lstinline=post-script=.
+\lstinline=env-script=, \lstinline=err-script=, \lstinline=exit-script=,
+\lstinline=pre-script=, \lstinline=script=, and \lstinline=post-script=.
 
 \begin{myitemize}
 \item {\em type:} string
@@ -1182,30 +1182,46 @@ \subsection{[runtime]}
 
 \paragraph[env-script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow env-script}
 
-This is invoked by the task job script between the cylc-defined environment
+Custom script invoked by the task job script between the cylc-defined environment
 (suite and task identity, etc.) and the user-defined task runtime environment -
-i.e.\ it has access to the cylc environment, and the task environment has
-access to variables defined by this scripting. It can be a single command or
-multiple lines of scripting.  See also \lstinline=init-script=,
-\lstinline=err-script=, \lstinline=pre-script=, \lstinline=script=, and
-\lstinline=post-script=.
+so it has access to the cylc environment (and the task environment has
+access to variables defined by this scripting). It can be an external command
+or script, or inlined scripting. See also \lstinline=init-script=,
+\lstinline=err-script=, \lstinline=exit-script=, \lstinline=pre-script=,
+\lstinline=script=, and \lstinline=post-script=.
 
 \begin{myitemize}
 \item {\em type:} string
 \item {\em default:} (none)
 \item {\em example:} \lstinline@env-script = "echo Hello World"@
 \end{myitemize}
 
+\paragraph[exit-script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow exit-script}
+
+Custom script invoked at the very end of {\em successful} job execution, just
+before the job script exits. It should execute very quickly. Companion of
+\lstinline=err-script=, which is executed on job failure. It can be an external
+command or script, or inlined scripting. See also \lstinline=init-script=,
+\lstinline=env-script=, \lstinline=exit-script=, \lstinline=pre-script=,
+\lstinline=script=, and \lstinline=post-script=.
+
+\begin{myitemize}
+\item {\em type:} string
+\item {\em default:} (none)
+\item {\em example:} \lstinline@exit-script = "rm -f $TMP_FILES"@
+\end{myitemize}
+
 \paragraph[err-script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow err-script}
 
-This is any custom script to be invoked at the end of the error trap, (if the
-error trap is triggered due to failure of a command in the task job). The
+Custom script to be invoked at the end of the error trap, which is triggered
+due to failure of a command in the task job script or trapable job kill. The
 output of this will always be sent to STDERR and \lstinline=$1= is set to the
 name of the signal caught by the error trap. The script should be fast and use
 very little system resource to ensure that the error trap can return quickly.
-It can be a single command or multiple lines of scripting. See also
-\lstinline=init-script=, \lstinline=env-script=, \lstinline=pre-script=,
-\lstinline=script=, and \lstinline=post-script=.
+Companion of \lstinline=exit-script=, which is executed on job success.
+It can be an external command or script, or inlined scripting. See also
+\lstinline=init-script=, \lstinline=env-script=, \lstinline=exit-script=,
+\lstinline=pre-script=, \lstinline=script=, and \lstinline=post-script=.
 
 \begin{myitemize}
 \item {\em type:} string
@@ -1215,10 +1231,11 @@ \subsection{[runtime]}
 
 \paragraph[pre-script]{ [runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow pre-script}
 
-This is invoked by the task job script immediately before the \lstinline=script=
-item (just below). It can be a single command or multiple lines of scripting.
-See also \lstinline=init-script=, \lstinline=env-script=, \lstinline=err-script=,
-\lstinline=script=, and \lstinline=post-script=.
+Custom script invoked by the task job script immediately before the \lstinline=script=
+item (just below). It can be an external command or script, or inlined scripting.
+See also \lstinline=init-script=, \lstinline=env-script=,
+\lstinline=err-script=, \lstinline=exit-script=, \lstinline=script=, and
+\lstinline=post-script=.
 
 \begin{myitemize}
 \item {\em type:} string
@@ -1234,23 +1251,22 @@ \subsection{[runtime]}
 \paragraph[script]{[runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow script}
 \label{ScriptItem}
 
-The is the main user-defined scripting to run when the task is ready. It can be a
-single command or multiple lines of scripting. See also \lstinline=init-script=,
-\lstinline=env-script=, \lstinline=err-script=, \lstinline=pre-script=, and
-\lstinline=post-script=.
+The main custom script invoked from the task job script. It can be an
+external command or script, or inlined scripting. See also
+\lstinline=init-script=, \lstinline=env-script=, \lstinline=err-script=,
+\lstinline=exit-script=, \lstinline=pre-script=, and \lstinline=post-script=.
 
 \begin{myitemize}
 \item {\em type:} string
 \item {\em root default:} (none)
 \end{myitemize}
 
 \paragraph[post-script]{ [runtime] \textrightarrow [[\_\_NAME\_\_]] \textrightarrow post-script}
-
-This is invoked by the task job script immediately after the \lstinline=script=
-item (just above).  It can be a single command or multiple lines of scripting.
-See also
+Custom script invoked by the task job script immediately after the
+\lstinline=script= item (just above). It can be an external command or script,
+or inlined scripting.  See also
 \lstinline=init-script=, \lstinline=env-script=, \lstinline=err-script=,
-\lstinline=pre-script=, and \lstinline=script=.
+\lstinline=exit-script=, \lstinline=pre-script=, and \lstinline=script=.
 
 \begin{myitemize}
 \item {\em type:} string

lib/cylc/cfgspec/suite.py

@@ -165,6 +165,7 @@
             'init-script': [VDR.V_STRING],
             'env-script': [VDR.V_STRING],
             'err-script': [VDR.V_STRING],
+            'exit-script': [VDR.V_STRING],
             'pre-script': [VDR.V_STRING],
             'script': [VDR.V_STRING],
             'post-script': [VDR.V_STRING],

lib/cylc/job.sh

@@ -144,6 +144,8 @@ cylc__job__main() {
     wait "${CYLC_TASK_MESSAGE_STARTED_PID}" 2>'/dev/null' || true
     cylc message -- "${CYLC_SUITE_NAME}" "${CYLC_TASK_JOB}" 'succeeded' || true
     trap '' ${CYLC_VACATION_SIGNALS:-} ${CYLC_FAIL_SIGNALS}
+    # Execute success exit script
+    cylc__job__run_inst_func 'exit_script'
     exit 0
 }
 
@@ -175,7 +177,7 @@ cylc__job__run_inst_func() {
 #     see "cylc help message" for format of messages.
 # Returns:
 #   exit 1
-cylc__job_finish() {
+cylc__job_finish_err() {
     typeset signal="$1"
     typeset run_err_script="$2"
     shift 2
@@ -192,21 +194,21 @@ cylc__job_finish() {
 }
 
 ###############################################################################
-# Wrap cylc__job_finish to abort with a user-defined error message.
+# Wrap cylc__job_finish_err to abort with a user-defined error message.
 cylc__job_abort() {
-    cylc__job_finish "EXIT" true "CRITICAL: aborted/\"${1}\""
+    cylc__job_finish_err "EXIT" true "CRITICAL: aborted/\"${1}\""
 }
 
 ###############################################################################
-# Wrap cylc__job_finish for job preempt/vacation signal trap.
+# Wrap cylc__job_finish_err for job preempt/vacation signal trap.
 cylc__job_vacation() {
-    cylc__job_finish "${1}" false "WARNING: vacated/${1}"
+    cylc__job_finish_err "${1}" false "WARNING: vacated/${1}"
 }
 
 ###############################################################################
-# Wrap cylc__job_finish for automatic job exit signal trap.
+# Wrap cylc__job_finish_err for automatic job exit signal trap.
 cylc__job_err() {
-    cylc__job_finish "${1}" true "CRITICAL: failed/${1}"
+    cylc__job_finish_err "${1}" true "CRITICAL: failed/${1}"
 }
 
 ###############################################################################

lib/cylc/job_file.py

@@ -296,9 +296,10 @@ def _write_global_init_script(cls, handle, job_conf):
     def _write_script(cls, handle, job_conf):
         """Write (*-)script in functions.
 
-        init-script, env-script, err-script, pre-script, script, post-script
+        init-script, env-script, err-script, pre-script, script, post-script,
+        exit-script
         """
-        for prefix in ['init-', 'env-', 'err-', 'pre-', '', 'post-']:
+        for prefix in ['init-', 'env-', 'err-', 'pre-', '', 'post-', 'exit-']:
             value = job_conf[prefix + 'script']
             if cls._check_script_value(value):
                 handle.write("\n\ncylc__job__inst__%sscript() {" % (

lib/cylc/task_job_mgr.py

@@ -849,6 +849,7 @@ def _prep_submit_task_job_impl(self, suite, itask, rtconfig):
             'execution_time_limit': itask.summary[self.KEY_EXECUTE_TIME_LIMIT],
             'env-script': rtconfig['env-script'],
             'err-script': rtconfig['err-script'],
+            'exit-script': rtconfig['exit-script'],
             'host': itask.task_host,
             'init-script': rtconfig['init-script'],
             'job_file_path': job_file_path,

tests/cylc-get-config/00-simple/section2.stdout

@@ -2,6 +2,7 @@
     script = echo "RUN: run-var.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 
@@ -82,6 +83,7 @@
     script = echo "RUN: run-ops.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 
@@ -161,6 +163,7 @@
     script = echo "RUN: run-ops.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 
@@ -241,6 +244,7 @@
     script = echo "RUN: run-ops.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 
@@ -321,6 +325,7 @@
     script = echo "RUN: run-ops.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 
@@ -401,6 +406,7 @@
     script = echo "RUN: run-ops.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 
@@ -481,6 +487,7 @@
     script = echo "RUN: run-var.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 
@@ -561,6 +568,7 @@
     script = echo "RUN: run-var.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 
@@ -640,6 +648,7 @@
     script = echo "RUN: run-var.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 
@@ -719,6 +728,7 @@
 [[SERIAL]]
     env-script = 
     err-script = 
+    exit-script = 
     script = 
     extra log files = 
     work sub-directory = 
@@ -799,6 +809,7 @@
 [[root]]
     env-script = 
     err-script = 
+    exit-script = 
     script = 
     extra log files = 
     work sub-directory = 
@@ -878,6 +889,7 @@
 [[PARALLEL]]
     env-script = 
     err-script = 
+    exit-script = 
     script = 
     extra log files = 
     work sub-directory = 
@@ -959,6 +971,7 @@
     script = echo "RUN: run-var.sh"
     env-script = 
     err-script = 
+    exit-script = 
     extra log files = 
     work sub-directory = 
     init-script = 

tests/jobscript/19-exit-script.t

@@ -0,0 +1,49 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2018 NIWA & British Crown (Met Office) & Contributors.
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+#------------------------------------------------------------------------------
+# Test exit-script.
+
+. "$(dirname "${0}")/test_header"
+set_test_number 9
+
+install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
+
+run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
+
+# 1) Should run on normal successful job exit.
+run_ok "${TEST_NAME_BASE}-run" \
+  cylc run --debug --no-detach "${SUITE_NAME}" 
+grep_ok 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out"
+grep_fail 'Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err"
+
+# 2) Should not run on internal early EXIT.
+run_fail "${TEST_NAME_BASE}-run" \
+  cylc run --debug --no-detach --set=EXIT=true "${SUITE_NAME}" 
+grep_fail 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out"
+grep_ok 'EXIT Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err"
+
+# 3) Should not run on external job TERM.
+cylc run --set=NSLEEP=30 "${SUITE_NAME}"
+sleep 3
+CYLC_JOB_PID=$(sed -n 's/^CYLC_JOB_PID=//p' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.status")
+kill -s TERM $CYLC_JOB_PID
+sleep 5
+grep_fail 'Cheesy peas!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.out"
+grep_ok 'TERM Oops!' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.err"
+
+purge_suite "${SUITE_NAME}"

tests/jobscript/19-exit-script/suite.rc

@@ -0,0 +1,30 @@
+#!Jinja2
+
+{% set NSLEEP = NSLEEP | default(1) %}
+{% set EXIT = EXIT | default("false") %}
+
+[cylc]
+    [[events]]
+        abort on stalled = True
+[scheduling]
+    [[dependencies]]
+        graph = foo
+[runtime]
+    [[foo]]
+        exit-script = echo "Cheesy peas!"
+        err-script = echo "$1 Oops!"
+        script = """
+echo "HELLO"
+if {{EXIT}}; then
+    exit 0
+fi
+# This must not be a single monolothic sleep. If a command is running when
+# bash receives a signal that a trap is set for, it waits for the command to
+# complete before executing the trap. We want a quick exit, to make it
+# obvious that the TERM signal had the intended effect.
+for I in $(seq 1 {{NSLEEP}}); do
+    echo $I
+    sleep 1
+done
+echo "BYE"
+        """