job-script: run the global init-script before sourcing the job script

* Closes #4520
* Allows jobs to be run on machienes which don't have a home directory
  by setting $HOME in the global init-script.
* This comes with some caveats:
  * The global init-script will no longer be covered by error trapping.
  * The job environment will no longer be available to the global init-script.
  * In debug mode the global init-script will not be included in xtrace output.

CHANGES.md

@@ -64,6 +64,9 @@ This only affects you if you have used a prior Cylc 8 pre-release.
 now configurable in `global.cylc[install]max depth`, and `cylc install` will
 fail if the workflow ID would exceed this depth.
 
+[#4534](https://github.com/cylc/cylc-flow/pull/4534)
+- Permit jobs to be run on platforms with no $HOME directory.
+
 [#4536](https://github.com/cylc/cylc-flow/pull/4536) - `cylc extract-resources`
 renamed `cylc get-resources` and small changes made:
 - Cylc wrapper script made available.

cylc/flow/cfgspec/globalcfg.py

@@ -1012,12 +1012,25 @@
                    the wrapper script.
             ''')
             Conf('global init-script', VDR.V_STRING, desc='''
-                Add a script before the init-script of all jobs on this
-                platform.
+                A per-platform script which is run before other job scrips.
+
+                This should be used sparingly to perform any shell
+                configuration that cannot be performed via other means.
+
+                .. versionchanged:: 8.0.0
+
+                   The ``global init-script`` now runs *before* any job
+                   scripting which introduces caveats outlined below.
+
+                .. warning::
+
+                   The ``global init-script`` has the following caveats,
+                   as compared to the other task ``script-*`` items:
 
-                If specified, the value of this setting will be inserted to
-                just before the ``init-script`` section of all job scripts that
-                are to be submitted to the specified platform.
+                   * The script is not covered by error trapping.
+                   * The job environment is not available to this script.
+                   * In debug mode this script will not be included in
+                     xtrace output.
             ''')
             Conf('copyable environment variables', VDR.V_STRING_LIST, '',
                  desc='''

cylc/flow/etc/job.sh

@@ -31,14 +31,13 @@ cylc__job__main() {
     if "${CYLC_DEBUG:-false}"; then
         if [[ -n "${BASH:-}" ]]; then
             PS4='+[\D{%Y%m%dT%H%M%S%z}]\u@\h '
-            exec 19>>"${CYLC_WORKFLOW_RUN_DIR}/log/job/${CYLC_TASK_JOB}/job.xtrace"
+            exec 19>>"${CYLC_RUN_DIR}/${CYLC_WORKFLOW_ID}/log/job/${CYLC_TASK_JOB}/job.xtrace"
             export BASH_XTRACEFD=19
             >&2 echo "Sending DEBUG MODE xtrace to job.xtrace"
         fi
         set -x
     fi
     # Init-Script
-    cylc__job__run_inst_func 'global_init_script'
     cylc__job__run_inst_func 'init_script'
     # Start error and vacation traps
     typeset signal_name=
@@ -70,6 +69,7 @@ cylc__job__main() {
     echo "User@Host: ${USER}@${host}"
     echo
     # Derived environment variables
+    export CYLC_WORKFLOW_RUN_DIR="${CYLC_RUN_DIR}/${CYLC_WORKFLOW_ID}"
     export CYLC_WORKFLOW_LOG_DIR="${CYLC_WORKFLOW_RUN_DIR}/log/workflow"
     export CYLC_WORKFLOW_SHARE_DIR="${CYLC_WORKFLOW_RUN_DIR}/share"
     export CYLC_WORKFLOW_WORK_DIR="${CYLC_WORKFLOW_RUN_DIR}/work"

cylc/flow/job_file.py

@@ -26,7 +26,6 @@
 from cylc.flow.job_runner_mgr import JobRunnerManager
 import cylc.flow.flags
 from cylc.flow.option_parsers import verbosity_to_env
-from cylc.flow.pathutil import get_remote_workflow_run_dir
 from cylc.flow.config import interpolate_template, ParamExpandError
 
 
@@ -57,22 +56,21 @@ def write(self, local_job_file_path, job_conf, check_syntax=True):
         # that cylc commands can be used in defining user environment
         # variables: NEXT_CYCLE=$( cylc cycle-point --offset-hours=6 )
         tmp_name = os.path.expandvars(local_job_file_path + '.tmp')
-        run_d = get_remote_workflow_run_dir(job_conf['workflow_name'])
         try:
             with open(tmp_name, 'w') as handle:
                 self._write_header(handle, job_conf)
                 self._write_directives(handle, job_conf)
                 self._write_reinvocation(handle)
                 self._write_prelude(handle, job_conf)
-                self._write_workflow_environment(handle, job_conf, run_d)
+                self._write_workflow_environment(handle, job_conf)
                 self._write_task_environment(handle, job_conf)
-                self._write_global_init_script(handle, job_conf)
                 # workflow bin access must be before runtime environment
                 # because workflow bin commands may be used in variable
                 # assignment expressions: FOO=$(command args).
                 self._write_runtime_environment(handle, job_conf)
                 self._write_script(handle, job_conf)
-                self._write_epilogue(handle, job_conf, run_d)
+                self._write_global_init_script(handle, job_conf)
+                self._write_epilogue(handle, job_conf)
         except IOError as exc:
             # Remove temporary file
             with suppress(OSError):
@@ -184,6 +182,10 @@ def _write_prelude(self, handle, job_conf):
         else:
             handle.write(
                 "\nexport CYLC_ENV_NAME='%s'" % CYLC_ENV_NAME)
+        handle.write(
+            '\nexport CYLC_WORKFLOW_ID='
+            f'"{job_conf["workflow_name"]}"'
+        )
         env_vars = (
             (job_conf['platform']['copyable environment variables'] or [])
             # pass CYLC_COVERAGE into the job execution environment
@@ -193,21 +195,18 @@ def _write_prelude(self, handle, job_conf):
             if key in os.environ:
                 handle.write("\nexport %s='%s'" % (key, os.environ[key]))
 
-    def _write_workflow_environment(self, handle, job_conf, run_d):
+    def _write_workflow_environment(self, handle, job_conf):
         """Workflow and task environment."""
         handle.write("\n\ncylc__job__inst__cylc_env() {")
         handle.write("\n    # CYLC WORKFLOW ENVIRONMENT:")
         # write the static workflow variables
         for var, val in sorted(self.workflow_env.items()):
-            if var not in ('CYLC_DEBUG', 'CYLC_VERBOSE'):
+            if var not in ('CYLC_DEBUG', 'CYLC_VERBOSE', 'CYLC_WORKFLOW_ID'):
                 handle.write('\n    export %s="%s"' % (var, val))
 
         if str(self.workflow_env.get('CYLC_UTC')) == 'True':
             handle.write('\n    export TZ="UTC"')
 
-        handle.write('\n')
-        # override and write task-host-specific workflow variables
-        handle.write('\n    export CYLC_WORKFLOW_RUN_DIR="%s"' % run_d)
         handle.write(
             '\n    export CYLC_WORKFLOW_UUID="%s"' % job_conf['uuid_str'])
 
@@ -314,10 +313,8 @@ def _write_global_init_script(cls, handle, job_conf):
         """Global Init-script."""
         global_init_script = job_conf['platform']['global init-script']
         if cls._check_script_value(global_init_script):
-            handle.write("\n\ncylc__job__inst__global_init_script() {")
-            handle.write("\n# GLOBAL-INIT-SCRIPT:\n")
+            handle.write("\n\n# GLOBAL INIT-SCRIPT:\n")
             handle.write(global_init_script)
-            handle.write("\n}")
 
     @classmethod
     def _write_script(cls, handle, job_conf):
@@ -336,8 +333,13 @@ def _write_script(cls, handle, job_conf):
                 handle.write("\n}")
 
     @staticmethod
-    def _write_epilogue(handle, job_conf, run_d):
+    def _write_epilogue(handle, job_conf):
         """Write epilogue."""
-        handle.write(f'\n\n. "{run_d}/.service/etc/job.sh"\ncylc__job__main')
+        handle.write('\n\nCYLC_RUN_DIR="${CYLC_RUN_DIR:-$HOME/cylc-run}"')
+        handle.write(
+            '\n. '
+            '"${CYLC_RUN_DIR}/${CYLC_WORKFLOW_ID}/.service/etc/job.sh"'
+            '\ncylc__job__main'
+        )
         handle.write("\n\n%s%s\n" % (
             JobRunnerManager.LINE_PREFIX_EOF, job_conf['job_d']))

tests/functional/jinja2/11-logging.t

@@ -35,4 +35,5 @@ TEST_NAME="${TEST_NAME_BASE}-validate"
 run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" --debug
 grep_ok 'Hello World!' "${TEST_NAME}.stderr"  # debug lines go to stderr
 
+purge
 exit

tests/unit/test_job_file.py

@@ -17,14 +17,18 @@
 # Tests for functions contained in cylc.flow.job_file.
 # TODO remove the unittest dependency - it should not be necessary.
 
+from contextlib import suppress
 import io
 import os
+from pathlib import Path
 import pytest
-from contextlib import suppress
 from tempfile import NamedTemporaryFile
-from unittest import mock
+from textwrap import dedent
 
-from cylc.flow import __version__
+from cylc.flow import (
+    __version__,
+    __file__ as cylc_flow_file,
+)
 from cylc.flow.job_file import JobFileWriter
 from cylc.flow.platforms import platform_from_name
 
@@ -67,8 +71,7 @@ def inner_func(custom_settings=None):
     yield inner_func
 
 
-@mock.patch("cylc.flow.job_file.get_remote_workflow_run_dir")
-def test_write(mocked_get_remote_workflow_run_dir, fixture_get_platform):
+def test_write(fixture_get_platform):
     """Test write function outputs jobscript file correctly."""
     with NamedTemporaryFile() as local_job_file_path:
         local_job_file_path = local_job_file_path.name
@@ -103,7 +106,6 @@ def test_write(mocked_get_remote_workflow_run_dir, fixture_get_platform):
             "post-script": "This is the post script",
             "exit-script": "This is the exit script",
         }
-        mocked_get_remote_workflow_run_dir.return_value = "run/dir"
         JobFileWriter().write(local_job_file_path, job_conf)
 
         assert (os.path.exists(local_job_file_path))
@@ -237,7 +239,8 @@ def test_traps_for_each_job_runner(job_runner: str):
     })
     job_conf = {
         "platform": platform,
-        "directives": {}
+        "directives": {},
+        "workflow_name": 'test_traps_for_each_job_runner',
     }
 
     with io.StringIO() as fake_file:
@@ -271,8 +274,10 @@ def test_write_prelude(monkeypatch, fixture_get_platform, set_CYLC_ENV_NAME):
                 f'\nexport CYLC_VERSION=\'{__version__}\'')
     if set_CYLC_ENV_NAME:
         expected += '\nexport CYLC_ENV_NAME=\'myenv\''
+    expected += '\nexport CYLC_WORKFLOW_ID="test_write_prelude"'
     expected += '\nexport CYLC_WORKFLOW_INITIAL_CYCLE_POINT=\'20200101T0000Z\''
     job_conf = {
+        "workflow_name": "test_write_prelude",
         "platform": fixture_get_platform({
             "job runner": "loadleveler",
             "job runner command template": "test_workflow",
@@ -303,8 +308,8 @@ def test_write_workflow_environment(fixture_get_platform, monkeypatch):
     # workflow env not correctly setting...check this
     expected = ('\n\ncylc__job__inst__cylc_env() {\n    # CYLC WORKFLOW '
                 'ENVIRONMENT:\n    export CYLC_CYCLING_MODE="integer"\n  '
-                '  export CYLC_UTC="True"\n    export TZ="UTC"\n\n   '
-                ' export CYLC_WORKFLOW_RUN_DIR="cylc-run/farm_noises"\n   '
+                '  export CYLC_UTC="True"\n    export TZ="UTC"'
+                '\n   '
                 ' export CYLC_WORKFLOW_UUID="neigh"')
     job_conf = {
         "platform": fixture_get_platform({
@@ -313,9 +318,8 @@ def test_write_workflow_environment(fixture_get_platform, monkeypatch):
         "workflow_name": "farm_noises",
         "uuid_str": "neigh"
     }
-    rund = "cylc-run/farm_noises"
     with io.StringIO() as fake_file:
-        job_file_writer._write_workflow_environment(fake_file, job_conf, rund)
+        job_file_writer._write_workflow_environment(fake_file, job_conf)
         result = fake_file.getvalue()
         assert result == expected
 
@@ -343,6 +347,7 @@ def test_write_script():
         "script": "This is the script",
         "post-script": "This is the post script",
         "exit-script": "This is the exit script",
+        "workflow_name": "test_write_script",
     }
 
     with io.StringIO() as fake_file:
@@ -362,7 +367,8 @@ def test_no_script_section_with_comment_only_script():
         "pre-script": "#This is the pre script/n #moo /n#baa",
         "script": "",
         "post-script": "",
-        "exit-script": ""
+        "exit-script": "",
+        "workflow_name": "test_no_script_section_with_comment_only_script"
     }
 
     with io.StringIO() as fake_file:
@@ -395,7 +401,8 @@ def test_write_task_environment():
         "flow_nums": {1},
         "param_var": {"duck": "quack",
                       "mouse": "squeak"},
-        "work_d": "farm_noises/work_d"
+        "work_d": "farm_noises/work_d",
+        "workflow_name": "test_write_task_environment",
     }
     with io.StringIO() as fake_file:
         JobFileWriter()._write_task_environment(fake_file, job_conf)
@@ -411,9 +418,12 @@ def test_write_runtime_environment():
         '    cow=~/"moo"\n    sheep=~baa/"baa"\n    '
         'duck=~quack\n}')
     job_conf = {
-        'environment': {'cow': '~/moo',
-                        'sheep': '~baa/baa',
-                        'duck': '~quack'}
+        'environment': {
+            'cow': '~/moo',
+            'sheep': '~baa/baa',
+            'duck': '~quack'
+        },
+        "workflow_name": "test_write_runtime_environment",
     }
     with io.StringIO() as fake_file:
         JobFileWriter()._write_runtime_environment(fake_file, job_conf)
@@ -422,13 +432,16 @@ def test_write_runtime_environment():
 
 def test_write_epilogue():
     """Test epilogue is correctly written in jobscript"""
+    expected = '\n' + dedent('''
+        CYLC_RUN_DIR="${CYLC_RUN_DIR:-$HOME/cylc-run}"
+        . "${CYLC_RUN_DIR}/${CYLC_WORKFLOW_ID}/.service/etc/job.sh"
+        cylc__job__main
 
-    expected = ('\n\n. \"cylc-run/farm_noises/.service/etc/job.sh\"\n'
-                'cylc__job__main\n\n#EOF: 1/moo/01\n')
+        #EOF: 1/moo/01
+    ''')
     job_conf = {'job_d': "1/moo/01"}
-    run_d = "cylc-run/farm_noises"
     with io.StringIO() as fake_file:
-        JobFileWriter()._write_epilogue(fake_file, job_conf, run_d)
+        JobFileWriter()._write_epilogue(fake_file, job_conf)
         assert(fake_file.getvalue() == expected)
 
 
@@ -438,16 +451,76 @@ def test_write_global_init_scripts(fixture_get_platform):
     job_conf = {
         "platform": fixture_get_platform({
             "global init-script": (
-                'global init-script = \n'
                 'export COW=moo\n'
                 'export PIG=oink\n'
                 'export DONKEY=HEEHAW\n'
             )
         })
     }
-    expected = ('\n\ncylc__job__inst__global_init_script() {\n'
-                '# GLOBAL-INIT-SCRIPT:\nglobal init-script = \nexport '
-                'COW=moo\nexport PIG=oink\nexport DONKEY=HEEHAW\n\n}')
+    expected = '\n' + dedent('''
+        # GLOBAL INIT-SCRIPT:
+        export COW=moo
+        export PIG=oink
+        export DONKEY=HEEHAW
+    ''')
     with io.StringIO() as fake_file:
         JobFileWriter()._write_global_init_script(fake_file, job_conf)
         assert(fake_file.getvalue() == expected)
+
+
+def test_homeless_platform(fixture_get_platform):
+    """Ensure there are no uses of $HOME before the global init-script.
+
+    This is to allow users to configure a $HOME on machines with no $HOME
+    directory.
+    """
+    job_conf = {
+        "platform": fixture_get_platform({
+            'global init-script': 'some-script'
+        }),
+        "task_id": "a",
+        "workflow_name": "b",
+        "work_d": "c/d",
+        "uuid_str": "e",
+        'environment': {},
+        'cow': '~/moo',
+        "job_d": "1/a/01",
+        "try_num": 1,
+        "flow_nums": {1},
+        # "job_runner_name": "background",
+        "param_var": {},
+        "execution_time_limit": None,
+        "namespace_hierarchy": [],
+        "dependencies": [],
+        "init-script": "",
+        "env-script": "",
+        "err-script": "",
+        "pre-script": "",
+        "script": "",
+        "post-script": "",
+        "exit-script": "",
+    }
+
+    with NamedTemporaryFile() as local_job_file_path:
+        local_job_file_path = local_job_file_path.name
+        JobFileWriter().write(local_job_file_path, job_conf)
+        with open(local_job_file_path, 'r') as local_job_file:
+            job_script = local_job_file.read()
+
+    # ensure that $HOME is not used before the global init-script
+    for line in job_script.splitlines():
+        if line.startswith(' '):
+            # ignore env/script functions which aren't run until later
+            continue
+        if line == '# GLOBAL INIT-SCRIPT:':
+            # quit once we've hit the global init-script
+            break
+        if 'HOME' in line:
+            # bail if $HOME is used
+            raise Exception(f'$HOME found in {line}\n{job_script}')
+
+    # also ensure there is no use of $HOME in the job.sh script
+    with open(Path(cylc_flow_file).parent / 'etc/job.sh', 'r') as job_sh:
+        job_sh_txt = job_sh.read()
+        if 'HOME' in job_sh_txt:
+            raise Exception('$HOME found in job.sh\n{job_sh_txt}')