Remove configurable run/work dir from global config

CHANGES.md

@@ -49,13 +49,21 @@ ones in. -->
 -------------------------------------------------------------------------------
 ## __cylc-8.0b2 (<span actions:bind='release-date'>Released 2021-??-??</span>)__
 
+### Enhancements
+
+[#4177](https://github.com/cylc/cylc-flow/pull/4177) - Remove obsolete
+configuration items from `global.cylc[platforms][<platform name>]`:
+`run directory`, `work directory` and `suite definition directory`. This
+functionality is now provided by `[symlink dirs]`.
+
 ### Fixes
 
 [#4180](https://github.com/cylc/cylc-flow/pull/4180) - Fix bug where installing
 a workflow that uses the deprecated `suite.rc` filename would symlink `flow.cylc`
 to the `suite.rc` in the source dir instead of the run dir. Also fixes a couple
 of other, small bugs.
 
+
 -------------------------------------------------------------------------------
 ## __cylc-8.0b1 (<span actions:bind='release-date'>Released 2021-04-21</span>)__
 

cylc/flow/cfgspec/globalcfg.py

@@ -424,22 +424,6 @@
             ''')
             Conf('job runner command template', VDR.V_STRING)
             Conf('shell', VDR.V_STRING, '/bin/bash')
-            Conf('run directory', VDR.V_STRING, '$HOME/cylc-run', desc='''
-                The directory in which to install workflows.
-            ''')
-            Conf('work directory', VDR.V_STRING, '$HOME/cylc-run', desc='''
-                The top level for suite work and share directories. Can contain
-                ``$HOME`` or ``$USER`` but not other environment variables (the
-                item cannot actually be evaluated by the shell on HOST before
-                use, but the remote home directory is where ``rsync`` and
-                ``ssh`` naturally land, and the remote username is known by the
-                suite server program).
-
-                Example::
-
-                   /nfs/data/$USER/cylc-run
-            ''')
-            Conf('suite definition directory', VDR.V_STRING)
             Conf('communication method',
                  VDR.V_STRING, 'zmq',
                  options=[meth.value for meth in CommsMeth], desc='''
@@ -840,30 +824,25 @@ class GlobalConfig(ParsecConfig):
     CONF_BASENAME: str = "global.cylc"
     DEFAULT_SITE_CONF_PATH: str = os.path.join(os.sep, 'etc', 'cylc')
     USER_CONF_PATH: str = os.path.join(
-        os.getenv('HOME') or get_user_home(),
-        '.cylc',
-        'flow'
+        os.getenv('HOME') or get_user_home(), '.cylc', 'flow'
     )
     VERSION_HIERARCHY: List[str] = get_version_hierarchy(CYLC_VERSION)
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, **kwargs) -> None:
+        site_conf_root = (
+            os.getenv('CYLC_SITE_CONF_PATH') or self.DEFAULT_SITE_CONF_PATH
+        )
         self.conf_dir_hierarchy: List[Tuple[str, str]] = [
             *[
-                (
-                    upgrader.SITE_CONFIG,
-                    os.path.join(
-                        (
-                            os.getenv('CYLC_SITE_CONF_PATH')
-                            or self.DEFAULT_SITE_CONF_PATH
-                        ),
-                        'flow',
-                        ver
-                    )
-                )
+                (upgrader.SITE_CONFIG,
+                 os.path.join(site_conf_root, 'flow', ver))
                 for ver in self.VERSION_HIERARCHY
             ],
-            *[(upgrader.USER_CONFIG, os.path.join(self.USER_CONF_PATH, ver))
-              for ver in self.VERSION_HIERARCHY]
+            *[
+                (upgrader.USER_CONFIG,
+                 os.path.join(self.USER_CONF_PATH, ver))
+                for ver in self.VERSION_HIERARCHY
+            ]
         ]
         super().__init__(*args, **kwargs)
 

cylc/flow/cfgspec/suite.py

@@ -856,8 +856,7 @@
 
                 The top level share and work directory location can be changed
                 (e.g. to a large data area) by a global config setting (see
-                :cylc:conf:`global.cylc
-                [platforms][<platform name>]work directory`).
+                :cylc:conf:`global.cylc[symlink dirs]`).
 
                 .. note::
 

cylc/flow/dbstatecheck.py

@@ -19,6 +19,8 @@
 import os
 import sqlite3
 import sys
+
+from cylc.flow.pathutil import expand_path
 from cylc.flow.rundb import CylcSuiteDAO
 from cylc.flow.task_state import (
     TASK_STATUS_SUBMITTED,
@@ -55,9 +57,9 @@ class CylcSuiteDBChecker:
     }
 
     def __init__(self, rund, suite):
-        db_path = os.path.join(
-            os.path.expanduser(rund), suite, "log",
-            CylcSuiteDAO.DB_FILE_BASE_NAME)
+        db_path = expand_path(
+            rund, suite, "log", CylcSuiteDAO.DB_FILE_BASE_NAME
+        )
         if not os.path.exists(db_path):
             raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), db_path)
         self.conn = sqlite3.connect(db_path, timeout=10.0)

cylc/flow/etc/job.sh

@@ -71,9 +71,8 @@ cylc__job__main() {
     echo
     # Derived environment variables
     export CYLC_SUITE_LOG_DIR="${CYLC_SUITE_RUN_DIR}/log/suite"
-    CYLC_SUITE_WORK_DIR_ROOT="${CYLC_SUITE_WORK_DIR_ROOT:-${CYLC_SUITE_RUN_DIR}}"
-    export CYLC_SUITE_SHARE_DIR="${CYLC_SUITE_WORK_DIR_ROOT}/share"
-    export CYLC_SUITE_WORK_DIR="${CYLC_SUITE_WORK_DIR_ROOT}/work"
+    export CYLC_SUITE_SHARE_DIR="${CYLC_SUITE_RUN_DIR}/share"
+    export CYLC_SUITE_WORK_DIR="${CYLC_SUITE_RUN_DIR}/work"
     CYLC_TASK_CYCLE_POINT="$(cut -d '/' -f 1 <<<"${CYLC_TASK_JOB}")"
     CYLC_TASK_NAME="$(cut -d '/' -f 2 <<<"${CYLC_TASK_JOB}")"
     export CYLC_TASK_NAME CYLC_TASK_CYCLE_POINT ISODATETIMEREF

cylc/flow/etc/syntax/cylc.lang

@@ -105,7 +105,6 @@
         <keyword>task parameters</keyword>
         <keyword>task event batch interval</keyword>
         <keyword>suite state polling</keyword>
-        <keyword>suite definition directory</keyword>
         <keyword>succeeded handler</keyword>
         <keyword>submitted handler</keyword>
         <keyword>submission timeout handler</keyword>

cylc/flow/etc/syntax/cylc.xml

@@ -31,7 +31,6 @@
         <RegExpr attribute='Keyword' String=' task parameters '/>
         <RegExpr attribute='Keyword' String=' task event batch interval '/>
         <RegExpr attribute='Keyword' String=' suite state polling '/>
-        <RegExpr attribute='Keyword' String=' suite definition directory '/>
         <RegExpr attribute='Keyword' String=' succeeded handler '/>
         <RegExpr attribute='Keyword' String=' submitted handler '/>
         <RegExpr attribute='Keyword' String=' submission timeout handler '/>

cylc/flow/job_file.py

@@ -24,9 +24,7 @@
 from cylc.flow import __version__ as CYLC_VERSION
 from cylc.flow.job_runner_mgr import JobRunnerManager
 import cylc.flow.flags
-from cylc.flow.pathutil import (
-    get_remote_suite_run_dir,
-    get_remote_suite_work_dir)
+from cylc.flow.pathutil import get_remote_suite_run_dir
 from cylc.flow.config import interpolate_template, ParamExpandError
 
 
@@ -56,9 +54,8 @@ def write(self, local_job_file_path, job_conf, check_syntax=True):
         # Access to cylc must be configured before user environment so
         # that cylc commands can be used in defining user environment
         # variables: NEXT_CYCLE=$( cylc cycle-point --offset-hours=6 )
-        platform = job_conf['platform']
         tmp_name = os.path.expandvars(local_job_file_path + '.tmp')
-        run_d = get_remote_suite_run_dir(platform, job_conf['suite_name'])
+        run_d = get_remote_suite_run_dir(job_conf['suite_name'])
         try:
             with open(tmp_name, 'w') as handle:
                 self._write_header(handle, job_conf)
@@ -201,14 +198,7 @@ def _write_suite_environment(self, handle, job_conf, run_d):
 
         handle.write('\n')
         # override and write task-host-specific suite variables
-        work_d = get_remote_suite_work_dir(
-            job_conf["platform"], job_conf['suite_name'])
         handle.write('\n    export CYLC_SUITE_RUN_DIR="%s"' % run_d)
-        if work_d != run_d:
-            # Note: not an environment variable, but used by job.sh
-            handle.write(
-                '\n    export CYLC_SUITE_WORK_DIR_ROOT="%s"' % work_d
-            )
         handle.write(
             '\n    export CYLC_SUITE_UUID="%s"' % job_conf['uuid_str'])
 

cylc/flow/pathutil.py

@@ -24,10 +24,12 @@
 from cylc.flow import LOG
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.exceptions import WorkflowFilesError
-from cylc.flow.platforms import (
-    get_localhost_install_target,
-    platform_from_name
-)
+from cylc.flow.platforms import get_localhost_install_target
+
+
+# Note: do not import this elsewhere, as it might bypass unit test
+# monkeypatching:
+_CYLC_RUN_DIR = '$HOME/cylc-run'
 
 
 def expand_path(*args: Union[Path, str]) -> str:
@@ -37,23 +39,20 @@ def expand_path(*args: Union[Path, str]) -> str:
     ))
 
 
-def get_remote_suite_run_dir(platform, suite, *args):
-    """Return remote suite run directory, join any extra args."""
-    return os.path.join(
-        platform['run directory'], suite, *args)
-
-
-def get_remote_suite_run_job_dir(platform, suite, *args):
-    """Return remote suite run directory, join any extra args."""
-    return get_remote_suite_run_dir(
-        platform, suite, 'log', 'job', *args)
+def get_remote_suite_run_dir(
+    flow_name: Union[Path, str], *args: Union[Path, str]
+) -> str:
+    """Return remote workflow run directory, joining any extra args,
+    NOT expanding vars or user."""
+    return os.path.join(_CYLC_RUN_DIR, flow_name, *args)
 
 
-def get_remote_suite_work_dir(platform, suite, *args):
-    """Return remote suite work directory root, join any extra args."""
-    return os.path.join(
-        platform['work directory'], suite, *args
-    )
+def get_remote_suite_run_job_dir(
+    flow_name: Union[Path, str], *args: Union[Path, str]
+) -> str:
+    """Return remote workflow job log directory, joining any extra args,
+    NOT expanding vars or user."""
+    return get_remote_suite_run_dir(flow_name, 'log', 'job', *args)
 
 
 def get_workflow_run_dir(
@@ -64,11 +63,7 @@ def get_workflow_run_dir(
 
     Does not check that the directory exists.
     """
-    return expand_path(
-        os.path.join(
-            platform_from_name()['run directory'], flow_name, *args
-        )
-    )
+    return expand_path(_CYLC_RUN_DIR, flow_name, *args)
 
 
 def get_suite_run_job_dir(suite, *args):
@@ -103,16 +98,12 @@ def get_suite_run_pub_db_name(suite):
 
 def get_suite_run_share_dir(suite, *args):
     """Return local suite work/share directory, join any extra args."""
-    return expand_path(os.path.join(
-        platform_from_name()['work directory'], suite, 'share', *args
-    ))
+    return get_workflow_run_dir(suite, 'share', *args)
 
 
 def get_suite_run_work_dir(suite, *args):
     """Return local suite work/work directory, join any extra args."""
-    return expand_path(os.path.join(
-        platform_from_name()['work directory'], suite, 'work', *args
-    ))
+    return get_workflow_run_dir(suite, 'work', *args)
 
 
 def get_suite_test_log_name(suite):
@@ -122,8 +113,6 @@ def get_suite_test_log_name(suite):
 
 def make_suite_run_tree(suite):
     """Create all top-level cylc-run output dirs on the suite host."""
-    dir_ = get_workflow_run_dir(suite)
-    # Create
     for dir_ in (
         get_workflow_run_dir(suite),
         get_suite_run_log_dir(suite),
@@ -156,7 +145,7 @@ def make_localhost_symlinks(rund, named_sub_dir):
             dst = rund
         else:
             dst = os.path.join(rund, key)
-        src = os.path.expandvars(value)
+        src = expand_path(value)
         if '$' in src:
             raise WorkflowFilesError(
                 f'Unable to create symlink to {src}.'

cylc/flow/scripts/cat_log.py

@@ -423,7 +423,6 @@ def main(parser, options, *args, color=False):
                             and live_job_id is None)
         if log_is_remote and (not log_is_retrieved or options.force_remote):
             logpath = os.path.normpath(get_remote_suite_run_job_dir(
-                platform,
                 suite_name, point, task, options.submit_num, options.filename))
             tail_tmpl = platform["tail command template"]
             # Reinvoke the cat-log command on the remote account.

cylc/flow/scripts/diff.py

@@ -26,7 +26,7 @@
 after inlining has occurred.
 
 Files in the suite bin directory and other sub-directories of the
-suite definition directory are not currently differenced."""
+run directory are not currently differenced."""
 
 import sys