diff --git a/bin/cylc-cat-log b/bin/cylc-cat-log
index 3f9332f3088..c3cc3c535d6 100755
--- a/bin/cylc-cat-log
+++ b/bin/cylc-cat-log
@@ -53,15 +53,20 @@ import shlex
from glob import glob
from shlex import quote
from stat import S_IRUSR
-from tempfile import mkstemp
+from tempfile import NamedTemporaryFile
from subprocess import Popen, PIPE
+from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
from cylc.flow.exceptions import UserInputError
import cylc.flow.flags
+from cylc.flow.hostuserutil import is_remote
from cylc.flow.option_parsers import CylcOptionParser as COP
+from cylc.flow.pathutil import (
+ get_remote_suite_run_job_dir,
+ get_suite_run_job_dir,
+ get_suite_run_log_name,
+ get_suite_run_pub_db_name)
from cylc.flow.rundb import CylcSuiteDAO
-from cylc.flow.hostuserutil import is_remote
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
from cylc.flow.task_id import TaskID
from cylc.flow.task_job_logs import (
JOB_LOG_OUT, JOB_LOG_ERR, JOB_LOG_OPTS, NN, JOB_LOGS_LOCAL)
@@ -149,14 +154,14 @@ def view_log(logpath, mode, tailer_tmpl, batchview_cmd=None, remote=False):
elif not remote and mode == 'edit':
# Copy the log to a temporary read-only file for viewing in editor.
# Copy only BUFSIZE bytes at time, in case the file is huge.
- outfile = mkstemp(dir=glbl_cfg().get_tmpdir())[1]
+ outfile = NamedTemporaryFile()
with open(logpath, 'rb') as log:
- with open(outfile, 'wb') as out:
+ data = log.read(BUFSIZE)
+ while data:
+ outfile.write(data)
data = log.read(BUFSIZE)
- while data:
- out.write(data)
- data = log.read(BUFSIZE)
- os.chmod(outfile, S_IRUSR)
+ os.chmod(outfile.name, S_IRUSR)
+ outfile.seek(0, 0)
return outfile
elif mode == 'cat' or (remote and mode == 'edit'):
# Just cat file contents to stdout.
@@ -234,11 +239,7 @@ def get_task_job_attrs(suite_name, point, task, submit_num):
"""
suite_dao = CylcSuiteDAO(
- os.path.join(
- glbl_cfg().get_derived_host_item(
- suite_name, "suite run directory"),
- "log", CylcSuiteDAO.DB_FILE_BASE_NAME),
- is_public=True)
+ get_suite_run_pub_db_name(suite_name), is_public=True)
task_job_data = suite_dao.select_task_job(point, task, submit_num)
suite_dao.close()
if task_job_data is None:
@@ -264,17 +265,17 @@ def tmpfile_edit(tmpfile, geditor=False):
editor = glbl_cfg().get(['editors', 'gui'])
else:
editor = glbl_cfg().get(['editors', 'terminal'])
- modtime1 = os.stat(tmpfile).st_mtime
+ modtime1 = os.stat(tmpfile.name).st_mtime
cmd = shlex.split(editor)
- cmd.append(tmpfile)
+ cmd.append(tmpfile.name)
proc = Popen(cmd, stderr=PIPE)
err = proc.communicate()[1].decode()
ret_code = proc.wait()
if ret_code == 0:
- if os.stat(tmpfile).st_mtime > modtime1:
+ if os.stat(tmpfile.name).st_mtime > modtime1:
sys.stderr.write(
'WARNING: you edited a TEMPORARY COPY of %s\n' % (
- os.path.basename(tmpfile)))
+ os.path.basename(tmpfile.name)))
if ret_code and err:
sys.stderr.write(err)
@@ -317,7 +318,7 @@ def main():
if len(args) == 1:
# Cat suite logs, local only.
- logpath = glbl_cfg().get_derived_host_item(suite_name, "suite log")
+ logpath = get_suite_run_log_name(suite_name)
if options.rotation_num:
logs = glob('%s.*' % logpath)
logs.sort(key=os.path.getmtime, reverse=True)
@@ -392,10 +393,9 @@ def main():
log_is_retrieved = (glbl_cfg().get_host_item('retrieve job logs', host)
and live_job_id is None)
if log_is_remote and (not log_is_retrieved or options.force_remote):
- logpath = os.path.normpath(os.path.join(
- glbl_cfg().get_derived_host_item(
- suite_name, "suite job log directory", host, user),
- point, task, options.submit_num, options.filename))
+ logpath = os.path.normpath(get_remote_suite_run_job_dir(
+ host, user,
+ suite_name, point, task, options.submit_num, options.filename))
tail_tmpl = str(glbl_cfg().get_host_item(
"tail command template", host, user))
# Reinvoke the cat-log command on the remote account.
@@ -407,31 +407,29 @@ def main():
if batchview_cmd:
cmd.append('--remote-arg=%s' % quote(batchview_cmd))
cmd.append(suite_name)
- capture = (mode == 'edit')
+ is_edit_mode = (mode == 'edit')
try:
proc = remote_cylc_cmd(
- cmd, user, host, capture_process=capture,
+ cmd, user, host, capture_process=is_edit_mode,
manage=(mode == 'tail'))
except KeyboardInterrupt:
# Ctrl-C while tailing.
pass
else:
- if capture:
+ if is_edit_mode:
# Write remote stdout to a temp file for viewing in editor.
# Only BUFSIZE bytes at a time in case huge stdout volume.
- out = mkstemp(dir=glbl_cfg().get_tmpdir())[1]
- with open(out, 'wb') as outf:
+ out = NamedTemporaryFile()
+ data = proc.stdout.read(BUFSIZE)
+ while data:
+ out.write(data)
data = proc.stdout.read(BUFSIZE)
- while data:
- outf.write(data)
- data = proc.stdout.read(BUFSIZE)
- os.chmod(out, S_IRUSR)
+ os.chmod(out.name, S_IRUSR)
+ out.seek(0, 0)
else:
# Local task job or local job log.
- logpath = os.path.normpath(os.path.join(
- glbl_cfg().get_derived_host_item(
- suite_name, "suite job log directory"),
- point, task, options.submit_num, options.filename))
+ logpath = os.path.normpath(get_suite_run_job_dir(
+ suite_name, point, task, options.submit_num, options.filename))
tail_tmpl = str(glbl_cfg().get_host_item("tail command template"))
out = view_log(logpath, mode, tail_tmpl, batchview_cmd)
if mode != 'edit':
diff --git a/bin/cylc-ls-checkpoints b/bin/cylc-ls-checkpoints
index 1562f6fcc4f..03fd85c26aa 100755
--- a/bin/cylc-ls-checkpoints
+++ b/bin/cylc-ls-checkpoints
@@ -30,8 +30,8 @@ if remrun():
import os
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
from cylc.flow.option_parsers import CylcOptionParser as COP
+from cylc.flow.pathutil import get_suite_run_pub_db_name
from cylc.flow.rundb import CylcSuiteDAO
from cylc.flow.terminal import cli_function
@@ -98,12 +98,7 @@ def list_checkpoints(suite, callback):
def _get_dao(suite):
"""Return the DAO (public) for suite."""
-
- suite_log_dir = glbl_cfg().get_derived_host_item(
- suite, 'suite log directory')
- pub_db_path = os.path.join(os.path.dirname(suite_log_dir),
- CylcSuiteDAO.DB_FILE_BASE_NAME)
- return CylcSuiteDAO(pub_db_path, is_public=True)
+ return CylcSuiteDAO(get_suite_run_pub_db_name(suite), is_public=True)
def _write_row(title, row_idx, row):
diff --git a/bin/cylc-report-timings b/bin/cylc-report-timings
index 2653cff3c75..eaccbc02e55 100755
--- a/bin/cylc-report-timings
+++ b/bin/cylc-report-timings
@@ -58,8 +58,8 @@ import contextlib
import os
from cylc.flow.exceptions import CylcError
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
from cylc.flow.option_parsers import CylcOptionParser as COP
+from cylc.flow.pathutil import get_suite_run_pub_db_name
from cylc.flow.rundb import CylcSuiteDAO
from cylc.flow.terminal import cli_function
@@ -160,14 +160,7 @@ def format_rows(header, rows):
def _get_dao(suite):
"""Return the DAO (public) for suite."""
- suite_log_dir = glbl_cfg().get_derived_host_item(
- suite, 'suite log directory'
- )
- pub_db_path = os.path.join(
- os.path.dirname(suite_log_dir),
- CylcSuiteDAO.DB_FILE_BASE_NAME
- )
- return CylcSuiteDAO(pub_db_path, is_public=True)
+ return CylcSuiteDAO(get_suite_run_pub_db_name(suite), is_public=True)
class TimingSummary(object):
diff --git a/bin/cylc-submit b/bin/cylc-submit
index 443aa8e99e3..b798b1b0b85 100755
--- a/bin/cylc-submit
+++ b/bin/cylc-submit
@@ -39,13 +39,13 @@ import os
from time import sleep
from cylc.flow import LOG
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
from cylc.flow.config import SuiteConfig
from cylc.flow.cycling.loader import get_point
from cylc.flow.exceptions import UserInputError
import cylc.flow.flags
from cylc.flow.subprocpool import SubProcPool
from cylc.flow.option_parsers import CylcOptionParser as COP
+from cylc.flow.pathutil import make_suite_run_tree
from cylc.flow.suite_db_mgr import SuiteDatabaseManager
from cylc.flow.broadcast_mgr import BroadcastMgr
from cylc.flow.suite_srv_files_mgr import SuiteSrvFilesManager
@@ -106,7 +106,7 @@ def main():
taskdef, get_point(point_str).standardise(), is_startup=True))
# Initialise job submit environment
- glbl_cfg().create_cylc_run_tree(suite)
+ make_suite_run_tree(suite)
pool = SubProcPool()
db_mgr = SuiteDatabaseManager()
task_job_mgr = TaskJobManager(
diff --git a/bin/cylc-view b/bin/cylc-view
index 2f5da7eaad1..63175785b9d 100755
--- a/bin/cylc-view
+++ b/bin/cylc-view
@@ -42,10 +42,10 @@ from subprocess import call
from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
from cylc.flow.option_parsers import CylcOptionParser as COP
+from cylc.flow.parsec.fileparse import read_and_proc
from cylc.flow.suite_srv_files_mgr import SuiteSrvFilesManager
from cylc.flow.templatevars import load_template_vars
from cylc.flow.terminal import cli_function
-from cylc.flow.parsec.fileparse import read_and_proc
def parse_args():
@@ -119,7 +119,6 @@ def main():
options, args = parse_args()
suite, suiterc = SuiteSrvFilesManager().parse_suite_arg(options, args[0])
- cylc_tmpdir = glbl_cfg().get_tmpdir()
if options.geditor:
editor = glbl_cfg().get(['editors', 'gui'])
else:
@@ -148,7 +147,6 @@ def main():
# write to a temporary file
viewfile = NamedTemporaryFile(
suffix=".suite.rc", prefix=suite.replace('/', '_') + '.',
- dir=cylc_tmpdir
)
for line in lines:
viewfile.write((line + '\n').encode())
diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
index 0ecb5236a2b..c8157cacab9 100644
--- a/cylc/flow/cfgspec/globalcfg.py
+++ b/cylc/flow/cfgspec/globalcfg.py
@@ -17,15 +17,11 @@
# along with this program. If not, see .
"""Cylc site and user configuration file spec."""
-import atexit
import os
import re
-import shutil
-from tempfile import mkdtemp
from cylc.flow import LOG
from cylc.flow import __version__ as CYLC_VERSION
-from cylc.flow.exceptions import GlobalConfigError
from cylc.flow.hostuserutil import get_user_home, is_remote_user
from cylc.flow.network import Priv
from cylc.flow.parsec.config import ParsecConfig
@@ -41,21 +37,21 @@
# - default: the default value (optional).
# - allowed_2, ...: the only other allowed values of this setting (optional).
SPEC = {
+ # suite
'process pool size': [VDR.V_INTEGER, 4],
'process pool timeout': [VDR.V_INTERVAL, DurationFloat(600)],
- 'temporary directory': [VDR.V_STRING],
- 'state dump rolling archive length': [VDR.V_INTEGER, 10],
+ # client
'disable interactive command prompts': [VDR.V_BOOLEAN, True],
- 'enable run directory housekeeping': [VDR.V_BOOLEAN],
- 'run directory rolling archive length': [VDR.V_INTEGER, 2],
- 'task host select command timeout': [VDR.V_INTERVAL, DurationFloat(10)],
- 'xtrigger function timeout': [VDR.V_INTERVAL, DurationFloat(10)],
+ # suite
+ 'run directory rolling archive length': [VDR.V_INTEGER, -1],
+ # suite-task communication
'task messaging': {
'retry interval': [VDR.V_INTERVAL, DurationFloat(5)],
'maximum number of tries': [VDR.V_INTEGER, 7],
'connection timeout': [VDR.V_INTERVAL, DurationFloat(30)],
},
+ # suite
'cylc': {
'UTC mode': [VDR.V_BOOLEAN],
'health check interval': [VDR.V_INTERVAL, DurationFloat(600)],
@@ -82,11 +78,13 @@
},
},
+ # suite
'suite logging': {
'rolling archive length': [VDR.V_INTEGER, 5],
'maximum size in bytes': [VDR.V_INTEGER, 1000000],
},
+ # general
'documentation': {
'local': [VDR.V_STRING, ''],
'online': [VDR.V_STRING,
@@ -94,19 +92,23 @@
'cylc homepage': [VDR.V_STRING, 'http://cylc.github.io/'],
},
+ # general
'document viewers': {
'html': [VDR.V_STRING, 'firefox'],
},
+ # client
'editors': {
'terminal': [VDR.V_STRING, 'vim'],
'gui': [VDR.V_STRING, 'gvim -f'],
},
+ # client
'monitor': {
'sort order': [VDR.V_STRING, 'definition', 'alphanumeric'],
},
+ # task
'hosts': {
'localhost': {
'run directory': [VDR.V_STRING, '$HOME/cylc-run'],
@@ -175,6 +177,7 @@
},
},
+ # task
'task events': {
'execution timeout': [VDR.V_INTERVAL],
'handlers': [VDR.V_STRING_LIST],
@@ -188,6 +191,7 @@
'submission timeout': [VDR.V_INTERVAL],
},
+ # client
'test battery': {
'remote host with shared fs': [VDR.V_STRING],
'remote host': [VDR.V_STRING],
@@ -204,12 +208,14 @@
},
},
+ # suite
'suite host self-identification': {
'method': [VDR.V_STRING, 'name', 'address', 'hardwired'],
'target': [VDR.V_STRING, 'google.com'],
'host': [VDR.V_STRING],
},
+ # suite
'authentication': {
# Allow owners to grant public shutdown rights at the most, not full
# control.
@@ -220,6 +226,7 @@
Priv.STATE_TOTALS, Priv.READ, Priv.SHUTDOWN]])
},
+ # suite
'suite servers': {
'run hosts': [VDR.V_SPACELESS_STRING_LIST],
'run ports': [VDR.V_INTEGER_LIST, list(range(43001, 43101))],
@@ -258,6 +265,10 @@ def upg(cfg, descr):
u.obsolete('8.0.0', ['suite servers', 'scan hosts'])
u.obsolete('8.0.0', ['suite servers', 'scan ports'])
u.obsolete('8.0.0', ['communication'])
+ u.obsolete('8.0.0', ['temporary directory'])
+ u.obsolete('8.0.0', ['task host select command timeout'])
+ u.obsolete('8.0.0', ['xtrigger function timeout'])
+ u.obsolete('8.0.0', ['enable run directory housekeeping'])
u.upgrade()
@@ -266,9 +277,6 @@ class GlobalConfig(ParsecConfig):
"""
Handle global (all suites) site and user configuration for cylc.
User file values override site file values.
-
- For all derived items - paths hardwired under the configurable top
- levels - use the get_derived_host_item(suite,host) method.
"""
_DEFAULT = None
@@ -328,49 +336,7 @@ def load(self):
LOG.error('bad %s %s', conf_type, fname)
raise
# (OK if no flow.rc is found, just use system defaults).
- self.transform()
-
- def get_derived_host_item(
- self, suite, item, host=None, owner=None, replace_home=False):
- """Compute hardwired paths relative to the configurable top dirs."""
-
- # suite run dir
- srdir = os.path.join(
- self.get_host_item('run directory', host, owner, replace_home),
- suite)
- # suite workspace
- swdir = os.path.join(
- self.get_host_item('work directory', host, owner, replace_home),
- suite)
-
- if item == 'suite run directory':
- value = srdir
-
- elif item == 'suite log directory':
- value = os.path.join(srdir, 'log', 'suite')
-
- elif item == 'suite log':
- value = os.path.join(srdir, 'log', 'suite', 'log')
-
- elif item == 'suite job log directory':
- value = os.path.join(srdir, 'log', 'job')
-
- elif item == 'suite config log directory':
- value = os.path.join(srdir, 'log', 'suiterc')
-
- elif item == 'suite work root':
- value = swdir
-
- elif item == 'suite work directory':
- value = os.path.join(swdir, 'work')
-
- elif item == 'suite share directory':
- value = os.path.join(swdir, 'share')
-
- else:
- raise GlobalConfigError("Illegal derived item: " + item)
-
- return value
+ self._transform()
def get_host_item(self, item, host=None, owner=None, replace_home=False,
owner_home=None):
@@ -420,85 +386,7 @@ def get_host_item(self, item, host=None, owner=None, replace_home=False,
value = 'zmq'
return value
- def roll_directory(self, dir_, name, archlen=0):
- """Create a directory after rolling back any previous instances of it.
-
- E.g. if archlen = 2 we keep:
- dir_, dir_.1, dir_.2. If 0 keep no old ones.
- """
- for i in range(archlen, -1, -1): # archlen...0
- if i > 0:
- dpath = dir_ + '.' + str(i)
- else:
- dpath = dir_
- if os.path.exists(dpath):
- if i >= archlen:
- # remove oldest backup
- shutil.rmtree(dpath)
- else:
- # roll others over
- os.rename(dpath, dir_ + '.' + str(i + 1))
- self.create_directory(dir_, name)
-
- @staticmethod
- def create_directory(dir_, name):
- """Create directory. Raise GlobalConfigError on error."""
- try:
- os.makedirs(dir_, exist_ok=True)
- except OSError as exc:
- LOG.exception(exc)
- raise GlobalConfigError(
- 'Failed to create directory "' + name + '"')
-
- def create_cylc_run_tree(self, suite):
- """Create all top-level cylc-run output dirs on the suite host."""
- cfg = self.get()
- item = 'suite run directory'
- idir = self.get_derived_host_item(suite, item)
- LOG.debug('creating %s: %s', item, idir)
- if cfg['enable run directory housekeeping']:
- self.roll_directory(
- idir, item, cfg['run directory rolling archive length'])
-
- for item in [
- 'suite log directory',
- 'suite job log directory',
- 'suite config log directory',
- 'suite work directory',
- 'suite share directory']:
- idir = self.get_derived_host_item(suite, item)
- LOG.debug('creating %s: %s', item, idir)
- self.create_directory(idir, item)
-
- item = 'temporary directory'
- value = cfg[item]
- if value:
- self.create_directory(value, item)
-
- def get_tmpdir(self):
- """Make a new temporary directory and arrange for it to be
- deleted automatically when we're finished with it. Call this
- explicitly just before use to ensure the directory is not
- deleted by other processes before it is needed. THIS IS
- CURRENTLY ONLY USED BY A FEW CYLC COMMANDS. If cylc suites
- ever need it this must be called AFTER FORKING TO DAEMON MODE or
- atexit() will delete the directory when the initial process
- exits after forking."""
-
- cfg = self.get()
- tdir = cfg['temporary directory']
- if tdir:
- tdir = os.path.expandvars(tdir)
- tmpdir = mkdtemp(prefix="cylc-", dir=os.path.expandvars(tdir))
- else:
- tmpdir = mkdtemp(prefix="cylc-")
- # self-cleanup
- atexit.register(lambda: shutil.rmtree(tmpdir))
- # now replace the original item to allow direct access
- cfg['temporary directory'] = tmpdir
- return tmpdir
-
- def transform(self):
+ def _transform(self):
"""Transform various settings.
Host item values of None default to modified localhost values.
diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index 6e7d181d6fa..0a92554623f 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -56,6 +56,12 @@
from cylc.flow.cycling.iso8601 import ingest_time
import cylc.flow.flags
from cylc.flow.graphnode import GraphNodeParser
+from cylc.flow.pathutil import (
+ get_suite_run_dir,
+ get_suite_run_log_dir,
+ get_suite_run_share_dir,
+ get_suite_run_work_dir,
+)
from cylc.flow.print_tree import print_tree
from cylc.flow.subprocctx import SubFuncContext
from cylc.flow.subprocpool import get_func
@@ -119,14 +125,10 @@ def __init__(self, suite, fpath, template_vars=None,
self.suite = suite # suite name
self.fpath = fpath # suite definition
self.fdir = os.path.dirname(fpath)
- self.run_dir = run_dir or glbl_cfg().get_derived_host_item(
- self.suite, 'suite run directory')
- self.log_dir = log_dir or glbl_cfg().get_derived_host_item(
- self.suite, 'suite log directory')
- self.work_dir = work_dir or glbl_cfg().get_derived_host_item(
- self.suite, 'suite work directory')
- self.share_dir = share_dir or glbl_cfg().get_derived_host_item(
- self.suite, 'suite share directory')
+ self.run_dir = run_dir or get_suite_run_dir(self.suite)
+ self.log_dir = log_dir or get_suite_run_log_dir(self.suite)
+ self.share_dir = share_dir or get_suite_run_share_dir(self.suite)
+ self.work_dir = work_dir or get_suite_run_work_dir(self.suite)
self.owner = owner
self.run_mode = run_mode
self.strict = strict
diff --git a/cylc/flow/daemonize.py b/cylc/flow/daemonize.py
index 31cef52ab5b..4f81fce4fe9 100644
--- a/cylc/flow/daemonize.py
+++ b/cylc/flow/daemonize.py
@@ -22,7 +22,7 @@
import sys
from time import sleep, time
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
+from cylc.flow.pathutil import get_suite_run_log_name
SUITE_SCAN_INFO_TMPL = r"""
@@ -53,9 +53,9 @@ def daemonize(server):
http://code.activestate.com/recipes/66012-fork-a-daemon-process-on-unix/
"""
- logpath = glbl_cfg().get_derived_host_item(server.suite, 'suite log')
+ logfname = get_suite_run_log_name(server.suite)
try:
- old_log_mtime = os.stat(logpath).st_mtime
+ old_log_mtime = os.stat(logfname).st_mtime
except OSError:
old_log_mtime = None
# fork 1
@@ -75,11 +75,11 @@ def daemonize(server):
# LOG-PREFIX Suite server program: url=URL, pid=PID
# Otherwise, something has gone wrong, print the suite log
# and exit with an error.
- log_stat = os.stat(logpath)
+ log_stat = os.stat(logfname)
if (log_stat.st_mtime == old_log_mtime or
log_stat.st_size == 0):
continue
- for line in open(logpath):
+ for line in open(logfname):
if server.START_MESSAGE_PREFIX in line:
suite_url, suite_pid = (
item.rsplit("=", 1)[-1]
@@ -88,7 +88,7 @@ def daemonize(server):
elif ' ERROR -' in line or ' CRITICAL -' in line:
# ERROR and CRITICAL before suite starts
try:
- sys.stderr.write(open(logpath).read())
+ sys.stderr.write(open(logfname).read())
sys.exit(1)
except IOError:
sys.exit("Suite server program exited")
diff --git a/cylc/flow/etc/job.sh b/cylc/flow/etc/job.sh
index c7c7261e9dc..ae11ead17ec 100644
--- a/cylc/flow/etc/job.sh
+++ b/cylc/flow/etc/job.sh
@@ -121,8 +121,7 @@ cylc__job__main() {
# System paths:
# * suite directory (installed run-dir first).
export PATH="${CYLC_SUITE_RUN_DIR}/bin:${CYLC_SUITE_DEF_PATH}/bin:${PATH}"
- export
- PYTHONPATH="${CYLC_SUITE_RUN_DIR}/lib/python:${CYLC_SUITE_DEF_PATH}/lib/python:${PYTHONPATH:-}"
+ export PYTHONPATH="${CYLC_SUITE_RUN_DIR}/lib/python:${CYLC_SUITE_DEF_PATH}/lib/python:${PYTHONPATH:-}"
# Create share and work directories
mkdir -p "${CYLC_SUITE_SHARE_DIR}" || true
mkdir -p "$(dirname "${CYLC_TASK_WORK_DIR}")" || true
diff --git a/cylc/flow/exceptions.py b/cylc/flow/exceptions.py
index ec963ae7025..31bd6b49201 100644
--- a/cylc/flow/exceptions.py
+++ b/cylc/flow/exceptions.py
@@ -55,10 +55,6 @@ class SuiteConfigError(CylcConfigError):
"""Exception for configuration errors in a Cylc suite configuration."""
-class GlobalConfigError(CylcConfigError):
- """Exception for configuration errors in a Cylc global configuration."""
-
-
class GraphParseError(SuiteConfigError):
"""Exception for errors in Cylc suite graphing."""
diff --git a/cylc/flow/job_file.py b/cylc/flow/job_file.py
index 4037551a94c..540774e9a25 100644
--- a/cylc/flow/job_file.py
+++ b/cylc/flow/job_file.py
@@ -26,6 +26,9 @@
from cylc.flow.batch_sys_manager import BatchSysManager
from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
import cylc.flow.flags
+from cylc.flow.pathutil import (
+ get_remote_suite_run_dir,
+ get_remote_suite_work_dir)
class JobFileWriter(object):
@@ -56,7 +59,8 @@ def write(self, local_job_file_path, job_conf, check_syntax=True):
# variables: NEXT_CYCLE=$( cylc cycle-point --offset-hours=6 )
tmp_name = local_job_file_path + '.tmp'
- run_d = self._get_derived_host_item(job_conf, 'suite run directory')
+ run_d = get_remote_suite_run_dir(
+ job_conf['host'], job_conf['owner'], job_conf['suite_name'])
try:
with open(tmp_name, 'w') as handle:
self._write_header(handle, job_conf)
@@ -116,12 +120,6 @@ def _check_script_value(value):
return True
return False
- @staticmethod
- def _get_derived_host_item(job_conf, key):
- """Return derived host item from glbl_cfg()."""
- return glbl_cfg().get_derived_host_item(
- job_conf['suite_name'], key, job_conf["host"], job_conf["owner"])
-
@staticmethod
def _get_host_item(job_conf, key):
"""Return host item from glbl_cfg()."""
@@ -194,7 +192,8 @@ def _write_environment_1(self, handle, job_conf, run_d):
handle.write('\n')
# override and write task-host-specific suite variables
- work_d = self._get_derived_host_item(job_conf, 'suite work root')
+ work_d = get_remote_suite_work_dir(
+ job_conf["host"], job_conf["owner"], job_conf['suite_name'])
handle.write('\n export CYLC_SUITE_RUN_DIR="%s"' % run_d)
if work_d != run_d:
# Note: not an environment variable, but used by job.sh
diff --git a/cylc/flow/loggingutil.py b/cylc/flow/loggingutil.py
index 08da1727390..7dfb7947b41 100644
--- a/cylc/flow/loggingutil.py
+++ b/cylc/flow/loggingutil.py
@@ -31,6 +31,7 @@
from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
+from cylc.flow.pathutil import get_suite_run_log_name
from cylc.flow.wallclock import (
get_current_time_string, get_time_string_from_unix_time)
@@ -77,8 +78,7 @@ class TimestampRotatingFileHandler(logging.FileHandler):
MIN_BYTES = 1024
def __init__(self, suite, no_detach=False):
- logging.FileHandler.__init__(
- self, glbl_cfg().get_derived_host_item(suite, 'suite log'))
+ logging.FileHandler.__init__(self, get_suite_run_log_name(suite))
self.no_detach = no_detach
self.stamp = None
self.formatter = CylcLogFormatter()
diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
new file mode 100644
index 00000000000..678b294995f
--- /dev/null
+++ b/cylc/flow/pathutil.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+"""Functions to return paths to common suite files and directories."""
+
+import os
+from shutil import rmtree
+
+
+from cylc.flow import LOG
+from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
+
+
+def get_remote_suite_run_dir(host, owner, suite, *args):
+ """Return remote suite run directory, join any extra args."""
+ return os.path.join(
+ glbl_cfg().get_host_item('run directory', host, owner), suite, *args)
+
+
+def get_remote_suite_run_job_dir(host, owner, suite, *args):
+ """Return remote suite run directory, join any extra args."""
+ return get_remote_suite_run_dir(
+ host, owner, suite, 'log', 'job', *args)
+
+
+def get_remote_suite_work_dir(host, owner, suite, *args):
+ """Return remote suite work directory root, join any extra args."""
+ return os.path.join(
+ glbl_cfg().get_host_item('work directory', host, owner),
+ suite,
+ *args)
+
+
+def get_suite_run_dir(suite, *args):
+ """Return local suite run directory, join any extra args."""
+ return os.path.join(
+ glbl_cfg().get_host_item('run directory'), suite, *args)
+
+
+def get_suite_run_job_dir(suite, *args):
+ """Return suite run job (log) directory, join any extra args."""
+ return get_suite_run_dir(suite, 'log', 'job', *args)
+
+
+def get_suite_run_log_dir(suite, *args):
+ """Return suite run log directory, join any extra args."""
+ return get_suite_run_dir(suite, 'log', 'suite', *args)
+
+
+def get_suite_run_log_name(suite):
+ """Return suite run log file path."""
+ return get_suite_run_dir(suite, 'log', 'suite', 'log')
+
+
+def get_suite_run_rc_dir(suite, *args):
+ """Return suite run suite.rc log directory, join any extra args."""
+ return get_suite_run_dir(suite, 'log', 'suiterc', *args)
+
+
+def get_suite_run_pub_db_name(suite):
+ """Return suite run public database file path."""
+ return get_suite_run_dir(suite, 'log', 'db')
+
+
+def get_suite_run_share_dir(suite, *args):
+ """Return local suite work/share directory, join any extra args."""
+ return os.path.join(
+ glbl_cfg().get_host_item('work directory'), suite, 'share', *args)
+
+
+def get_suite_run_work_dir(suite, *args):
+ """Return local suite work/work directory, join any extra args."""
+ return os.path.join(
+ glbl_cfg().get_host_item('work directory'), suite, 'work', *args)
+
+
+def make_suite_run_tree(suite):
+ """Create all top-level cylc-run output dirs on the suite host."""
+ cfg = glbl_cfg().get()
+ # Roll archive
+ archlen = cfg['run directory rolling archive length']
+ dir_ = get_suite_run_dir(suite)
+ for i in range(archlen, -1, -1): # archlen...0
+ if i > 0:
+ dpath = dir_ + '.' + str(i)
+ else:
+ dpath = dir_
+ if os.path.exists(dpath):
+ if i >= archlen:
+ # remove oldest backup
+ rmtree(dpath)
+ else:
+ # roll others over
+ os.rename(dpath, dir_ + '.' + str(i + 1))
+ # Create
+ for dir_ in (
+ get_suite_run_dir(suite),
+ get_suite_run_log_dir(suite),
+ get_suite_run_job_dir(suite),
+ get_suite_run_rc_dir(suite),
+ get_suite_run_share_dir(suite),
+ get_suite_run_work_dir(suite),
+ ):
+ if dir_:
+ os.makedirs(dir_, exist_ok=True)
+ LOG.debug('%s: directory created', dir_)
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index fdc4489018d..088b09838e7 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -48,6 +48,14 @@
ReferenceLogFileHandler
from cylc.flow.log_diagnosis import LogSpec
from cylc.flow.network.server import SuiteRuntimeServer
+from cylc.flow.pathutil import (
+ get_suite_run_dir,
+ get_suite_run_log_dir,
+ get_suite_run_rc_dir,
+ get_suite_run_share_dir,
+ get_suite_run_work_dir,
+ make_suite_run_tree,
+)
from cylc.flow.profiler import Profiler
from cylc.flow.state_summary_mgr import StateSummaryMgr
from cylc.flow.subprocpool import SubProcPool
@@ -145,14 +153,10 @@ def __init__(self, is_restart, options, args):
# For user-defined batch system handlers
sys.path.append(os.path.join(self.suite_dir, 'python'))
sys.path.append(os.path.join(self.suite_dir, 'lib', 'python'))
- self.suite_run_dir = glbl_cfg().get_derived_host_item(
- self.suite, 'suite run directory')
- self.suite_work_dir = glbl_cfg().get_derived_host_item(
- self.suite, 'suite work directory')
- self.suite_share_dir = glbl_cfg().get_derived_host_item(
- self.suite, 'suite share directory')
- self.suite_log_dir = glbl_cfg().get_derived_host_item(
- self.suite, 'suite log directory')
+ self.suite_run_dir = get_suite_run_dir(self.suite)
+ self.suite_work_dir = get_suite_run_work_dir(self.suite)
+ self.suite_share_dir = get_suite_run_share_dir(self.suite)
+ self.suite_log_dir = get_suite_run_log_dir(self.suite)
self.config = None
@@ -240,7 +244,7 @@ def start(self):
"""Start the server."""
self._start_print_blurb()
- glbl_cfg().create_cylc_run_tree(self.suite)
+ make_suite_run_tree(self.suite)
if self.is_restart:
self.suite_db_mgr.restart_upgrade()
@@ -983,8 +987,6 @@ def load_suiterc(self, is_reload=False):
)
self.suiterc_update_time = time()
# Dump the loaded suiterc for future reference.
- cfg_logdir = glbl_cfg().get_derived_host_item(
- self.suite, 'suite config log directory')
time_str = get_current_time_string(
override_use_utc=True, use_basic_format=True,
display_sub_seconds=False
@@ -995,8 +997,8 @@ def load_suiterc(self, is_reload=False):
load_type = "restart"
else:
load_type = "run"
- base_name = "%s-%s.rc" % (time_str, load_type)
- file_name = os.path.join(cfg_logdir, base_name)
+ file_name = get_suite_run_rc_dir(
+ self.suite, f"{time_str}-{load_type}.rc")
with open(file_name, "wb") as handle:
handle.write(b"# cylc-version: %s\n" % CYLC_VERSION.encode())
printcfg(self.config.cfg, none_str=None, handle=handle)
diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py
index 2c60eeb093f..3bbe0b2f3af 100755
--- a/cylc/flow/scheduler_cli.py
+++ b/cylc/flow/scheduler_cli.py
@@ -20,11 +20,11 @@
import os
import sys
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
import cylc.flow.flags
from cylc.flow.host_appointer import HostAppointer, EmptyHostList
from cylc.flow.hostuserutil import is_remote_host
from cylc.flow.option_parsers import CylcOptionParser as COP
+from cylc.flow.pathutil import get_suite_run_dir
from cylc.flow.remote import remrun, remote_cylc_cmd
from cylc.flow.scheduler import Scheduler
from cylc.flow.suite_srv_files_mgr import (
@@ -123,9 +123,7 @@ def main(is_restart=False):
SuiteSrvFilesManager().get_suite_source_dir(args[0], options.owner)
except SuiteServiceFileError:
# Source path is assumed to be the run directory
- SuiteSrvFilesManager().register(
- args[0],
- glbl_cfg().get_derived_host_item(args[0], 'suite run directory'))
+ SuiteSrvFilesManager().register(args[0], get_suite_run_dir(args[0]))
try:
scheduler = Scheduler(is_restart, options, args)
diff --git a/cylc/flow/suite_srv_files_mgr.py b/cylc/flow/suite_srv_files_mgr.py
index 3b7ff3c029d..3b5e09f2e8f 100644
--- a/cylc/flow/suite_srv_files_mgr.py
+++ b/cylc/flow/suite_srv_files_mgr.py
@@ -26,6 +26,7 @@
from cylc.flow import LOG
from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
from cylc.flow.exceptions import SuiteServiceFileError
+from cylc.flow.pathutil import get_remote_suite_run_dir, get_suite_run_dir
from cylc.flow.pkg_resources import extract_pkg_resources
import cylc.flow.flags
from cylc.flow.hostuserutil import (
@@ -345,8 +346,7 @@ def get_suite_srv_dir(self, reg, suite_owner=None):
run_d = os.getenv("CYLC_SUITE_RUN_DIR")
if (not run_d or os.getenv("CYLC_SUITE_NAME") != reg or
os.getenv("CYLC_SUITE_OWNER") != suite_owner):
- run_d = glbl_cfg().get_derived_host_item(
- reg, 'suite run directory')
+ run_d = get_suite_run_dir(reg)
return os.path.join(run_d, self.DIR_BASE_SRV)
def list_suites(self, regfilter=None):
@@ -615,9 +615,7 @@ def _load_remote_item(self, item, reg, owner, host):
if item == self.FILE_BASE_CONTACT and not is_remote_host(host):
# Attempt to read suite contact file via the local filesystem.
path = r'%(run_d)s/%(srv_base)s' % {
- 'run_d': glbl_cfg().get_derived_host_item(
- reg, 'suite run directory', 'localhost', owner,
- replace_home=False),
+ 'run_d': get_remote_suite_run_dir('localhost', owner, reg),
'srv_base': self.DIR_BASE_SRV,
}
content = self._load_local_item(item, path)
@@ -632,8 +630,7 @@ def _load_remote_item(self, item, reg, owner, host):
r'''cat "%(run_d)s/%(srv_base)s/%(item)s"'''
) % {
'prefix': prefix,
- 'run_d': glbl_cfg().get_derived_host_item(
- reg, 'suite run directory', host, owner),
+ 'run_d': get_remote_suite_run_dir(host, owner, reg),
'srv_base': self.DIR_BASE_SRV,
'item': item
}
diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py
index ffb3bcac551..c159a0c9e3f 100644
--- a/cylc/flow/task_events_mgr.py
+++ b/cylc/flow/task_events_mgr.py
@@ -39,6 +39,9 @@
from cylc.flow import LOG
from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
from cylc.flow.hostuserutil import get_host, get_user
+from cylc.flow.pathutil import (
+ get_remote_suite_run_job_dir,
+ get_suite_run_job_dir)
from cylc.flow.subprocctx import SubProcContext
from cylc.flow.task_action_timer import TaskActionTimer
from cylc.flow.task_job_logs import (
@@ -577,11 +580,11 @@ def _process_job_logs_retrieval(self, schd_ctx, ctx, id_keys):
cmd += ["--include=%s" % (include) for include in sorted(includes)]
cmd.append("--exclude=/**") # exclude everything else
# Remote source
- cmd.append(ctx.user_at_host + ":" + glbl_cfg().get_derived_host_item(
- schd_ctx.suite, "suite job log directory", s_host, s_user) + "/")
+ cmd.append("%s:%s/" % (
+ ctx.user_at_host,
+ get_remote_suite_run_job_dir(s_host, s_user, schd_ctx.suite)))
# Local target
- cmd.append(glbl_cfg().get_derived_host_item(
- schd_ctx.suite, "suite job log directory") + "/")
+ cmd.append(get_suite_run_job_dir(schd_ctx.suite) + "/")
self.proc_pool.put_command(
SubProcContext(ctx, cmd, env=dict(os.environ), id_keys=id_keys),
self._job_logs_retrieval_callback, [schd_ctx])
diff --git a/cylc/flow/task_job_logs.py b/cylc/flow/task_job_logs.py
index 7a034f8f9f7..5b4760ae2ac 100644
--- a/cylc/flow/task_job_logs.py
+++ b/cylc/flow/task_job_logs.py
@@ -18,7 +18,7 @@
"""Define task job log filenames and option names."""
import os
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
+from cylc.flow.pathutil import get_suite_run_job_dir
# Task job log filenames.
JOB_LOG_JOB = "job"
@@ -56,7 +56,7 @@ def get_task_job_id(point, name, submit_num=None):
def get_task_job_log(suite, point, name, submit_num=None, suffix=None):
"""Return the full job log path."""
args = [
- glbl_cfg().get_derived_host_item(suite, "suite job log directory"),
+ get_suite_run_job_dir(suite),
get_task_job_id(point, name, submit_num)]
if suffix is not None:
args.append(suffix)
diff --git a/cylc/flow/task_job_mgr.py b/cylc/flow/task_job_mgr.py
index fd4f7912c49..640dc996e64 100644
--- a/cylc/flow/task_job_mgr.py
+++ b/cylc/flow/task_job_mgr.py
@@ -37,17 +37,17 @@
from cylc.flow import LOG
from cylc.flow.batch_sys_manager import JobPollContext
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
from cylc.flow.hostuserutil import get_host, is_remote_host, is_remote_user
from cylc.flow.job_file import JobFileWriter
-from cylc.flow.task_job_logs import (
- JOB_LOG_JOB, get_task_job_log, get_task_job_job_log,
- get_task_job_activity_log, get_task_job_id, NN)
+from cylc.flow.pathutil import get_remote_suite_run_job_dir
from cylc.flow.subprocpool import SubProcPool
from cylc.flow.subprocctx import SubProcContext
from cylc.flow.task_action_timer import TaskActionTimer
from cylc.flow.task_events_mgr import TaskEventsManager, log_task_job_activity
from cylc.flow.task_message import FAIL_MESSAGE_PREFIX
+from cylc.flow.task_job_logs import (
+ JOB_LOG_JOB, get_task_job_log, get_task_job_job_log,
+ get_task_job_activity_log, get_task_job_id, NN)
from cylc.flow.task_outputs import (
TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED,
TASK_OUTPUT_FAILED)
@@ -282,8 +282,7 @@ def submit_task_jobs(self, suite, itasks, is_simulation=False):
if remote_mode:
cmd.append('--remote-mode')
cmd.append('--')
- cmd.append(glbl_cfg().get_derived_host_item(
- suite, 'suite job log directory', host, owner))
+ cmd.append(get_remote_suite_run_job_dir(host, owner, suite))
# Chop itasks into a series of shorter lists if it's very big
# to prevent overloading of stdout and stderr pipes.
itasks = sorted(itasks, key=lambda itask: itask.identity)
@@ -651,8 +650,7 @@ def _run_job_cmd(self, cmd_key, suite, itasks, callback):
if is_remote_user(owner):
cmd.append("--user=%s" % (owner))
cmd.append("--")
- cmd.append(glbl_cfg().get_derived_host_item(
- suite, "suite job log directory", host, owner))
+ cmd.append(get_remote_suite_run_job_dir(host, owner, suite))
job_log_dirs = []
for itask in sorted(itasks, key=lambda itask: itask.identity):
job_log_dirs.append(get_task_job_id(
@@ -852,11 +850,8 @@ def _prep_submit_task_job_impl(self, suite, itask, rtconfig):
self._create_job_log_path(suite, itask)
job_d = get_task_job_id(
itask.point, itask.tdef.name, itask.submit_num)
- job_file_path = os.path.join(
- glbl_cfg().get_derived_host_item(
- suite, "suite job log directory",
- itask.task_host, itask.task_owner),
- job_d, JOB_LOG_JOB)
+ job_file_path = get_remote_suite_run_job_dir(
+ itask.task_host, itask.task_owner, suite, job_d, JOB_LOG_JOB)
return {
'batch_system_name': rtconfig['job']['batch system'],
'batch_submit_command_template': (
diff --git a/cylc/flow/task_message.py b/cylc/flow/task_message.py
index f55ebd81ec4..bc79694a001 100644
--- a/cylc/flow/task_message.py
+++ b/cylc/flow/task_message.py
@@ -28,9 +28,9 @@
import sys
-from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
import cylc.flow.flags
from cylc.flow.network.client import SuiteRuntimeClient
+from cylc.flow.pathutil import get_suite_run_job_dir
from cylc.flow.task_outputs import TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED
from cylc.flow.wallclock import get_current_time_string
@@ -93,9 +93,7 @@ def _append_job_status_file(suite, task_job, event_time, messages):
"""Write messages to job status file."""
job_log_name = os.getenv('CYLC_TASK_LOG_ROOT')
if not job_log_name:
- job_log_name = os.path.join(
- glbl_cfg().get_derived_host_item(suite, 'suite job log directory'),
- 'job')
+ job_log_name = get_suite_run_job_dir(suite, task_job, 'job')
try:
job_status_file = open(job_log_name + '.status', 'a')
except IOError:
diff --git a/cylc/flow/task_remote_mgr.py b/cylc/flow/task_remote_mgr.py
index cd1cbe634ea..12e00354115 100644
--- a/cylc/flow/task_remote_mgr.py
+++ b/cylc/flow/task_remote_mgr.py
@@ -36,6 +36,7 @@
from cylc.flow.exceptions import TaskRemoteMgmtError
import cylc.flow.flags
from cylc.flow.hostuserutil import is_remote, is_remote_host, is_remote_user
+from cylc.flow.pathutil import get_remote_suite_run_dir
from cylc.flow.subprocctx import SubProcContext
from cylc.flow.task_remote_cmd import (
FILE_BASE_UUID, REMOTE_INIT_DONE, REMOTE_INIT_NOT_REQUIRED)
@@ -95,14 +96,11 @@ def remote_host_select(self, host_str):
host_str = value # command succeeded
else:
# Command not launched (or already reset)
- timeout = glbl_cfg().get(['task host select command timeout'])
- if timeout:
- cmd = ['timeout', str(int(timeout)), 'bash', '-c', cmd_str]
- else:
- cmd = ['bash', '-c', cmd_str]
self.proc_pool.put_command(
SubProcContext(
- 'remote-host-select', cmd, env=dict(os.environ)),
+ 'remote-host-select',
+ ['bash', '-c', cmd_str],
+ env=dict(os.environ)),
self._remote_host_select_callback, [cmd_str])
self.remote_host_str_map[cmd_str] = None
return self.remote_host_str_map[cmd_str]
@@ -198,8 +196,7 @@ def remote_init(self, host, owner):
if comm_meth in ['ssh']:
cmd.append('--indirect-comm=%s' % comm_meth)
cmd.append(str(self.uuid_str))
- cmd.append(glbl_cfg().get_derived_host_item(
- self.suite, 'suite run directory', host, owner))
+ cmd.append(get_remote_suite_run_dir(host, owner, self.suite))
self.proc_pool.put_command(
SubProcContext('remote-init', cmd, stdin_files=[tmphandle]),
self._remote_init_callback,
@@ -237,8 +234,7 @@ def remote_tidy(self):
cmd.append('--user=%s' % owner)
if cylc.flow.flags.debug:
cmd.append('--debug')
- cmd.append(os.path.join(glbl_cfg().get_derived_host_item(
- self.suite, 'suite run directory', host, owner)))
+ cmd.append(get_remote_suite_run_dir(host, owner, self.suite))
procs[(host, owner)] = (
cmd,
Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=open(os.devnull)))
diff --git a/cylc/flow/tests/test_loggingutil.py b/cylc/flow/tests/test_loggingutil.py
index 2f611db07cb..4a297f24bb5 100644
--- a/cylc/flow/tests/test_loggingutil.py
+++ b/cylc/flow/tests/test_loggingutil.py
@@ -28,8 +28,13 @@
class TestLoggingutil(unittest.TestCase):
+ @mock.patch("cylc.flow.loggingutil.get_suite_run_log_name")
@mock.patch("cylc.flow.loggingutil.glbl_cfg")
- def test_value_error_raises_system_exit(self, mocked_glbl_cfg):
+ def test_value_error_raises_system_exit(
+ self,
+ mocked_glbl_cfg,
+ mocked_get_suite_run_log_name,
+ ):
"""Test that a ValueError when writing to a log stream won't result
in multiple exceptions (what could lead to infinite loop in some
occasions. Instead, it **must** raise a SystemExit"""
@@ -37,8 +42,8 @@ def test_value_error_raises_system_exit(self, mocked_glbl_cfg):
# mock objects used when creating the file handler
mocked = mock.MagicMock()
mocked_glbl_cfg.return_value = mocked
- mocked.get_derived_host_item.return_value = tf.name
mocked.get.return_value = 100
+ mocked_get_suite_run_log_name.return_value = tf.name
file_handler = TimestampRotatingFileHandler("suiteA", False)
# next line is important as pytest can have a "Bad file descriptor"
# due to a FileHandler with default "a" (pytest tries to r/w).
diff --git a/cylc/flow/tests/test_pathutil.py b/cylc/flow/tests/test_pathutil.py
new file mode 100644
index 00000000000..6a5f1c53457
--- /dev/null
+++ b/cylc/flow/tests/test_pathutil.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+"""Tests for "cylc.flow.pathutil"."""
+
+from unittest import TestCase
+from unittest.mock import call, patch, MagicMock
+
+from cylc.flow.pathutil import (
+ get_remote_suite_run_dir,
+ get_remote_suite_run_job_dir,
+ get_remote_suite_work_dir,
+ get_suite_run_dir,
+ get_suite_run_job_dir,
+ get_suite_run_log_dir,
+ get_suite_run_log_name,
+ get_suite_run_pub_db_name,
+ get_suite_run_rc_dir,
+ get_suite_run_share_dir,
+ get_suite_run_work_dir,
+ make_suite_run_tree,
+)
+
+
+class TestPathutil(TestCase):
+ """Tests for functions in "cylc.flow.pathutil"."""
+
+ @patch('cylc.flow.pathutil.glbl_cfg')
+ def test_get_remote_suite_run_dirs(self, mocked_glbl_cfg):
+ """Usage of get_remote_suite_run_*dir."""
+ mocked = MagicMock()
+ mocked_glbl_cfg.return_value = mocked
+ mocked.get_host_item.return_value = '/home/sweet/cylc-run'
+ # func = get_remote_* function to test
+ # cfg = configuration used in mocked global configuration
+ # tail1 = expected tail of return value from configuration
+ # args = extra *args
+ # tail2 = expected tail of return value from extra args
+ for func, cfg, tail1 in (
+ (get_remote_suite_run_dir, 'run directory', ''),
+ (get_remote_suite_run_job_dir, 'run directory', '/log/job'),
+ (get_remote_suite_work_dir, 'work directory', ''),
+ ):
+ for args, tail2 in (
+ ((), ''),
+ (('comes', 'true'), '/comes/true'),
+ ):
+ self.assertEqual(
+ f'/home/sweet/cylc-run/my-suite/dream{tail1}{tail2}',
+ func('myhost', 'myuser', 'my-suite/dream', *args),
+ )
+ mocked.get_host_item.assert_called_with(
+ cfg, 'myhost', 'myuser')
+ mocked.get_host_item.reset_mock()
+
+ @patch('cylc.flow.pathutil.glbl_cfg')
+ def test_get_suite_run_dirs(self, mocked_glbl_cfg):
+ """Usage of get_suite_run_*dir."""
+ mocked = MagicMock()
+ mocked_glbl_cfg.return_value = mocked
+ mocked.get_host_item.return_value = '/home/sweet/cylc-run'
+ # func = get_remote_* function to test
+ # cfg = configuration used in mocked global configuration
+ # tail1 = expected tail of return value from configuration
+ # args = extra *args
+ # tail2 = expected tail of return value from extra args
+ for func, cfg, tail1 in (
+ (get_suite_run_dir, 'run directory', ''),
+ (get_suite_run_job_dir, 'run directory', '/log/job'),
+ (get_suite_run_log_dir, 'run directory', '/log/suite'),
+ (get_suite_run_rc_dir, 'run directory', '/log/suiterc'),
+ (get_suite_run_share_dir, 'work directory', '/share'),
+ (get_suite_run_work_dir, 'work directory', '/work'),
+ ):
+ for args, tail2 in (
+ ((), ''),
+ (('comes', 'true'), '/comes/true'),
+ ):
+ self.assertEqual(
+ f'/home/sweet/cylc-run/my-suite/dream{tail1}{tail2}',
+ func('my-suite/dream', *args),
+ )
+ mocked.get_host_item.assert_called_with(cfg)
+ mocked.get_host_item.reset_mock()
+
+ @patch('cylc.flow.pathutil.glbl_cfg')
+ def test_get_suite_run_names(self, mocked_glbl_cfg):
+ """Usage of get_suite_run_*name."""
+ mocked = MagicMock()
+ mocked_glbl_cfg.return_value = mocked
+ mocked.get_host_item.return_value = '/home/sweet/cylc-run'
+ # func = get_remote_* function to test
+ # cfg = configuration used in mocked global configuration
+ # tail1 = expected tail of return value from configuration
+ for func, cfg, tail1 in (
+ (get_suite_run_log_name, 'run directory', '/log/suite/log'),
+ (get_suite_run_pub_db_name, 'run directory', '/log/db'),
+ ):
+ self.assertEqual(
+ f'/home/sweet/cylc-run/my-suite/dream{tail1}',
+ func('my-suite/dream'),
+ )
+ mocked.get_host_item.assert_called_with(cfg)
+ mocked.get_host_item.reset_mock()
+
+ @patch('cylc.flow.pathutil.os.makedirs')
+ @patch('cylc.flow.pathutil.glbl_cfg')
+ def test_make_suite_run_tree(self, mocked_glbl_cfg, mocked_makedirs):
+ """Usage of make_suite_run_tree."""
+ mocked = MagicMock()
+ mocked_glbl_cfg.return_value = mocked
+ mocked.get_host_item.return_value = '/home/sweet/cylc-run'
+ mocked_cfg = MagicMock()
+ mocked_cfg['run directory rolling archive length'] = 0
+ mocked.get.return_value = mocked_cfg
+ make_suite_run_tree('my-suite/dream')
+ self.assertEqual(mocked_makedirs.call_count, 6)
+ mocked_makedirs.assert_has_calls((
+ call(f'/home/sweet/cylc-run/my-suite/dream{tail}', exist_ok=True)
+ for tail in (
+ '',
+ '/log/suite',
+ '/log/job',
+ '/log/suiterc',
+ '/share',
+ '/work',
+ )
+ ))
+
+
+if __name__ == '__main__':
+ from unittest import main
+ main()
diff --git a/tests/host-select/01-timeout.t b/tests/host-select/01-timeout.t
deleted file mode 100644
index 0245384103c..00000000000
--- a/tests/host-select/01-timeout.t
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#-------------------------------------------------------------------------------
-# Test host selection, with a command that times out.
-. "$(dirname "$0")/test_header"
-
-set_test_number 3
-
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-create_test_globalrc 'task host select command timeout = PT1S'
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
-suite_run_ok "${TEST_NAME_BASE}" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
-grep_ok 'ERROR - \[jobs-submit cmd\] (remote host select)' \
- "$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/log/suite/log"
-purge_suite "${SUITE_NAME}"
-exit
diff --git a/tests/job-submission/06-garbage.t b/tests/job-submission/06-garbage.t
index 654e6189541..4fc6349f3ee 100755
--- a/tests/job-submission/06-garbage.t
+++ b/tests/job-submission/06-garbage.t
@@ -20,7 +20,7 @@
set_test_number 2
install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-if [[ -n "${PYTHONPATH}" ]]; then
+if [[ -n "${PYTHONPATH:-}" ]]; then
export PYTHONPATH="${PWD}/lib:${PYTHONPATH}"
else
export PYTHONPATH="${PWD}/lib"
diff --git a/tests/rnd/02-lib-python-in-job/suite.rc b/tests/rnd/02-lib-python-in-job/suite.rc
index 9284fe88d54..2ed2a06dc11 100644
--- a/tests/rnd/02-lib-python-in-job/suite.rc
+++ b/tests/rnd/02-lib-python-in-job/suite.rc
@@ -12,7 +12,7 @@
grep -q "${CYLC_SUITE_RUN_DIR}/lib/python" <<< "${PYTHONPATH}"
# run a toy example
- python -c '
+ python3 -c '
from pub import beer
assert beer.drink() == "98 bottles of beer on the wall."
assert beer.drink() == "97 bottles of beer on the wall."