diff --git a/bin/cylc-cat-log b/bin/cylc-cat-log index 3f9332f3088..c3cc3c535d6 100755 --- a/bin/cylc-cat-log +++ b/bin/cylc-cat-log @@ -53,15 +53,20 @@ import shlex from glob import glob from shlex import quote from stat import S_IRUSR -from tempfile import mkstemp +from tempfile import NamedTemporaryFile from subprocess import Popen, PIPE +from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.exceptions import UserInputError import cylc.flow.flags +from cylc.flow.hostuserutil import is_remote from cylc.flow.option_parsers import CylcOptionParser as COP +from cylc.flow.pathutil import ( + get_remote_suite_run_job_dir, + get_suite_run_job_dir, + get_suite_run_log_name, + get_suite_run_pub_db_name) from cylc.flow.rundb import CylcSuiteDAO -from cylc.flow.hostuserutil import is_remote -from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.task_id import TaskID from cylc.flow.task_job_logs import ( JOB_LOG_OUT, JOB_LOG_ERR, JOB_LOG_OPTS, NN, JOB_LOGS_LOCAL) @@ -149,14 +154,14 @@ def view_log(logpath, mode, tailer_tmpl, batchview_cmd=None, remote=False): elif not remote and mode == 'edit': # Copy the log to a temporary read-only file for viewing in editor. # Copy only BUFSIZE bytes at time, in case the file is huge. - outfile = mkstemp(dir=glbl_cfg().get_tmpdir())[1] + outfile = NamedTemporaryFile() with open(logpath, 'rb') as log: - with open(outfile, 'wb') as out: + data = log.read(BUFSIZE) + while data: + outfile.write(data) data = log.read(BUFSIZE) - while data: - out.write(data) - data = log.read(BUFSIZE) - os.chmod(outfile, S_IRUSR) + os.chmod(outfile.name, S_IRUSR) + outfile.seek(0, 0) return outfile elif mode == 'cat' or (remote and mode == 'edit'): # Just cat file contents to stdout. @@ -234,11 +239,7 @@ def get_task_job_attrs(suite_name, point, task, submit_num): """ suite_dao = CylcSuiteDAO( - os.path.join( - glbl_cfg().get_derived_host_item( - suite_name, "suite run directory"), - "log", CylcSuiteDAO.DB_FILE_BASE_NAME), - is_public=True) + get_suite_run_pub_db_name(suite_name), is_public=True) task_job_data = suite_dao.select_task_job(point, task, submit_num) suite_dao.close() if task_job_data is None: @@ -264,17 +265,17 @@ def tmpfile_edit(tmpfile, geditor=False): editor = glbl_cfg().get(['editors', 'gui']) else: editor = glbl_cfg().get(['editors', 'terminal']) - modtime1 = os.stat(tmpfile).st_mtime + modtime1 = os.stat(tmpfile.name).st_mtime cmd = shlex.split(editor) - cmd.append(tmpfile) + cmd.append(tmpfile.name) proc = Popen(cmd, stderr=PIPE) err = proc.communicate()[1].decode() ret_code = proc.wait() if ret_code == 0: - if os.stat(tmpfile).st_mtime > modtime1: + if os.stat(tmpfile.name).st_mtime > modtime1: sys.stderr.write( 'WARNING: you edited a TEMPORARY COPY of %s\n' % ( - os.path.basename(tmpfile))) + os.path.basename(tmpfile.name))) if ret_code and err: sys.stderr.write(err) @@ -317,7 +318,7 @@ def main(): if len(args) == 1: # Cat suite logs, local only. - logpath = glbl_cfg().get_derived_host_item(suite_name, "suite log") + logpath = get_suite_run_log_name(suite_name) if options.rotation_num: logs = glob('%s.*' % logpath) logs.sort(key=os.path.getmtime, reverse=True) @@ -392,10 +393,9 @@ def main(): log_is_retrieved = (glbl_cfg().get_host_item('retrieve job logs', host) and live_job_id is None) if log_is_remote and (not log_is_retrieved or options.force_remote): - logpath = os.path.normpath(os.path.join( - glbl_cfg().get_derived_host_item( - suite_name, "suite job log directory", host, user), - point, task, options.submit_num, options.filename)) + logpath = os.path.normpath(get_remote_suite_run_job_dir( + host, user, + suite_name, point, task, options.submit_num, options.filename)) tail_tmpl = str(glbl_cfg().get_host_item( "tail command template", host, user)) # Reinvoke the cat-log command on the remote account. @@ -407,31 +407,29 @@ def main(): if batchview_cmd: cmd.append('--remote-arg=%s' % quote(batchview_cmd)) cmd.append(suite_name) - capture = (mode == 'edit') + is_edit_mode = (mode == 'edit') try: proc = remote_cylc_cmd( - cmd, user, host, capture_process=capture, + cmd, user, host, capture_process=is_edit_mode, manage=(mode == 'tail')) except KeyboardInterrupt: # Ctrl-C while tailing. pass else: - if capture: + if is_edit_mode: # Write remote stdout to a temp file for viewing in editor. # Only BUFSIZE bytes at a time in case huge stdout volume. - out = mkstemp(dir=glbl_cfg().get_tmpdir())[1] - with open(out, 'wb') as outf: + out = NamedTemporaryFile() + data = proc.stdout.read(BUFSIZE) + while data: + out.write(data) data = proc.stdout.read(BUFSIZE) - while data: - outf.write(data) - data = proc.stdout.read(BUFSIZE) - os.chmod(out, S_IRUSR) + os.chmod(out.name, S_IRUSR) + out.seek(0, 0) else: # Local task job or local job log. - logpath = os.path.normpath(os.path.join( - glbl_cfg().get_derived_host_item( - suite_name, "suite job log directory"), - point, task, options.submit_num, options.filename)) + logpath = os.path.normpath(get_suite_run_job_dir( + suite_name, point, task, options.submit_num, options.filename)) tail_tmpl = str(glbl_cfg().get_host_item("tail command template")) out = view_log(logpath, mode, tail_tmpl, batchview_cmd) if mode != 'edit': diff --git a/bin/cylc-ls-checkpoints b/bin/cylc-ls-checkpoints index 1562f6fcc4f..03fd85c26aa 100755 --- a/bin/cylc-ls-checkpoints +++ b/bin/cylc-ls-checkpoints @@ -30,8 +30,8 @@ if remrun(): import os -from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.option_parsers import CylcOptionParser as COP +from cylc.flow.pathutil import get_suite_run_pub_db_name from cylc.flow.rundb import CylcSuiteDAO from cylc.flow.terminal import cli_function @@ -98,12 +98,7 @@ def list_checkpoints(suite, callback): def _get_dao(suite): """Return the DAO (public) for suite.""" - - suite_log_dir = glbl_cfg().get_derived_host_item( - suite, 'suite log directory') - pub_db_path = os.path.join(os.path.dirname(suite_log_dir), - CylcSuiteDAO.DB_FILE_BASE_NAME) - return CylcSuiteDAO(pub_db_path, is_public=True) + return CylcSuiteDAO(get_suite_run_pub_db_name(suite), is_public=True) def _write_row(title, row_idx, row): diff --git a/bin/cylc-report-timings b/bin/cylc-report-timings index 2653cff3c75..eaccbc02e55 100755 --- a/bin/cylc-report-timings +++ b/bin/cylc-report-timings @@ -58,8 +58,8 @@ import contextlib import os from cylc.flow.exceptions import CylcError -from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.option_parsers import CylcOptionParser as COP +from cylc.flow.pathutil import get_suite_run_pub_db_name from cylc.flow.rundb import CylcSuiteDAO from cylc.flow.terminal import cli_function @@ -160,14 +160,7 @@ def format_rows(header, rows): def _get_dao(suite): """Return the DAO (public) for suite.""" - suite_log_dir = glbl_cfg().get_derived_host_item( - suite, 'suite log directory' - ) - pub_db_path = os.path.join( - os.path.dirname(suite_log_dir), - CylcSuiteDAO.DB_FILE_BASE_NAME - ) - return CylcSuiteDAO(pub_db_path, is_public=True) + return CylcSuiteDAO(get_suite_run_pub_db_name(suite), is_public=True) class TimingSummary(object): diff --git a/bin/cylc-submit b/bin/cylc-submit index 443aa8e99e3..b798b1b0b85 100755 --- a/bin/cylc-submit +++ b/bin/cylc-submit @@ -39,13 +39,13 @@ import os from time import sleep from cylc.flow import LOG -from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.config import SuiteConfig from cylc.flow.cycling.loader import get_point from cylc.flow.exceptions import UserInputError import cylc.flow.flags from cylc.flow.subprocpool import SubProcPool from cylc.flow.option_parsers import CylcOptionParser as COP +from cylc.flow.pathutil import make_suite_run_tree from cylc.flow.suite_db_mgr import SuiteDatabaseManager from cylc.flow.broadcast_mgr import BroadcastMgr from cylc.flow.suite_srv_files_mgr import SuiteSrvFilesManager @@ -106,7 +106,7 @@ def main(): taskdef, get_point(point_str).standardise(), is_startup=True)) # Initialise job submit environment - glbl_cfg().create_cylc_run_tree(suite) + make_suite_run_tree(suite) pool = SubProcPool() db_mgr = SuiteDatabaseManager() task_job_mgr = TaskJobManager( diff --git a/bin/cylc-view b/bin/cylc-view index 2f5da7eaad1..63175785b9d 100755 --- a/bin/cylc-view +++ b/bin/cylc-view @@ -42,10 +42,10 @@ from subprocess import call from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.option_parsers import CylcOptionParser as COP +from cylc.flow.parsec.fileparse import read_and_proc from cylc.flow.suite_srv_files_mgr import SuiteSrvFilesManager from cylc.flow.templatevars import load_template_vars from cylc.flow.terminal import cli_function -from cylc.flow.parsec.fileparse import read_and_proc def parse_args(): @@ -119,7 +119,6 @@ def main(): options, args = parse_args() suite, suiterc = SuiteSrvFilesManager().parse_suite_arg(options, args[0]) - cylc_tmpdir = glbl_cfg().get_tmpdir() if options.geditor: editor = glbl_cfg().get(['editors', 'gui']) else: @@ -148,7 +147,6 @@ def main(): # write to a temporary file viewfile = NamedTemporaryFile( suffix=".suite.rc", prefix=suite.replace('/', '_') + '.', - dir=cylc_tmpdir ) for line in lines: viewfile.write((line + '\n').encode()) diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 0ecb5236a2b..c8157cacab9 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -17,15 +17,11 @@ # along with this program. If not, see . """Cylc site and user configuration file spec.""" -import atexit import os import re -import shutil -from tempfile import mkdtemp from cylc.flow import LOG from cylc.flow import __version__ as CYLC_VERSION -from cylc.flow.exceptions import GlobalConfigError from cylc.flow.hostuserutil import get_user_home, is_remote_user from cylc.flow.network import Priv from cylc.flow.parsec.config import ParsecConfig @@ -41,21 +37,21 @@ # - default: the default value (optional). # - allowed_2, ...: the only other allowed values of this setting (optional). SPEC = { + # suite 'process pool size': [VDR.V_INTEGER, 4], 'process pool timeout': [VDR.V_INTERVAL, DurationFloat(600)], - 'temporary directory': [VDR.V_STRING], - 'state dump rolling archive length': [VDR.V_INTEGER, 10], + # client 'disable interactive command prompts': [VDR.V_BOOLEAN, True], - 'enable run directory housekeeping': [VDR.V_BOOLEAN], - 'run directory rolling archive length': [VDR.V_INTEGER, 2], - 'task host select command timeout': [VDR.V_INTERVAL, DurationFloat(10)], - 'xtrigger function timeout': [VDR.V_INTERVAL, DurationFloat(10)], + # suite + 'run directory rolling archive length': [VDR.V_INTEGER, -1], + # suite-task communication 'task messaging': { 'retry interval': [VDR.V_INTERVAL, DurationFloat(5)], 'maximum number of tries': [VDR.V_INTEGER, 7], 'connection timeout': [VDR.V_INTERVAL, DurationFloat(30)], }, + # suite 'cylc': { 'UTC mode': [VDR.V_BOOLEAN], 'health check interval': [VDR.V_INTERVAL, DurationFloat(600)], @@ -82,11 +78,13 @@ }, }, + # suite 'suite logging': { 'rolling archive length': [VDR.V_INTEGER, 5], 'maximum size in bytes': [VDR.V_INTEGER, 1000000], }, + # general 'documentation': { 'local': [VDR.V_STRING, ''], 'online': [VDR.V_STRING, @@ -94,19 +92,23 @@ 'cylc homepage': [VDR.V_STRING, 'http://cylc.github.io/'], }, + # general 'document viewers': { 'html': [VDR.V_STRING, 'firefox'], }, + # client 'editors': { 'terminal': [VDR.V_STRING, 'vim'], 'gui': [VDR.V_STRING, 'gvim -f'], }, + # client 'monitor': { 'sort order': [VDR.V_STRING, 'definition', 'alphanumeric'], }, + # task 'hosts': { 'localhost': { 'run directory': [VDR.V_STRING, '$HOME/cylc-run'], @@ -175,6 +177,7 @@ }, }, + # task 'task events': { 'execution timeout': [VDR.V_INTERVAL], 'handlers': [VDR.V_STRING_LIST], @@ -188,6 +191,7 @@ 'submission timeout': [VDR.V_INTERVAL], }, + # client 'test battery': { 'remote host with shared fs': [VDR.V_STRING], 'remote host': [VDR.V_STRING], @@ -204,12 +208,14 @@ }, }, + # suite 'suite host self-identification': { 'method': [VDR.V_STRING, 'name', 'address', 'hardwired'], 'target': [VDR.V_STRING, 'google.com'], 'host': [VDR.V_STRING], }, + # suite 'authentication': { # Allow owners to grant public shutdown rights at the most, not full # control. @@ -220,6 +226,7 @@ Priv.STATE_TOTALS, Priv.READ, Priv.SHUTDOWN]]) }, + # suite 'suite servers': { 'run hosts': [VDR.V_SPACELESS_STRING_LIST], 'run ports': [VDR.V_INTEGER_LIST, list(range(43001, 43101))], @@ -258,6 +265,10 @@ def upg(cfg, descr): u.obsolete('8.0.0', ['suite servers', 'scan hosts']) u.obsolete('8.0.0', ['suite servers', 'scan ports']) u.obsolete('8.0.0', ['communication']) + u.obsolete('8.0.0', ['temporary directory']) + u.obsolete('8.0.0', ['task host select command timeout']) + u.obsolete('8.0.0', ['xtrigger function timeout']) + u.obsolete('8.0.0', ['enable run directory housekeeping']) u.upgrade() @@ -266,9 +277,6 @@ class GlobalConfig(ParsecConfig): """ Handle global (all suites) site and user configuration for cylc. User file values override site file values. - - For all derived items - paths hardwired under the configurable top - levels - use the get_derived_host_item(suite,host) method. """ _DEFAULT = None @@ -328,49 +336,7 @@ def load(self): LOG.error('bad %s %s', conf_type, fname) raise # (OK if no flow.rc is found, just use system defaults). - self.transform() - - def get_derived_host_item( - self, suite, item, host=None, owner=None, replace_home=False): - """Compute hardwired paths relative to the configurable top dirs.""" - - # suite run dir - srdir = os.path.join( - self.get_host_item('run directory', host, owner, replace_home), - suite) - # suite workspace - swdir = os.path.join( - self.get_host_item('work directory', host, owner, replace_home), - suite) - - if item == 'suite run directory': - value = srdir - - elif item == 'suite log directory': - value = os.path.join(srdir, 'log', 'suite') - - elif item == 'suite log': - value = os.path.join(srdir, 'log', 'suite', 'log') - - elif item == 'suite job log directory': - value = os.path.join(srdir, 'log', 'job') - - elif item == 'suite config log directory': - value = os.path.join(srdir, 'log', 'suiterc') - - elif item == 'suite work root': - value = swdir - - elif item == 'suite work directory': - value = os.path.join(swdir, 'work') - - elif item == 'suite share directory': - value = os.path.join(swdir, 'share') - - else: - raise GlobalConfigError("Illegal derived item: " + item) - - return value + self._transform() def get_host_item(self, item, host=None, owner=None, replace_home=False, owner_home=None): @@ -420,85 +386,7 @@ def get_host_item(self, item, host=None, owner=None, replace_home=False, value = 'zmq' return value - def roll_directory(self, dir_, name, archlen=0): - """Create a directory after rolling back any previous instances of it. - - E.g. if archlen = 2 we keep: - dir_, dir_.1, dir_.2. If 0 keep no old ones. - """ - for i in range(archlen, -1, -1): # archlen...0 - if i > 0: - dpath = dir_ + '.' + str(i) - else: - dpath = dir_ - if os.path.exists(dpath): - if i >= archlen: - # remove oldest backup - shutil.rmtree(dpath) - else: - # roll others over - os.rename(dpath, dir_ + '.' + str(i + 1)) - self.create_directory(dir_, name) - - @staticmethod - def create_directory(dir_, name): - """Create directory. Raise GlobalConfigError on error.""" - try: - os.makedirs(dir_, exist_ok=True) - except OSError as exc: - LOG.exception(exc) - raise GlobalConfigError( - 'Failed to create directory "' + name + '"') - - def create_cylc_run_tree(self, suite): - """Create all top-level cylc-run output dirs on the suite host.""" - cfg = self.get() - item = 'suite run directory' - idir = self.get_derived_host_item(suite, item) - LOG.debug('creating %s: %s', item, idir) - if cfg['enable run directory housekeeping']: - self.roll_directory( - idir, item, cfg['run directory rolling archive length']) - - for item in [ - 'suite log directory', - 'suite job log directory', - 'suite config log directory', - 'suite work directory', - 'suite share directory']: - idir = self.get_derived_host_item(suite, item) - LOG.debug('creating %s: %s', item, idir) - self.create_directory(idir, item) - - item = 'temporary directory' - value = cfg[item] - if value: - self.create_directory(value, item) - - def get_tmpdir(self): - """Make a new temporary directory and arrange for it to be - deleted automatically when we're finished with it. Call this - explicitly just before use to ensure the directory is not - deleted by other processes before it is needed. THIS IS - CURRENTLY ONLY USED BY A FEW CYLC COMMANDS. If cylc suites - ever need it this must be called AFTER FORKING TO DAEMON MODE or - atexit() will delete the directory when the initial process - exits after forking.""" - - cfg = self.get() - tdir = cfg['temporary directory'] - if tdir: - tdir = os.path.expandvars(tdir) - tmpdir = mkdtemp(prefix="cylc-", dir=os.path.expandvars(tdir)) - else: - tmpdir = mkdtemp(prefix="cylc-") - # self-cleanup - atexit.register(lambda: shutil.rmtree(tmpdir)) - # now replace the original item to allow direct access - cfg['temporary directory'] = tmpdir - return tmpdir - - def transform(self): + def _transform(self): """Transform various settings. Host item values of None default to modified localhost values. diff --git a/cylc/flow/config.py b/cylc/flow/config.py index 6e7d181d6fa..0a92554623f 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -56,6 +56,12 @@ from cylc.flow.cycling.iso8601 import ingest_time import cylc.flow.flags from cylc.flow.graphnode import GraphNodeParser +from cylc.flow.pathutil import ( + get_suite_run_dir, + get_suite_run_log_dir, + get_suite_run_share_dir, + get_suite_run_work_dir, +) from cylc.flow.print_tree import print_tree from cylc.flow.subprocctx import SubFuncContext from cylc.flow.subprocpool import get_func @@ -119,14 +125,10 @@ def __init__(self, suite, fpath, template_vars=None, self.suite = suite # suite name self.fpath = fpath # suite definition self.fdir = os.path.dirname(fpath) - self.run_dir = run_dir or glbl_cfg().get_derived_host_item( - self.suite, 'suite run directory') - self.log_dir = log_dir or glbl_cfg().get_derived_host_item( - self.suite, 'suite log directory') - self.work_dir = work_dir or glbl_cfg().get_derived_host_item( - self.suite, 'suite work directory') - self.share_dir = share_dir or glbl_cfg().get_derived_host_item( - self.suite, 'suite share directory') + self.run_dir = run_dir or get_suite_run_dir(self.suite) + self.log_dir = log_dir or get_suite_run_log_dir(self.suite) + self.share_dir = share_dir or get_suite_run_share_dir(self.suite) + self.work_dir = work_dir or get_suite_run_work_dir(self.suite) self.owner = owner self.run_mode = run_mode self.strict = strict diff --git a/cylc/flow/daemonize.py b/cylc/flow/daemonize.py index 31cef52ab5b..4f81fce4fe9 100644 --- a/cylc/flow/daemonize.py +++ b/cylc/flow/daemonize.py @@ -22,7 +22,7 @@ import sys from time import sleep, time -from cylc.flow.cfgspec.glbl_cfg import glbl_cfg +from cylc.flow.pathutil import get_suite_run_log_name SUITE_SCAN_INFO_TMPL = r""" @@ -53,9 +53,9 @@ def daemonize(server): http://code.activestate.com/recipes/66012-fork-a-daemon-process-on-unix/ """ - logpath = glbl_cfg().get_derived_host_item(server.suite, 'suite log') + logfname = get_suite_run_log_name(server.suite) try: - old_log_mtime = os.stat(logpath).st_mtime + old_log_mtime = os.stat(logfname).st_mtime except OSError: old_log_mtime = None # fork 1 @@ -75,11 +75,11 @@ def daemonize(server): # LOG-PREFIX Suite server program: url=URL, pid=PID # Otherwise, something has gone wrong, print the suite log # and exit with an error. - log_stat = os.stat(logpath) + log_stat = os.stat(logfname) if (log_stat.st_mtime == old_log_mtime or log_stat.st_size == 0): continue - for line in open(logpath): + for line in open(logfname): if server.START_MESSAGE_PREFIX in line: suite_url, suite_pid = ( item.rsplit("=", 1)[-1] @@ -88,7 +88,7 @@ def daemonize(server): elif ' ERROR -' in line or ' CRITICAL -' in line: # ERROR and CRITICAL before suite starts try: - sys.stderr.write(open(logpath).read()) + sys.stderr.write(open(logfname).read()) sys.exit(1) except IOError: sys.exit("Suite server program exited") diff --git a/cylc/flow/etc/job.sh b/cylc/flow/etc/job.sh index c7c7261e9dc..ae11ead17ec 100644 --- a/cylc/flow/etc/job.sh +++ b/cylc/flow/etc/job.sh @@ -121,8 +121,7 @@ cylc__job__main() { # System paths: # * suite directory (installed run-dir first). export PATH="${CYLC_SUITE_RUN_DIR}/bin:${CYLC_SUITE_DEF_PATH}/bin:${PATH}" - export - PYTHONPATH="${CYLC_SUITE_RUN_DIR}/lib/python:${CYLC_SUITE_DEF_PATH}/lib/python:${PYTHONPATH:-}" + export PYTHONPATH="${CYLC_SUITE_RUN_DIR}/lib/python:${CYLC_SUITE_DEF_PATH}/lib/python:${PYTHONPATH:-}" # Create share and work directories mkdir -p "${CYLC_SUITE_SHARE_DIR}" || true mkdir -p "$(dirname "${CYLC_TASK_WORK_DIR}")" || true diff --git a/cylc/flow/exceptions.py b/cylc/flow/exceptions.py index ec963ae7025..31bd6b49201 100644 --- a/cylc/flow/exceptions.py +++ b/cylc/flow/exceptions.py @@ -55,10 +55,6 @@ class SuiteConfigError(CylcConfigError): """Exception for configuration errors in a Cylc suite configuration.""" -class GlobalConfigError(CylcConfigError): - """Exception for configuration errors in a Cylc global configuration.""" - - class GraphParseError(SuiteConfigError): """Exception for errors in Cylc suite graphing.""" diff --git a/cylc/flow/job_file.py b/cylc/flow/job_file.py index 4037551a94c..540774e9a25 100644 --- a/cylc/flow/job_file.py +++ b/cylc/flow/job_file.py @@ -26,6 +26,9 @@ from cylc.flow.batch_sys_manager import BatchSysManager from cylc.flow.cfgspec.glbl_cfg import glbl_cfg import cylc.flow.flags +from cylc.flow.pathutil import ( + get_remote_suite_run_dir, + get_remote_suite_work_dir) class JobFileWriter(object): @@ -56,7 +59,8 @@ def write(self, local_job_file_path, job_conf, check_syntax=True): # variables: NEXT_CYCLE=$( cylc cycle-point --offset-hours=6 ) tmp_name = local_job_file_path + '.tmp' - run_d = self._get_derived_host_item(job_conf, 'suite run directory') + run_d = get_remote_suite_run_dir( + job_conf['host'], job_conf['owner'], job_conf['suite_name']) try: with open(tmp_name, 'w') as handle: self._write_header(handle, job_conf) @@ -116,12 +120,6 @@ def _check_script_value(value): return True return False - @staticmethod - def _get_derived_host_item(job_conf, key): - """Return derived host item from glbl_cfg().""" - return glbl_cfg().get_derived_host_item( - job_conf['suite_name'], key, job_conf["host"], job_conf["owner"]) - @staticmethod def _get_host_item(job_conf, key): """Return host item from glbl_cfg().""" @@ -194,7 +192,8 @@ def _write_environment_1(self, handle, job_conf, run_d): handle.write('\n') # override and write task-host-specific suite variables - work_d = self._get_derived_host_item(job_conf, 'suite work root') + work_d = get_remote_suite_work_dir( + job_conf["host"], job_conf["owner"], job_conf['suite_name']) handle.write('\n export CYLC_SUITE_RUN_DIR="%s"' % run_d) if work_d != run_d: # Note: not an environment variable, but used by job.sh diff --git a/cylc/flow/loggingutil.py b/cylc/flow/loggingutil.py index 08da1727390..7dfb7947b41 100644 --- a/cylc/flow/loggingutil.py +++ b/cylc/flow/loggingutil.py @@ -31,6 +31,7 @@ from cylc.flow.cfgspec.glbl_cfg import glbl_cfg +from cylc.flow.pathutil import get_suite_run_log_name from cylc.flow.wallclock import ( get_current_time_string, get_time_string_from_unix_time) @@ -77,8 +78,7 @@ class TimestampRotatingFileHandler(logging.FileHandler): MIN_BYTES = 1024 def __init__(self, suite, no_detach=False): - logging.FileHandler.__init__( - self, glbl_cfg().get_derived_host_item(suite, 'suite log')) + logging.FileHandler.__init__(self, get_suite_run_log_name(suite)) self.no_detach = no_detach self.stamp = None self.formatter = CylcLogFormatter() diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py new file mode 100644 index 00000000000..678b294995f --- /dev/null +++ b/cylc/flow/pathutil.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Functions to return paths to common suite files and directories.""" + +import os +from shutil import rmtree + + +from cylc.flow import LOG +from cylc.flow.cfgspec.glbl_cfg import glbl_cfg + + +def get_remote_suite_run_dir(host, owner, suite, *args): + """Return remote suite run directory, join any extra args.""" + return os.path.join( + glbl_cfg().get_host_item('run directory', host, owner), suite, *args) + + +def get_remote_suite_run_job_dir(host, owner, suite, *args): + """Return remote suite run directory, join any extra args.""" + return get_remote_suite_run_dir( + host, owner, suite, 'log', 'job', *args) + + +def get_remote_suite_work_dir(host, owner, suite, *args): + """Return remote suite work directory root, join any extra args.""" + return os.path.join( + glbl_cfg().get_host_item('work directory', host, owner), + suite, + *args) + + +def get_suite_run_dir(suite, *args): + """Return local suite run directory, join any extra args.""" + return os.path.join( + glbl_cfg().get_host_item('run directory'), suite, *args) + + +def get_suite_run_job_dir(suite, *args): + """Return suite run job (log) directory, join any extra args.""" + return get_suite_run_dir(suite, 'log', 'job', *args) + + +def get_suite_run_log_dir(suite, *args): + """Return suite run log directory, join any extra args.""" + return get_suite_run_dir(suite, 'log', 'suite', *args) + + +def get_suite_run_log_name(suite): + """Return suite run log file path.""" + return get_suite_run_dir(suite, 'log', 'suite', 'log') + + +def get_suite_run_rc_dir(suite, *args): + """Return suite run suite.rc log directory, join any extra args.""" + return get_suite_run_dir(suite, 'log', 'suiterc', *args) + + +def get_suite_run_pub_db_name(suite): + """Return suite run public database file path.""" + return get_suite_run_dir(suite, 'log', 'db') + + +def get_suite_run_share_dir(suite, *args): + """Return local suite work/share directory, join any extra args.""" + return os.path.join( + glbl_cfg().get_host_item('work directory'), suite, 'share', *args) + + +def get_suite_run_work_dir(suite, *args): + """Return local suite work/work directory, join any extra args.""" + return os.path.join( + glbl_cfg().get_host_item('work directory'), suite, 'work', *args) + + +def make_suite_run_tree(suite): + """Create all top-level cylc-run output dirs on the suite host.""" + cfg = glbl_cfg().get() + # Roll archive + archlen = cfg['run directory rolling archive length'] + dir_ = get_suite_run_dir(suite) + for i in range(archlen, -1, -1): # archlen...0 + if i > 0: + dpath = dir_ + '.' + str(i) + else: + dpath = dir_ + if os.path.exists(dpath): + if i >= archlen: + # remove oldest backup + rmtree(dpath) + else: + # roll others over + os.rename(dpath, dir_ + '.' + str(i + 1)) + # Create + for dir_ in ( + get_suite_run_dir(suite), + get_suite_run_log_dir(suite), + get_suite_run_job_dir(suite), + get_suite_run_rc_dir(suite), + get_suite_run_share_dir(suite), + get_suite_run_work_dir(suite), + ): + if dir_: + os.makedirs(dir_, exist_ok=True) + LOG.debug('%s: directory created', dir_) diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py index fdc4489018d..088b09838e7 100644 --- a/cylc/flow/scheduler.py +++ b/cylc/flow/scheduler.py @@ -48,6 +48,14 @@ ReferenceLogFileHandler from cylc.flow.log_diagnosis import LogSpec from cylc.flow.network.server import SuiteRuntimeServer +from cylc.flow.pathutil import ( + get_suite_run_dir, + get_suite_run_log_dir, + get_suite_run_rc_dir, + get_suite_run_share_dir, + get_suite_run_work_dir, + make_suite_run_tree, +) from cylc.flow.profiler import Profiler from cylc.flow.state_summary_mgr import StateSummaryMgr from cylc.flow.subprocpool import SubProcPool @@ -145,14 +153,10 @@ def __init__(self, is_restart, options, args): # For user-defined batch system handlers sys.path.append(os.path.join(self.suite_dir, 'python')) sys.path.append(os.path.join(self.suite_dir, 'lib', 'python')) - self.suite_run_dir = glbl_cfg().get_derived_host_item( - self.suite, 'suite run directory') - self.suite_work_dir = glbl_cfg().get_derived_host_item( - self.suite, 'suite work directory') - self.suite_share_dir = glbl_cfg().get_derived_host_item( - self.suite, 'suite share directory') - self.suite_log_dir = glbl_cfg().get_derived_host_item( - self.suite, 'suite log directory') + self.suite_run_dir = get_suite_run_dir(self.suite) + self.suite_work_dir = get_suite_run_work_dir(self.suite) + self.suite_share_dir = get_suite_run_share_dir(self.suite) + self.suite_log_dir = get_suite_run_log_dir(self.suite) self.config = None @@ -240,7 +244,7 @@ def start(self): """Start the server.""" self._start_print_blurb() - glbl_cfg().create_cylc_run_tree(self.suite) + make_suite_run_tree(self.suite) if self.is_restart: self.suite_db_mgr.restart_upgrade() @@ -983,8 +987,6 @@ def load_suiterc(self, is_reload=False): ) self.suiterc_update_time = time() # Dump the loaded suiterc for future reference. - cfg_logdir = glbl_cfg().get_derived_host_item( - self.suite, 'suite config log directory') time_str = get_current_time_string( override_use_utc=True, use_basic_format=True, display_sub_seconds=False @@ -995,8 +997,8 @@ def load_suiterc(self, is_reload=False): load_type = "restart" else: load_type = "run" - base_name = "%s-%s.rc" % (time_str, load_type) - file_name = os.path.join(cfg_logdir, base_name) + file_name = get_suite_run_rc_dir( + self.suite, f"{time_str}-{load_type}.rc") with open(file_name, "wb") as handle: handle.write(b"# cylc-version: %s\n" % CYLC_VERSION.encode()) printcfg(self.config.cfg, none_str=None, handle=handle) diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py index 2c60eeb093f..3bbe0b2f3af 100755 --- a/cylc/flow/scheduler_cli.py +++ b/cylc/flow/scheduler_cli.py @@ -20,11 +20,11 @@ import os import sys -from cylc.flow.cfgspec.glbl_cfg import glbl_cfg import cylc.flow.flags from cylc.flow.host_appointer import HostAppointer, EmptyHostList from cylc.flow.hostuserutil import is_remote_host from cylc.flow.option_parsers import CylcOptionParser as COP +from cylc.flow.pathutil import get_suite_run_dir from cylc.flow.remote import remrun, remote_cylc_cmd from cylc.flow.scheduler import Scheduler from cylc.flow.suite_srv_files_mgr import ( @@ -123,9 +123,7 @@ def main(is_restart=False): SuiteSrvFilesManager().get_suite_source_dir(args[0], options.owner) except SuiteServiceFileError: # Source path is assumed to be the run directory - SuiteSrvFilesManager().register( - args[0], - glbl_cfg().get_derived_host_item(args[0], 'suite run directory')) + SuiteSrvFilesManager().register(args[0], get_suite_run_dir(args[0])) try: scheduler = Scheduler(is_restart, options, args) diff --git a/cylc/flow/suite_srv_files_mgr.py b/cylc/flow/suite_srv_files_mgr.py index 3b7ff3c029d..3b5e09f2e8f 100644 --- a/cylc/flow/suite_srv_files_mgr.py +++ b/cylc/flow/suite_srv_files_mgr.py @@ -26,6 +26,7 @@ from cylc.flow import LOG from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.exceptions import SuiteServiceFileError +from cylc.flow.pathutil import get_remote_suite_run_dir, get_suite_run_dir from cylc.flow.pkg_resources import extract_pkg_resources import cylc.flow.flags from cylc.flow.hostuserutil import ( @@ -345,8 +346,7 @@ def get_suite_srv_dir(self, reg, suite_owner=None): run_d = os.getenv("CYLC_SUITE_RUN_DIR") if (not run_d or os.getenv("CYLC_SUITE_NAME") != reg or os.getenv("CYLC_SUITE_OWNER") != suite_owner): - run_d = glbl_cfg().get_derived_host_item( - reg, 'suite run directory') + run_d = get_suite_run_dir(reg) return os.path.join(run_d, self.DIR_BASE_SRV) def list_suites(self, regfilter=None): @@ -615,9 +615,7 @@ def _load_remote_item(self, item, reg, owner, host): if item == self.FILE_BASE_CONTACT and not is_remote_host(host): # Attempt to read suite contact file via the local filesystem. path = r'%(run_d)s/%(srv_base)s' % { - 'run_d': glbl_cfg().get_derived_host_item( - reg, 'suite run directory', 'localhost', owner, - replace_home=False), + 'run_d': get_remote_suite_run_dir('localhost', owner, reg), 'srv_base': self.DIR_BASE_SRV, } content = self._load_local_item(item, path) @@ -632,8 +630,7 @@ def _load_remote_item(self, item, reg, owner, host): r'''cat "%(run_d)s/%(srv_base)s/%(item)s"''' ) % { 'prefix': prefix, - 'run_d': glbl_cfg().get_derived_host_item( - reg, 'suite run directory', host, owner), + 'run_d': get_remote_suite_run_dir(host, owner, reg), 'srv_base': self.DIR_BASE_SRV, 'item': item } diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py index ffb3bcac551..c159a0c9e3f 100644 --- a/cylc/flow/task_events_mgr.py +++ b/cylc/flow/task_events_mgr.py @@ -39,6 +39,9 @@ from cylc.flow import LOG from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.hostuserutil import get_host, get_user +from cylc.flow.pathutil import ( + get_remote_suite_run_job_dir, + get_suite_run_job_dir) from cylc.flow.subprocctx import SubProcContext from cylc.flow.task_action_timer import TaskActionTimer from cylc.flow.task_job_logs import ( @@ -577,11 +580,11 @@ def _process_job_logs_retrieval(self, schd_ctx, ctx, id_keys): cmd += ["--include=%s" % (include) for include in sorted(includes)] cmd.append("--exclude=/**") # exclude everything else # Remote source - cmd.append(ctx.user_at_host + ":" + glbl_cfg().get_derived_host_item( - schd_ctx.suite, "suite job log directory", s_host, s_user) + "/") + cmd.append("%s:%s/" % ( + ctx.user_at_host, + get_remote_suite_run_job_dir(s_host, s_user, schd_ctx.suite))) # Local target - cmd.append(glbl_cfg().get_derived_host_item( - schd_ctx.suite, "suite job log directory") + "/") + cmd.append(get_suite_run_job_dir(schd_ctx.suite) + "/") self.proc_pool.put_command( SubProcContext(ctx, cmd, env=dict(os.environ), id_keys=id_keys), self._job_logs_retrieval_callback, [schd_ctx]) diff --git a/cylc/flow/task_job_logs.py b/cylc/flow/task_job_logs.py index 7a034f8f9f7..5b4760ae2ac 100644 --- a/cylc/flow/task_job_logs.py +++ b/cylc/flow/task_job_logs.py @@ -18,7 +18,7 @@ """Define task job log filenames and option names.""" import os -from cylc.flow.cfgspec.glbl_cfg import glbl_cfg +from cylc.flow.pathutil import get_suite_run_job_dir # Task job log filenames. JOB_LOG_JOB = "job" @@ -56,7 +56,7 @@ def get_task_job_id(point, name, submit_num=None): def get_task_job_log(suite, point, name, submit_num=None, suffix=None): """Return the full job log path.""" args = [ - glbl_cfg().get_derived_host_item(suite, "suite job log directory"), + get_suite_run_job_dir(suite), get_task_job_id(point, name, submit_num)] if suffix is not None: args.append(suffix) diff --git a/cylc/flow/task_job_mgr.py b/cylc/flow/task_job_mgr.py index fd4f7912c49..640dc996e64 100644 --- a/cylc/flow/task_job_mgr.py +++ b/cylc/flow/task_job_mgr.py @@ -37,17 +37,17 @@ from cylc.flow import LOG from cylc.flow.batch_sys_manager import JobPollContext -from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.hostuserutil import get_host, is_remote_host, is_remote_user from cylc.flow.job_file import JobFileWriter -from cylc.flow.task_job_logs import ( - JOB_LOG_JOB, get_task_job_log, get_task_job_job_log, - get_task_job_activity_log, get_task_job_id, NN) +from cylc.flow.pathutil import get_remote_suite_run_job_dir from cylc.flow.subprocpool import SubProcPool from cylc.flow.subprocctx import SubProcContext from cylc.flow.task_action_timer import TaskActionTimer from cylc.flow.task_events_mgr import TaskEventsManager, log_task_job_activity from cylc.flow.task_message import FAIL_MESSAGE_PREFIX +from cylc.flow.task_job_logs import ( + JOB_LOG_JOB, get_task_job_log, get_task_job_job_log, + get_task_job_activity_log, get_task_job_id, NN) from cylc.flow.task_outputs import ( TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_FAILED) @@ -282,8 +282,7 @@ def submit_task_jobs(self, suite, itasks, is_simulation=False): if remote_mode: cmd.append('--remote-mode') cmd.append('--') - cmd.append(glbl_cfg().get_derived_host_item( - suite, 'suite job log directory', host, owner)) + cmd.append(get_remote_suite_run_job_dir(host, owner, suite)) # Chop itasks into a series of shorter lists if it's very big # to prevent overloading of stdout and stderr pipes. itasks = sorted(itasks, key=lambda itask: itask.identity) @@ -651,8 +650,7 @@ def _run_job_cmd(self, cmd_key, suite, itasks, callback): if is_remote_user(owner): cmd.append("--user=%s" % (owner)) cmd.append("--") - cmd.append(glbl_cfg().get_derived_host_item( - suite, "suite job log directory", host, owner)) + cmd.append(get_remote_suite_run_job_dir(host, owner, suite)) job_log_dirs = [] for itask in sorted(itasks, key=lambda itask: itask.identity): job_log_dirs.append(get_task_job_id( @@ -852,11 +850,8 @@ def _prep_submit_task_job_impl(self, suite, itask, rtconfig): self._create_job_log_path(suite, itask) job_d = get_task_job_id( itask.point, itask.tdef.name, itask.submit_num) - job_file_path = os.path.join( - glbl_cfg().get_derived_host_item( - suite, "suite job log directory", - itask.task_host, itask.task_owner), - job_d, JOB_LOG_JOB) + job_file_path = get_remote_suite_run_job_dir( + itask.task_host, itask.task_owner, suite, job_d, JOB_LOG_JOB) return { 'batch_system_name': rtconfig['job']['batch system'], 'batch_submit_command_template': ( diff --git a/cylc/flow/task_message.py b/cylc/flow/task_message.py index f55ebd81ec4..bc79694a001 100644 --- a/cylc/flow/task_message.py +++ b/cylc/flow/task_message.py @@ -28,9 +28,9 @@ import sys -from cylc.flow.cfgspec.glbl_cfg import glbl_cfg import cylc.flow.flags from cylc.flow.network.client import SuiteRuntimeClient +from cylc.flow.pathutil import get_suite_run_job_dir from cylc.flow.task_outputs import TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED from cylc.flow.wallclock import get_current_time_string @@ -93,9 +93,7 @@ def _append_job_status_file(suite, task_job, event_time, messages): """Write messages to job status file.""" job_log_name = os.getenv('CYLC_TASK_LOG_ROOT') if not job_log_name: - job_log_name = os.path.join( - glbl_cfg().get_derived_host_item(suite, 'suite job log directory'), - 'job') + job_log_name = get_suite_run_job_dir(suite, task_job, 'job') try: job_status_file = open(job_log_name + '.status', 'a') except IOError: diff --git a/cylc/flow/task_remote_mgr.py b/cylc/flow/task_remote_mgr.py index cd1cbe634ea..12e00354115 100644 --- a/cylc/flow/task_remote_mgr.py +++ b/cylc/flow/task_remote_mgr.py @@ -36,6 +36,7 @@ from cylc.flow.exceptions import TaskRemoteMgmtError import cylc.flow.flags from cylc.flow.hostuserutil import is_remote, is_remote_host, is_remote_user +from cylc.flow.pathutil import get_remote_suite_run_dir from cylc.flow.subprocctx import SubProcContext from cylc.flow.task_remote_cmd import ( FILE_BASE_UUID, REMOTE_INIT_DONE, REMOTE_INIT_NOT_REQUIRED) @@ -95,14 +96,11 @@ def remote_host_select(self, host_str): host_str = value # command succeeded else: # Command not launched (or already reset) - timeout = glbl_cfg().get(['task host select command timeout']) - if timeout: - cmd = ['timeout', str(int(timeout)), 'bash', '-c', cmd_str] - else: - cmd = ['bash', '-c', cmd_str] self.proc_pool.put_command( SubProcContext( - 'remote-host-select', cmd, env=dict(os.environ)), + 'remote-host-select', + ['bash', '-c', cmd_str], + env=dict(os.environ)), self._remote_host_select_callback, [cmd_str]) self.remote_host_str_map[cmd_str] = None return self.remote_host_str_map[cmd_str] @@ -198,8 +196,7 @@ def remote_init(self, host, owner): if comm_meth in ['ssh']: cmd.append('--indirect-comm=%s' % comm_meth) cmd.append(str(self.uuid_str)) - cmd.append(glbl_cfg().get_derived_host_item( - self.suite, 'suite run directory', host, owner)) + cmd.append(get_remote_suite_run_dir(host, owner, self.suite)) self.proc_pool.put_command( SubProcContext('remote-init', cmd, stdin_files=[tmphandle]), self._remote_init_callback, @@ -237,8 +234,7 @@ def remote_tidy(self): cmd.append('--user=%s' % owner) if cylc.flow.flags.debug: cmd.append('--debug') - cmd.append(os.path.join(glbl_cfg().get_derived_host_item( - self.suite, 'suite run directory', host, owner))) + cmd.append(get_remote_suite_run_dir(host, owner, self.suite)) procs[(host, owner)] = ( cmd, Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=open(os.devnull))) diff --git a/cylc/flow/tests/test_loggingutil.py b/cylc/flow/tests/test_loggingutil.py index 2f611db07cb..4a297f24bb5 100644 --- a/cylc/flow/tests/test_loggingutil.py +++ b/cylc/flow/tests/test_loggingutil.py @@ -28,8 +28,13 @@ class TestLoggingutil(unittest.TestCase): + @mock.patch("cylc.flow.loggingutil.get_suite_run_log_name") @mock.patch("cylc.flow.loggingutil.glbl_cfg") - def test_value_error_raises_system_exit(self, mocked_glbl_cfg): + def test_value_error_raises_system_exit( + self, + mocked_glbl_cfg, + mocked_get_suite_run_log_name, + ): """Test that a ValueError when writing to a log stream won't result in multiple exceptions (what could lead to infinite loop in some occasions. Instead, it **must** raise a SystemExit""" @@ -37,8 +42,8 @@ def test_value_error_raises_system_exit(self, mocked_glbl_cfg): # mock objects used when creating the file handler mocked = mock.MagicMock() mocked_glbl_cfg.return_value = mocked - mocked.get_derived_host_item.return_value = tf.name mocked.get.return_value = 100 + mocked_get_suite_run_log_name.return_value = tf.name file_handler = TimestampRotatingFileHandler("suiteA", False) # next line is important as pytest can have a "Bad file descriptor" # due to a FileHandler with default "a" (pytest tries to r/w). diff --git a/cylc/flow/tests/test_pathutil.py b/cylc/flow/tests/test_pathutil.py new file mode 100644 index 00000000000..6a5f1c53457 --- /dev/null +++ b/cylc/flow/tests/test_pathutil.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Tests for "cylc.flow.pathutil".""" + +from unittest import TestCase +from unittest.mock import call, patch, MagicMock + +from cylc.flow.pathutil import ( + get_remote_suite_run_dir, + get_remote_suite_run_job_dir, + get_remote_suite_work_dir, + get_suite_run_dir, + get_suite_run_job_dir, + get_suite_run_log_dir, + get_suite_run_log_name, + get_suite_run_pub_db_name, + get_suite_run_rc_dir, + get_suite_run_share_dir, + get_suite_run_work_dir, + make_suite_run_tree, +) + + +class TestPathutil(TestCase): + """Tests for functions in "cylc.flow.pathutil".""" + + @patch('cylc.flow.pathutil.glbl_cfg') + def test_get_remote_suite_run_dirs(self, mocked_glbl_cfg): + """Usage of get_remote_suite_run_*dir.""" + mocked = MagicMock() + mocked_glbl_cfg.return_value = mocked + mocked.get_host_item.return_value = '/home/sweet/cylc-run' + # func = get_remote_* function to test + # cfg = configuration used in mocked global configuration + # tail1 = expected tail of return value from configuration + # args = extra *args + # tail2 = expected tail of return value from extra args + for func, cfg, tail1 in ( + (get_remote_suite_run_dir, 'run directory', ''), + (get_remote_suite_run_job_dir, 'run directory', '/log/job'), + (get_remote_suite_work_dir, 'work directory', ''), + ): + for args, tail2 in ( + ((), ''), + (('comes', 'true'), '/comes/true'), + ): + self.assertEqual( + f'/home/sweet/cylc-run/my-suite/dream{tail1}{tail2}', + func('myhost', 'myuser', 'my-suite/dream', *args), + ) + mocked.get_host_item.assert_called_with( + cfg, 'myhost', 'myuser') + mocked.get_host_item.reset_mock() + + @patch('cylc.flow.pathutil.glbl_cfg') + def test_get_suite_run_dirs(self, mocked_glbl_cfg): + """Usage of get_suite_run_*dir.""" + mocked = MagicMock() + mocked_glbl_cfg.return_value = mocked + mocked.get_host_item.return_value = '/home/sweet/cylc-run' + # func = get_remote_* function to test + # cfg = configuration used in mocked global configuration + # tail1 = expected tail of return value from configuration + # args = extra *args + # tail2 = expected tail of return value from extra args + for func, cfg, tail1 in ( + (get_suite_run_dir, 'run directory', ''), + (get_suite_run_job_dir, 'run directory', '/log/job'), + (get_suite_run_log_dir, 'run directory', '/log/suite'), + (get_suite_run_rc_dir, 'run directory', '/log/suiterc'), + (get_suite_run_share_dir, 'work directory', '/share'), + (get_suite_run_work_dir, 'work directory', '/work'), + ): + for args, tail2 in ( + ((), ''), + (('comes', 'true'), '/comes/true'), + ): + self.assertEqual( + f'/home/sweet/cylc-run/my-suite/dream{tail1}{tail2}', + func('my-suite/dream', *args), + ) + mocked.get_host_item.assert_called_with(cfg) + mocked.get_host_item.reset_mock() + + @patch('cylc.flow.pathutil.glbl_cfg') + def test_get_suite_run_names(self, mocked_glbl_cfg): + """Usage of get_suite_run_*name.""" + mocked = MagicMock() + mocked_glbl_cfg.return_value = mocked + mocked.get_host_item.return_value = '/home/sweet/cylc-run' + # func = get_remote_* function to test + # cfg = configuration used in mocked global configuration + # tail1 = expected tail of return value from configuration + for func, cfg, tail1 in ( + (get_suite_run_log_name, 'run directory', '/log/suite/log'), + (get_suite_run_pub_db_name, 'run directory', '/log/db'), + ): + self.assertEqual( + f'/home/sweet/cylc-run/my-suite/dream{tail1}', + func('my-suite/dream'), + ) + mocked.get_host_item.assert_called_with(cfg) + mocked.get_host_item.reset_mock() + + @patch('cylc.flow.pathutil.os.makedirs') + @patch('cylc.flow.pathutil.glbl_cfg') + def test_make_suite_run_tree(self, mocked_glbl_cfg, mocked_makedirs): + """Usage of make_suite_run_tree.""" + mocked = MagicMock() + mocked_glbl_cfg.return_value = mocked + mocked.get_host_item.return_value = '/home/sweet/cylc-run' + mocked_cfg = MagicMock() + mocked_cfg['run directory rolling archive length'] = 0 + mocked.get.return_value = mocked_cfg + make_suite_run_tree('my-suite/dream') + self.assertEqual(mocked_makedirs.call_count, 6) + mocked_makedirs.assert_has_calls(( + call(f'/home/sweet/cylc-run/my-suite/dream{tail}', exist_ok=True) + for tail in ( + '', + '/log/suite', + '/log/job', + '/log/suiterc', + '/share', + '/work', + ) + )) + + +if __name__ == '__main__': + from unittest import main + main() diff --git a/tests/host-select/01-timeout.t b/tests/host-select/01-timeout.t deleted file mode 100644 index 0245384103c..00000000000 --- a/tests/host-select/01-timeout.t +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -#------------------------------------------------------------------------------- -# Test host selection, with a command that times out. -. "$(dirname "$0")/test_header" - -set_test_number 3 - -install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" - -create_test_globalrc 'task host select command timeout = PT1S' -run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" -suite_run_ok "${TEST_NAME_BASE}" \ - cylc run --reference-test --debug --no-detach "${SUITE_NAME}" -grep_ok 'ERROR - \[jobs-submit cmd\] (remote host select)' \ - "$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/log/suite/log" -purge_suite "${SUITE_NAME}" -exit diff --git a/tests/job-submission/06-garbage.t b/tests/job-submission/06-garbage.t index 654e6189541..4fc6349f3ee 100755 --- a/tests/job-submission/06-garbage.t +++ b/tests/job-submission/06-garbage.t @@ -20,7 +20,7 @@ set_test_number 2 install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" -if [[ -n "${PYTHONPATH}" ]]; then +if [[ -n "${PYTHONPATH:-}" ]]; then export PYTHONPATH="${PWD}/lib:${PYTHONPATH}" else export PYTHONPATH="${PWD}/lib" diff --git a/tests/rnd/02-lib-python-in-job/suite.rc b/tests/rnd/02-lib-python-in-job/suite.rc index 9284fe88d54..2ed2a06dc11 100644 --- a/tests/rnd/02-lib-python-in-job/suite.rc +++ b/tests/rnd/02-lib-python-in-job/suite.rc @@ -12,7 +12,7 @@ grep -q "${CYLC_SUITE_RUN_DIR}/lib/python" <<< "${PYTHONPATH}" # run a toy example - python -c ' + python3 -c ' from pub import beer assert beer.drink() == "98 bottles of beer on the wall." assert beer.drink() == "97 bottles of beer on the wall."