From 54959200e337c66df4896c066f042ffd2745f0d1 Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Wed, 11 Nov 2020 18:05:12 +0000
Subject: [PATCH 01/13] cylc install
---
cylc/flow/etc/cylc-bash-completion | 2 +-
cylc/flow/scheduler.py | 6 +--
cylc/flow/scheduler_cli.py | 20 ++++----
cylc/flow/scripts/install.py | 63 ++++++++++++++++++-----
cylc/flow/scripts/register.py | 82 ------------------------------
cylc/flow/suite_files.py | 30 +++++------
setup.cfg | 1 -
tests/unit/test_suite_files.py | 10 +++-
8 files changed, 88 insertions(+), 126 deletions(-)
delete mode 100755 cylc/flow/scripts/register.py
diff --git a/cylc/flow/etc/cylc-bash-completion b/cylc/flow/etc/cylc-bash-completion
index d67bed3f83e..ceab4cacdbf 100644
--- a/cylc/flow/etc/cylc-bash-completion
+++ b/cylc/flow/etc/cylc-bash-completion
@@ -38,7 +38,7 @@ _cylc() {
cur="${COMP_WORDS[COMP_CWORD]}"
sec="${COMP_WORDS[1]}"
opts="$(cylc scan -t name 2>/dev/null)"
- suite_cmds="broadcast|bcast|cat-log|check-versions|clean|compare|diff|dump|edit|ext-trigger|external-trigger|get-directory|get-suite-config|get-config|get-suite-version|get-cylc-version|graph|graph-diff|hold|insert|kill|list|log|ls|tui|ping|poll|print|register|release|unhold|reload|remove|report-timings|reset|restart|run|start|scan|search|grep|set-verbosity|show|set-outputs|stop|shutdown|single|suite-state|test-battery|trigger|validate|view|warranty"
+ suite_cmds="broadcast|bcast|cat-log|check-versions|clean|compare|diff|dump|edit|ext-trigger|external-trigger|get-directory|get-suite-config|get-config|get-suite-version|get-cylc-version|graph|graph-diff|hold|insert|install|kill|list|log|ls|tui|ping|poll|print|release|unhold|reload|remove|report-timings|reset|restart|run|start|scan|search|grep|set-verbosity|show|set-outputs|stop|shutdown|single|suite-state|test-battery|trigger|validate|view|warranty"
if [[ ${COMP_CWORD} -eq 1 ]]; then
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index f34e24089db..c98df73438b 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -280,18 +280,18 @@ def __init__(self, reg, options, is_restart=False):
async def install(self):
"""Get the filesystem in the right state to run the flow.
- * Register.
+ * Install.
* Install authentication files.
* Build the directory tree.
* Copy Python files.
"""
- # Register
+ # Install
try:
suite_files.get_suite_source_dir(self.suite)
except SuiteServiceFileError:
# Source path is assumed to be the run directory
- suite_files.register(self.suite, get_suite_run_dir(self.suite))
+ suite_files.install(self.suite, get_suite_run_dir(self.suite))
# Create ZMQ keys
key_housekeeping(self.suite, platform=self.options.host or 'localhost')
diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py
index 79bfb915730..452c9205035 100644
--- a/cylc/flow/scheduler_cli.py
+++ b/cylc/flow/scheduler_cli.py
@@ -52,14 +52,14 @@
The scheduler will run as a daemon unless you specify --no-detach.
-If the suite is not already registered (by "cylc register" or a previous run)
-it will be registered on the fly before start up.
+If the suite is not already installed (by "cylc install" or a previous run)
+it will be installed on the fly before start up.
Examples:
- # Run the suite registered with name REG.
+ # Run the suite installed with name REG.
$ cylc run REG
- # Register $PWD/flow.cylc as $(basename $PWD) and run it.
+ # Install $PWD/flow.cylc as $(basename $PWD) and run it.
# Note REG must be given explicitly if START_POINT is on the command line.
$ cylc run
@@ -261,10 +261,10 @@ def get_option_parser(is_restart, add_std_opts=False):
get_option_parser(is_restart=True, add_std_opts=True), DEFAULT_OPTS)
-def _auto_register():
- """Register a suite installed in the cylc-run directory."""
+def _auto_install():
+ """Install a suite installed in the cylc-run directory."""
try:
- reg = suite_files.register()
+ reg = suite_files.install()
except SuiteServiceFileError as exc:
sys.exit(exc)
# Replace this process with "cylc run REG ..." for 'ps -f'.
@@ -366,8 +366,8 @@ def scheduler_cli(parser, options, args, is_restart=False):
sys.exit(ret)
-def _check_registration(reg):
- """Ensure the flow is registered."""
+def _check_installation(reg):
+ """Ensure the flow is installed."""
suite_run_dir = get_suite_run_dir(reg)
if not os.path.exists(suite_run_dir):
sys.stderr.write(f'suite service directory not found '
@@ -436,7 +436,7 @@ def restart(parser, options, *args):
def run(parser, options, *args):
"""Implement cylc run."""
if not args:
- _auto_register()
+ _auto_install()
if options.startcp:
options.warm = True
if len(args) >= 2:
diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py
index a4970ae464e..4bae8507e0a 100755
--- a/cylc/flow/scripts/install.py
+++ b/cylc/flow/scripts/install.py
@@ -18,10 +18,43 @@
"""cylc install [OPTIONS] ARGS
-Test communication with a running suite.
+Install a new suite.
+
+
+Install the name REG for the suite definition in PATH. The suite server
+program can then be started, stopped, and targeted by name REG. (Note that
+"cylc run" can also install suites on the fly).
+
+Installation creates a suite run directory "~/cylc-run/REG/" containing a
+".service/source" symlink to the suite definition PATH. The .service directory
+will also be used for server authentication files at run time.
+
+Suite names can be hierarchical, corresponding to the path under ~/cylc-run.
+
+Examples:
+ # Register PATH/flow.cylc as dogs/fido
+ # (with run directory ~/cylc-run/dogs/fido)
+ $ cylc install dogs/fido PATH
+
+ # Install $PWD/flow.cylc as dogs/fido.
+ $ cylc install dogs/fido
+
+ # Install $PWD/flow.cylc as the parent directory
+ # name: $(basename $PWD).
+ $ cylc install
+
+The same suite can be installed with multiple names; this results in multiple
+suite run directories that link to the same suite definition.
+
+To "unregister" a suite, delete or rename its run directory (renaming it under
+~/cylc-run effectively re-registers the original suite with the new name).
+
+Use of "--redirect" is required to allow an existing name (and run directory)
+to be associated with a different suite definition. This is potentially
+dangerous because the new suite will overwrite files in the existing run
+directory. You should consider deleting or renaming an existing run directory
+rather than just re-use it with another suite."""
-If suite REG is running or TASK in suite REG is currently running,
-exit with success status, else exit with error status."""
import os
import pkg_resources
@@ -29,28 +62,32 @@
from cylc.flow.exceptions import PluginError
from cylc.flow.option_parsers import CylcOptionParser as COP
-from cylc.flow.terminal import cli_function
from cylc.flow.pathutil import get_suite_run_dir
-from cylc.flow.suite_files import parse_suite_arg
+from cylc.flow.suite_files import parse_suite_arg, install
+from cylc.flow.terminal import cli_function
def get_option_parser():
parser = COP(
__doc__, comms=True, prep=True,
- argdoc=[('REG', 'Suite name')])
+ argdoc=[("[REG]", "Suite name"),
+ ("[PATH]", "Suite definition directory (defaults to $PWD)")])
+
+ parser.add_option(
+ "--redirect", help="Allow an existing suite name and run directory"
+ " to be used with another suite.",
+ action="store_true", default=False, dest="redirect")
return parser
@cli_function(get_option_parser)
-def main(parser, options, reg):
- suite, flow_file = parse_suite_arg(options, reg)
-
+def main(parser, opts, flow_name=None, src=None):
for entry_point in pkg_resources.iter_entry_points(
'cylc.pre_configure'
):
try:
- entry_point.resolve()(Path(flow_file).parent)
+ entry_point.resolve()(opts.source)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
# this is to separate plugin from core Cylc errors
@@ -60,14 +97,16 @@ def main(parser, options, reg):
exc
) from None
+ flow_name = install(reg, src, redirect=opts.redirect, rundir=opts.rundir)
+
for entry_point in pkg_resources.iter_entry_points(
'cylc.post_install'
):
try:
entry_point.resolve()(
dir_=os.getcwd(),
- opts=options,
- dest_root=get_suite_run_dir(suite)
+ opts=opts,
+ dest_root=get_suite_run_dir(flow_name)
)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
diff --git a/cylc/flow/scripts/register.py b/cylc/flow/scripts/register.py
deleted file mode 100755
index ad0e82ec8cc..00000000000
--- a/cylc/flow/scripts/register.py
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-"""cylc register [OPTIONS] ARGS
-
-Register a new suite.
-
-Register the name REG for the suite definition in PATH. The suite server
-program can then be started, stopped, and targeted by name REG. (Note that
-"cylc run" can also register suites on the fly).
-
-Registration creates a suite run directory "~/cylc-run/REG/" containing a
-".service/source" symlink to the suite definition PATH. The .service directory
-will also be used for server authentication files at run time.
-
-Suite names can be hierarchical, corresponding to the path under ~/cylc-run.
-
-Examples:
- # Register PATH/flow.cylc as dogs/fido
- # (with run directory ~/cylc-run/dogs/fido)
- $ cylc register dogs/fido PATH
-
- # Register $PWD/flow.cylc as dogs/fido.
- $ cylc register dogs/fido
-
- # Register $PWD/flow.cylc as the parent directory
- # name: $(basename $PWD).
- $ cylc register
-
-The same suite can be registered with multiple names; this results in multiple
-suite run directories that link to the same suite definition.
-
-To "unregister" a suite, delete or rename its run directory (renaming it under
-~/cylc-run effectively re-registers the original suite with the new name).
-
-Use of "--redirect" is required to allow an existing name (and run directory)
-to be associated with a different suite definition. This is potentially
-dangerous because the new suite will overwrite files in the existing run
-directory. You should consider deleting or renaming an existing run directory
-rather than just re-use it with another suite."""
-
-from cylc.flow.option_parsers import CylcOptionParser as COP
-from cylc.flow.suite_files import register
-from cylc.flow.terminal import cli_function
-
-
-def get_option_parser():
- parser = COP(
- __doc__,
- argdoc=[("[REG]", "Suite name"),
- ("[PATH]", "Suite definition directory (defaults to $PWD)")])
-
- parser.add_option(
- "--redirect", help="Allow an existing suite name and run directory"
- " to be used with another suite.",
- action="store_true", default=False, dest="redirect")
-
- return parser
-
-
-@cli_function(get_option_parser)
-def main(parser, opts, reg=None, src=None):
- register(reg, src, redirect=opts.redirect)
-
-
-if __name__ == "__main__":
- main()
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index 9e7310a3e0d..57aff18c8e9 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -350,7 +350,7 @@ def get_flow_file(reg, suite_owner=None):
def get_suite_source_dir(reg, suite_owner=None):
"""Return the source directory path of a suite.
- Will register un-registered suites located in the cylc run dir.
+ Will install un-installed suites located in the cylc run dir.
"""
srv_d = get_suite_srv_dir(reg, suite_owner)
fname = os.path.join(srv_d, SuiteFiles.Service.SOURCE)
@@ -359,8 +359,8 @@ def get_suite_source_dir(reg, suite_owner=None):
except OSError:
suite_d = os.path.dirname(srv_d)
if os.path.exists(suite_d) and not is_remote_user(suite_owner):
- # suite exists but is not yet registered
- register(reg=reg, source=suite_d)
+ # suite exists but is not yet installed
+ install(reg=reg, source=suite_d)
return suite_d
raise SuiteServiceFileError(f"Suite not found: {reg}")
else:
@@ -369,7 +369,7 @@ def get_suite_source_dir(reg, suite_owner=None):
flow_file_path = os.path.join(source, SuiteFiles.FLOW_FILE)
if not os.path.exists(flow_file_path):
# suite exists but is probably using deprecated suite.rc
- register(reg=reg, source=source)
+ install(reg=reg, source=source)
return source
@@ -428,7 +428,7 @@ async def load_contact_file_async(reg, run_dir=None):
def parse_suite_arg(options, arg):
"""From CLI arg "SUITE", return suite name and flow.cylc path.
- If arg is a registered suite, suite name is the registered name.
+ If arg is a installed suite, suite name is the installed name.
If arg is a directory, suite name is the base name of the
directory.
If arg is a file, suite name is the base name of its container
@@ -460,8 +460,8 @@ def parse_suite_arg(options, arg):
return name, path
-def register(reg=None, source=None, redirect=False):
- """Register a suite, or renew its registration.
+def install(reg=None, source=None, redirect=False, rundir=None):
+ """Install a suite, or renew its installation.
Create suite service directory and symlink to suite source location.
@@ -471,14 +471,14 @@ def register(reg=None, source=None, redirect=False):
redirect (bool): allow reuse of existing name and run directory.
Return:
- str: The registered suite name (which may be computed here).
+ str: The installed suite name (which may be computed here).
Raise:
SuiteServiceFileError:
- - No flow.cylc file found in source location.
- - Illegal name (can look like a relative path, but not absolute).
- - Another suite already has this name (unless --redirect).
- - Trying to register a suite nested inside of another.
+ - No flow.cylc file found in source location.
+ - Illegal name (can look like a relative path, but not absolute).
+ - Another suite already has this name (unless --redirect).
+ - Trying to install a workflow that is nested inside of another.
"""
if reg is None:
reg = os.path.basename(os.getcwd())
@@ -546,7 +546,7 @@ def register(reg=None, source=None, redirect=False):
source_str = source
os.symlink(source_str, target)
- print(f'REGISTERED {reg} -> {source}')
+ print(f'INSTALLED {reg} -> {source}')
return reg
@@ -867,7 +867,7 @@ def _validate_reg(reg):
def check_nested_run_dirs(reg):
- """Disallow nested run dirs e.g. trying to register foo/bar where foo is
+ """Disallow nested run dirs e.g. trying to install foo/bar where foo is
already a valid suite directory.
Args:
@@ -880,7 +880,7 @@ def check_nested_run_dirs(reg):
depth)
"""
exc_msg = (
- 'Nested run directories not allowed - cannot register suite name '
+ 'Nested run directories not allowed - cannot install suite name '
'"%s" as "%s" is already a valid run directory.')
def _check_child_dirs(path, depth_count=1):
diff --git a/setup.cfg b/setup.cfg
index 2d39c5fe287..1203feb79ad 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -96,7 +96,6 @@ cylc.command =
ping = cylc.flow.scripts.ping:main
poll = cylc.flow.scripts.poll:main
psutils = cylc.flow.scripts.psutil:main
- register = cylc.flow.scripts.register:main
release = cylc.flow.scripts.release:main
reload = cylc.flow.scripts.reload:main
remote-init = cylc.flow.scripts.remote_init:main
diff --git a/tests/unit/test_suite_files.py b/tests/unit/test_suite_files.py
index 66b213a4017..a369aaa4f56 100644
--- a/tests/unit/test_suite_files.py
+++ b/tests/unit/test_suite_files.py
@@ -231,7 +231,7 @@ def mkdirs_standin(_, exist_ok=False):
mocked_readlink.side_effect = lambda x: readlink
if e_expected is None:
- reg = suite_files.register(reg, source, redirect)
+ reg = suite_files.install(reg, source, redirect)
assert reg == expected
if mocked_symlink.call_count > 0:
# first argument, of the first call
@@ -239,7 +239,7 @@ def mkdirs_standin(_, exist_ok=False):
assert arg0 == expected_symlink
else:
with pytest.raises(e_expected) as exc:
- suite_files.register(reg, source, redirect)
+ suite_files.install(reg, source, redirect)
if e_message is not None:
assert e_message in str(exc.value)
@@ -322,10 +322,16 @@ def mock_scandir(path):
for path in ('a/a/a', 'a/b'):
suite_files.check_nested_run_dirs(path)
# Run dir nested below - bad:
+
for path in ('a', 'a/a', 'a/c'):
with pytest.raises(WorkflowFilesError) as exc:
check_nested_run_dirs(path)
assert 'Nested run directories not allowed' in str(exc.value)
+ for func in (suite_files.check_nested_run_dirs, suite_files.install):
+ for path in ('a', 'a/a', 'a/c'):
+ with pytest.raises(SuiteServiceFileError) as exc:
+ func(path)
+ assert 'Nested run directories not allowed' in str(exc.value)
# Run dir nested below max scan depth - not ideal but passes:
suite_files.check_nested_run_dirs('a/d')
From 629bd09455a1da7cf2c0ce2b7c60896910fbcbd1 Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Thu, 19 Nov 2020 10:03:15 +0000
Subject: [PATCH 02/13] Update functional tests with cylc install
---
cylc/flow/scripts/cylc.py | 5 +-
cylc/flow/scripts/install.py | 19 +-
cylc/flow/suite_files.py | 45 ++--
.../functional/authentication/00-shared-fs.t | 2 +-
.../01-remote-suite-same-name.t | 2 +-
.../authentication/02-suite2-stop-suite1.t | 4 +-
tests/functional/cylc-diff/01-same.t | 2 +-
tests/functional/cylc-install/00-simple.t | 206 ++++++++++++++++++
.../test_header | 0
tests/functional/deprecations/03-suiterc.t | 2 +-
tests/functional/lib/bash/test_header | 4 +-
tests/functional/registration/00-simple.t | 63 +++++-
tests/functional/validate/48-reg-then-pwd.t | 2 +-
13 files changed, 316 insertions(+), 40 deletions(-)
create mode 100755 tests/functional/cylc-install/00-simple.t
rename tests/functional/{registration => cylc-install}/test_header (100%)
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 0060afbbd60..85e14b8f999 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -153,7 +153,10 @@ def get_version(long=False):
'check-software': (
'use standard tools to inspect the environment '
'e.g. https://pypi.org/project/pipdeptree/'
- )
+ ),
+ 'jobscript': 'cylc jobscript has been removed',
+ 'submit': 'cylc submit has been removed',
+ 'register': 'cylc register had been removed, use cylc install or cylc run'
}
diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py
index 4bae8507e0a..517324e71bd 100755
--- a/cylc/flow/scripts/install.py
+++ b/cylc/flow/scripts/install.py
@@ -49,11 +49,7 @@
To "unregister" a suite, delete or rename its run directory (renaming it under
~/cylc-run effectively re-registers the original suite with the new name).
-Use of "--redirect" is required to allow an existing name (and run directory)
-to be associated with a different suite definition. This is potentially
-dangerous because the new suite will overwrite files in the existing run
-directory. You should consider deleting or renaming an existing run directory
-rather than just re-use it with another suite."""
+"""
import os
@@ -70,14 +66,23 @@
def get_option_parser():
parser = COP(
__doc__, comms=True, prep=True,
- argdoc=[("[REG]", "Suite name"),
- ("[PATH]", "Suite definition directory (defaults to $PWD)")])
+ argdoc=[("[REG]", "Workflow name"),
+ ("[PATH]", "Workflow definition directory (defaults to $PWD)")
+ ])
parser.add_option(
"--redirect", help="Allow an existing suite name and run directory"
" to be used with another suite.",
action="store_true", default=False, dest="redirect")
+ parser.add_option(
+ "--run-name", help="Name the run ",
+ action="store", metavar="RUNDIR", default=None, dest="rundir")
+
+ parser.add_option(
+ "--run-dir", help="Symlink $HOME/cylc-run/REG to RUNDIR/REG.",
+ action="store", metavar="RUNDIR", default=None, dest="rundir")
+
return parser
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index 57aff18c8e9..152a2c7df10 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -360,7 +360,7 @@ def get_suite_source_dir(reg, suite_owner=None):
suite_d = os.path.dirname(srv_d)
if os.path.exists(suite_d) and not is_remote_user(suite_owner):
# suite exists but is not yet installed
- install(reg=reg, source=suite_d)
+ install(flow_name=reg, source=suite_d)
return suite_d
raise SuiteServiceFileError(f"Suite not found: {reg}")
else:
@@ -369,7 +369,7 @@ def get_suite_source_dir(reg, suite_owner=None):
flow_file_path = os.path.join(source, SuiteFiles.FLOW_FILE)
if not os.path.exists(flow_file_path):
# suite exists but is probably using deprecated suite.rc
- install(reg=reg, source=source)
+ install(flow_name=reg, source=source)
return source
@@ -460,13 +460,13 @@ def parse_suite_arg(options, arg):
return name, path
-def install(reg=None, source=None, redirect=False, rundir=None):
+def install(flow_name=None, source=None, redirect=False, rundir=None):
"""Install a suite, or renew its installation.
Create suite service directory and symlink to suite source location.
Args:
- reg (str): suite name, default basename($PWD).
+ flow_name (str): workflow name, default basename($PWD).
source (str): directory location of flow.cylc file, default $PWD.
redirect (bool): allow reuse of existing name and run directory.
@@ -480,10 +480,18 @@ def install(reg=None, source=None, redirect=False, rundir=None):
- Another suite already has this name (unless --redirect).
- Trying to install a workflow that is nested inside of another.
"""
- if reg is None:
- reg = os.path.basename(os.getcwd())
- _validate_reg(reg)
- check_nested_run_dirs(reg)
+ if flow_name is None:
+ flow_name = (Path.cwd().stem)
+ make_localhost_symlinks(flow_name)
+
+ is_valid, message = SuiteNameValidator.validate(flow_name)
+ if not is_valid:
+ raise SuiteServiceFileError(f'Invalid workflow name - {message}')
+
+ if Path.is_absolute(Path(flow_name)):
+ raise SuiteServiceFileError(
+ f'Workflow name cannot be an absolute path: {flow_name}')
+ check_nested_run_dirs(flow_name)
make_localhost_symlinks(reg)
@@ -525,11 +533,11 @@ def install(reg=None, source=None, redirect=False, rundir=None):
if orig_source is not None and source != orig_source:
if not redirect:
raise SuiteServiceFileError(
- f"the name '{reg}' already points to {orig_source}.\nUse "
+ f"the name '{flow_name}' already points to {orig_source}.\nUse "
"--redirect to re-use an existing name and run directory.")
LOG.warning(
- f"the name '{reg}' points to {orig_source}.\nIt will now be "
- f"redirected to {source}.\nFiles in the existing {reg} run "
+ f"the name '{flow_name}' points to {orig_source}.\nIt will now be "
+ f"redirected to {source}.\nFiles in the existing {flow_name} run "
"directory will be overwritten.\n")
# Remove symlink to the original suite.
os.unlink(os.path.join(srv_d, SuiteFiles.Service.SOURCE))
@@ -546,8 +554,8 @@ def install(reg=None, source=None, redirect=False, rundir=None):
source_str = source
os.symlink(source_str, target)
- print(f'INSTALLED {reg} -> {source}')
- return reg
+ print(f'INSTALLED {flow_name} -> {source}')
+ return flow_name
def _clean_check(reg, run_dir):
@@ -761,6 +769,17 @@ def _remove_empty_reg_parents(reg, path):
break
+def start_install_log(reg, no_detach):
+ if not no_detach:
+ while LOG.handlers:
+ LOG.handlers[0].close()
+ LOG.removeHandler(LOG.handlers[0])
+
+ install_log_path = get_install_log_name(reg)
+ handler = TimestampRotatingFileHandler(install_log_path, no_detach)
+ INSTALL_LOG.addHandler(handler)
+
+
def remove_keys_on_server(keys):
"""Removes server-held authentication keys"""
# WARNING, DESTRUCTIVE. Removes old keys if they already exist.
diff --git a/tests/functional/authentication/00-shared-fs.t b/tests/functional/authentication/00-shared-fs.t
index 98ec129c673..688c45daa34 100755
--- a/tests/functional/authentication/00-shared-fs.t
+++ b/tests/functional/authentication/00-shared-fs.t
@@ -28,7 +28,7 @@ SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BA
SUITE_RUN_DIR="$RUN_DIR/${SUITE_NAME}"
mkdir -p "$(dirname "${SUITE_RUN_DIR}")"
cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}" "${SUITE_RUN_DIR}"
-cylc register "${SUITE_NAME}" "${SUITE_RUN_DIR}" 2>'/dev/null'
+cylc install "${SUITE_NAME}" "${SUITE_RUN_DIR}" 2>'/dev/null'
run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
diff --git a/tests/functional/authentication/01-remote-suite-same-name.t b/tests/functional/authentication/01-remote-suite-same-name.t
index 3a019fc47b5..45854c1648f 100755
--- a/tests/functional/authentication/01-remote-suite-same-name.t
+++ b/tests/functional/authentication/01-remote-suite-same-name.t
@@ -34,7 +34,7 @@ scp ${SSH_OPTS} -pqr "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/"* \
# shellcheck disable=SC2086
run_ok "${TEST_NAME_BASE}-register" \
ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" \
- CYLC_VERSION="$(cylc version)" cylc register "${SUITE_NAME}" \
+ CYLC_VERSION="$(cylc version)" cylc install "${SUITE_NAME}" \
"cylc-run/${SUITE_NAME}"
suite_run_ok "${TEST_NAME_BASE}" \
diff --git a/tests/functional/authentication/02-suite2-stop-suite1.t b/tests/functional/authentication/02-suite2-stop-suite1.t
index b7c6733407d..bcc44fa9086 100755
--- a/tests/functional/authentication/02-suite2-stop-suite1.t
+++ b/tests/functional/authentication/02-suite2-stop-suite1.t
@@ -26,7 +26,7 @@ NAME2="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}-2
SUITE1_RUND="${RUND}/${NAME1}"
mkdir -p "${SUITE1_RUND}"
cp -p "${TEST_SOURCE_DIR}/basic/flow.cylc" "${SUITE1_RUND}"
-cylc register "${NAME1}" "${SUITE1_RUND}"
+cylc install "${NAME1}" "${SUITE1_RUND}"
SUITE2_RUND="${RUND}/${NAME2}"
mkdir -p "${SUITE2_RUND}"
cat >"${SUITE2_RUND}/flow.cylc" <<__FLOW_CONFIG__
@@ -39,7 +39,7 @@ cat >"${SUITE2_RUND}/flow.cylc" <<__FLOW_CONFIG__
[[t1]]
script=cylc shutdown "${NAME1}"
__FLOW_CONFIG__
-cylc register "${NAME2}" "${SUITE2_RUND}"
+cylc install "${NAME2}" "${SUITE2_RUND}"
cylc run --no-detach "${NAME1}" 1>'1.out' 2>&1 &
SUITE_RUN_DIR="${SUITE1_RUND}" poll_suite_running
run_ok "${TEST_NAME_BASE}" cylc run --no-detach --abort-if-any-task-fails "${NAME2}"
diff --git a/tests/functional/cylc-diff/01-same.t b/tests/functional/cylc-diff/01-same.t
index 2b026ba70d9..a8d9124abc1 100755
--- a/tests/functional/cylc-diff/01-same.t
+++ b/tests/functional/cylc-diff/01-same.t
@@ -33,7 +33,7 @@ init_suite "${TEST_NAME_BASE}-1" "${PWD}/flow.cylc"
SUITE_NAME1="${SUITE_NAME}"
# shellcheck disable=SC2153
SUITE_NAME2="${SUITE_NAME1%1}2"
-cylc register "${SUITE_NAME2}" "${TEST_DIR}/${SUITE_NAME1}" 2>'/dev/null'
+cylc install "${SUITE_NAME2}" "${TEST_DIR}/${SUITE_NAME1}" 2>'/dev/null'
run_ok "${TEST_NAME_BASE}" cylc diff "${SUITE_NAME1}" "${SUITE_NAME2}"
cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__
diff --git a/tests/functional/cylc-install/00-simple.t b/tests/functional/cylc-install/00-simple.t
new file mode 100755
index 00000000000..9aa239a2f7f
--- /dev/null
+++ b/tests/functional/cylc-install/00-simple.t
@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+#------------------------------------------------------------------------------
+# Test suite registration
+
+export RND_SUITE_NAME
+export RND_SUITE_SOURCE
+export RND_SUITE_RUNDIR
+
+function make_rnd_suite() {
+ # Create a randomly-named suite source directory.
+ # Define its run directory.
+ RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)
+ RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}"
+ mkdir -p "${RND_SUITE_SOURCE}"
+ touch "${RND_SUITE_SOURCE}/flow.cylc"
+ RND_SUITE_RUNDIR="${RUN_DIR}/${RND_SUITE_NAME}"
+}
+
+function purge_rnd_suite() {
+ # Remove the suite source created by make_rnd_suite().
+ # And remove its run-directory too.
+ RND_SUITE_SOURCE=${1:-$RND_SUITE_SOURCE}
+ RND_SUITE_RUNDIR=${2:-$RND_SUITE_RUNDIR}
+ rm -rf "${RND_SUITE_SOURCE}"
+ rm -rf "${RND_SUITE_RUNDIR}"
+}
+
+. "$(dirname "$0")/test_header"
+set_test_number 33
+
+# Use $SUITE_NAME and $SUITE_RUN_DIR defined by test_header
+
+#------------------------------
+# Test fail no suite source dir
+TEST_NAME="${TEST_NAME_BASE}-nodir"
+make_rnd_suite
+rm -rf "${RND_SUITE_SOURCE}"
+run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
+__ERR__
+purge_rnd_suite
+
+#---------------------------
+# Test fail no flow.cylc file
+TEST_NAME="${TEST_NAME_BASE}-nodir"
+make_rnd_suite
+rm -f "${RND_SUITE_SOURCE}/flow.cylc"
+run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
+__ERR__
+purge_rnd_suite
+
+#-------------------------------------------------------
+# Test default name: "cylc reg" (suite in $PWD, no args)
+TEST_NAME="${TEST_NAME_BASE}-pwd1"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}" cylc install
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+REGISTERED $RND_SUITE_NAME -> ${RND_SUITE_SOURCE}
+__OUT__
+popd || exit 1
+purge_rnd_suite
+
+#--------------------------------------------------
+# Test default path: "cylc reg REG" (suite in $PWD)
+TEST_NAME="${TEST_NAME_BASE}-pwd2"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
+__OUT__
+popd || exit 1
+purge_rnd_suite
+
+#-------------------------
+# Test "cylc reg REG PATH"
+TEST_NAME="${TEST_NAME_BASE}-normal"
+make_rnd_suite
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
+__OUT__
+purge_rnd_suite
+
+#--------------------------------------------------------------------
+# Test register existing run directory: "cylc reg REG ~/cylc-run/REG"
+TEST_NAME="${TEST_NAME_BASE}-reg-run-dir"
+make_rnd_suite
+mkdir -p "${RND_SUITE_RUNDIR}"
+cp "${RND_SUITE_SOURCE}/flow.cylc" "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
+__OUT__
+SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")"
+run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}"
+# Run it twice
+run_ok "${TEST_NAME}-2" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
+contains_ok "${TEST_NAME}-2.stdout" <<__OUT__
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
+__OUT__
+SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")"
+run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}"
+purge_rnd_suite
+
+#----------------------------------------------------------------
+# Test fail "cylc reg REG PATH" where REG already points to PATH2
+TEST_NAME="${TEST_NAME_BASE}-dup1"
+make_rnd_suite
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+RND_SUITE_NAME1="${RND_SUITE_NAME}"
+RND_SUITE_SOURCE1="${RND_SUITE_SOURCE}"
+RND_SUITE_RUNDIR1="${RND_SUITE_RUNDIR}"
+make_rnd_suite
+TEST_NAME="${TEST_NAME_BASE}-dup2"
+run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+SuiteServiceFileError: the name '${RND_SUITE_NAME1}' already points to ${RND_SUITE_SOURCE1}.
+Use --redirect to re-use an existing name and run directory.
+__ERR__
+# Now force it
+TEST_NAME="${TEST_NAME_BASE}-dup3"
+run_ok "${TEST_NAME}" cylc install --redirect "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
+sed -i 's/^\t//; s/^.* WARNING - /WARNING - /' "${TEST_NAME}.stderr"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+WARNING - the name '${RND_SUITE_NAME1}' points to ${RND_SUITE_SOURCE1}.
+It will now be redirected to ${RND_SUITE_SOURCE}.
+Files in the existing ${RND_SUITE_NAME1} run directory will be overwritten.
+__ERR__
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+REGISTERED ${RND_SUITE_NAME1} -> ${RND_SUITE_SOURCE}
+__OUT__
+
+TEST_NAME="${TEST_NAME_BASE}-get-dir"
+run_ok "${TEST_NAME}" cylc get-directory "${RND_SUITE_NAME1}"
+contains_ok "${TEST_NAME}.stdout" <<__ERR__
+${RND_SUITE_SOURCE}
+__ERR__
+
+purge_rnd_suite
+purge_rnd_suite "${RND_SUITE_SOURCE1}" "${RND_SUITE_RUNDIR1}"
+
+#-----------------------
+# Test alternate run dir
+# 1. Normal case.
+TEST_NAME="${TEST_NAME_BASE}-alt-run-dir"
+make_rnd_suite
+ALT_RUN_DIR="${PWD}/alt"
+run_ok "${TEST_NAME}" \
+ cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
+__OUT__
+run_ok "${TEST_NAME}-check-link" test -L "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}-rm-link" rm "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}-rm-alt-run-dir" rm -r "${ALT_RUN_DIR}"
+purge_rnd_suite
+
+# 2. If reg already exists (as a directory).
+TEST_NAME="${TEST_NAME_BASE}-alt-exists1"
+make_rnd_suite
+ALT_RUN_DIR="${PWD}/alt"
+mkdir -p "${RND_SUITE_RUNDIR}"
+run_fail "${TEST_NAME}" \
+ cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__OUT__
+SuiteServiceFileError: Run directory '${RND_SUITE_RUNDIR}' already exists.
+__OUT__
+purge_rnd_suite
+
+# 3. If reg already exists (as a valid symlink).
+TEST_NAME="${TEST_NAME_BASE}-alt-exists2"
+make_rnd_suite
+ALT_RUN_DIR="${PWD}/alt"
+TDIR=$(mktemp -d)
+mkdir -p "$(dirname "${RND_SUITE_RUNDIR}")"
+ln -s "${TDIR}" "${RND_SUITE_RUNDIR}"
+run_fail "${TEST_NAME}" \
+ cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__OUT__
+SuiteServiceFileError: Symlink '${RND_SUITE_RUNDIR}' already points to ${TDIR}.
+__OUT__
+purge_rnd_suite
+rm -rf "${TDIR}"
+
+exit
diff --git a/tests/functional/registration/test_header b/tests/functional/cylc-install/test_header
similarity index 100%
rename from tests/functional/registration/test_header
rename to tests/functional/cylc-install/test_header
diff --git a/tests/functional/deprecations/03-suiterc.t b/tests/functional/deprecations/03-suiterc.t
index e9312b399c1..e787b5aa0e1 100644
--- a/tests/functional/deprecations/03-suiterc.t
+++ b/tests/functional/deprecations/03-suiterc.t
@@ -39,7 +39,7 @@ TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate .
TEST_NAME="${TEST_NAME_BASE}-register"
-run_ok "${TEST_NAME}" cylc register "${SUITE_NAME}"
+run_ok "${TEST_NAME}" cylc install "${SUITE_NAME}"
exists_ok "flow.cylc"
diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header
index df5f9006eb2..3d3d6e749be 100644
--- a/tests/functional/lib/bash/test_header
+++ b/tests/functional/lib/bash/test_header
@@ -424,7 +424,7 @@ init_suite() {
SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}"
mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
cat "${FLOW_CONFIG}" >"${TEST_DIR}/${SUITE_NAME}/flow.cylc"
- cylc register "${SUITE_NAME}" "${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
+ cylc install "${SUITE_NAME}" "${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
cd "${TEST_DIR}/${SUITE_NAME}"
}
@@ -435,7 +435,7 @@ install_suite() {
SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}"
mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
cp -r "${TEST_SOURCE_DIR}/${TEST_SOURCE_BASE}/"* "${TEST_DIR}/${SUITE_NAME}/"
- cylc register "${SUITE_NAME}" "${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
+ cylc install "${SUITE_NAME}" "${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
cd "${TEST_DIR}/${SUITE_NAME}"
}
diff --git a/tests/functional/registration/00-simple.t b/tests/functional/registration/00-simple.t
index 0ec4ab342fd..ed875dec81f 100755
--- a/tests/functional/registration/00-simple.t
+++ b/tests/functional/registration/00-simple.t
@@ -51,7 +51,7 @@ set_test_number 24
TEST_NAME="${TEST_NAME_BASE}-nodir"
make_rnd_suite
rm -rf "${RND_SUITE_SOURCE}"
-run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
contains_ok "${TEST_NAME}.stderr" <<__ERR__
SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
__ERR__
@@ -62,7 +62,7 @@ purge_rnd_suite
TEST_NAME="${TEST_NAME_BASE}-nodir"
make_rnd_suite
rm -f "${RND_SUITE_SOURCE}/flow.cylc"
-run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
contains_ok "${TEST_NAME}.stderr" <<__ERR__
SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
__ERR__
@@ -73,7 +73,7 @@ purge_rnd_suite
TEST_NAME="${TEST_NAME_BASE}-pwd1"
make_rnd_suite
pushd "${RND_SUITE_SOURCE}" || exit 1
-run_ok "${TEST_NAME}" cylc register
+run_ok "${TEST_NAME}" cylc install
contains_ok "${TEST_NAME}.stdout" <<__OUT__
REGISTERED $RND_SUITE_NAME -> ${RND_SUITE_SOURCE}
__OUT__
@@ -85,7 +85,7 @@ purge_rnd_suite
TEST_NAME="${TEST_NAME_BASE}-pwd2"
make_rnd_suite
pushd "${RND_SUITE_SOURCE}" || exit 1
-run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}"
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}"
contains_ok "${TEST_NAME}.stdout" <<__OUT__
REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
__OUT__
@@ -96,7 +96,7 @@ purge_rnd_suite
# Test "cylc reg REG PATH"
TEST_NAME="${TEST_NAME_BASE}-normal"
make_rnd_suite
-run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
contains_ok "${TEST_NAME}.stdout" <<__OUT__
REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
__OUT__
@@ -108,14 +108,14 @@ TEST_NAME="${TEST_NAME_BASE}-reg-run-dir"
make_rnd_suite
mkdir -p "${RND_SUITE_RUNDIR}"
cp "${RND_SUITE_SOURCE}/flow.cylc" "${RND_SUITE_RUNDIR}"
-run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
contains_ok "${TEST_NAME}.stdout" <<__OUT__
REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
__OUT__
SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")"
run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}"
# Run it twice
-run_ok "${TEST_NAME}-2" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}-2" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
contains_ok "${TEST_NAME}-2.stdout" <<__OUT__
REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
__OUT__
@@ -127,20 +127,20 @@ purge_rnd_suite
# Test fail "cylc reg REG PATH" where REG already points to PATH2
TEST_NAME="${TEST_NAME_BASE}-dup1"
make_rnd_suite
-run_ok "${TEST_NAME}" cylc register "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
RND_SUITE_NAME1="${RND_SUITE_NAME}"
RND_SUITE_SOURCE1="${RND_SUITE_SOURCE}"
RND_SUITE_RUNDIR1="${RND_SUITE_RUNDIR}"
make_rnd_suite
TEST_NAME="${TEST_NAME_BASE}-dup2"
-run_fail "${TEST_NAME}" cylc register "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
+run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
contains_ok "${TEST_NAME}.stderr" <<__ERR__
SuiteServiceFileError: the name '${RND_SUITE_NAME1}' already points to ${RND_SUITE_SOURCE1}.
Use --redirect to re-use an existing name and run directory.
__ERR__
# Now force it
TEST_NAME="${TEST_NAME_BASE}-dup3"
-run_ok "${TEST_NAME}" cylc register --redirect "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
+run_ok "${TEST_NAME}" cylc install --redirect "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
sed -i 's/^\t//; s/^.* WARNING - /WARNING - /' "${TEST_NAME}.stderr"
contains_ok "${TEST_NAME}.stderr" <<__ERR__
WARNING - the name '${RND_SUITE_NAME1}' points to ${RND_SUITE_SOURCE1}.
@@ -160,4 +160,47 @@ __ERR__
purge_rnd_suite
purge_rnd_suite "${RND_SUITE_SOURCE1}" "${RND_SUITE_RUNDIR1}"
+#-----------------------
+# Test alternate run dir
+# 1. Normal case.
+TEST_NAME="${TEST_NAME_BASE}-alt-run-dir"
+make_rnd_suite
+ALT_RUN_DIR="${PWD}/alt"
+run_ok "${TEST_NAME}" \
+ cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
+__OUT__
+run_ok "${TEST_NAME}-check-link" test -L "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}-rm-link" rm "${RND_SUITE_RUNDIR}"
+run_ok "${TEST_NAME}-rm-alt-run-dir" rm -r "${ALT_RUN_DIR}"
+purge_rnd_suite
+
+# 2. If reg already exists (as a directory).
+TEST_NAME="${TEST_NAME_BASE}-alt-exists1"
+make_rnd_suite
+ALT_RUN_DIR="${PWD}/alt"
+mkdir -p "${RND_SUITE_RUNDIR}"
+run_fail "${TEST_NAME}" \
+ cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__OUT__
+SuiteServiceFileError: Run directory '${RND_SUITE_RUNDIR}' already exists.
+__OUT__
+purge_rnd_suite
+
+# 3. If reg already exists (as a valid symlink).
+TEST_NAME="${TEST_NAME_BASE}-alt-exists2"
+make_rnd_suite
+ALT_RUN_DIR="${PWD}/alt"
+TDIR=$(mktemp -d)
+mkdir -p "$(dirname "${RND_SUITE_RUNDIR}")"
+ln -s "${TDIR}" "${RND_SUITE_RUNDIR}"
+run_fail "${TEST_NAME}" \
+ cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__OUT__
+SuiteServiceFileError: Symlink '${RND_SUITE_RUNDIR}' already points to ${TDIR}.
+__OUT__
+purge_rnd_suite
+rm -rf "${TDIR}"
+
exit
diff --git a/tests/functional/validate/48-reg-then-pwd.t b/tests/functional/validate/48-reg-then-pwd.t
index 4ba9e2b30a3..03ce3d54899 100755
--- a/tests/functional/validate/48-reg-then-pwd.t
+++ b/tests/functional/validate/48-reg-then-pwd.t
@@ -43,7 +43,7 @@ __FLOW_CONFIG__
run_fail "${TEST_NAME_BASE}" cylc validate "${SUITE_NAME}"
# This should validate registered good suite
-cylc register "${SUITE_NAME}" "${PWD}/good"
+cylc install "${SUITE_NAME}" "${PWD}/good"
run_ok "${TEST_NAME_BASE}" cylc validate "${SUITE_NAME}"
purge
From a25cc3edeb9a4f11b7f3648f9036bda536ae09a9 Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Fri, 27 Nov 2020 12:11:59 +0000
Subject: [PATCH 03/13] Install logging
Cylc install, prohibit source in cylc-run
---
cylc/flow/config.py | 4 +-
cylc/flow/pathutil.py | 51 ++-
cylc/flow/scheduler.py | 31 +-
cylc/flow/scheduler_cli.py | 25 +-
cylc/flow/scripts/install.py | 105 +++--
cylc/flow/suite_files.py | 328 ++++++++++++---
cylc/flow/task_remote_mgr.py | 31 +-
setup.cfg | 1 -
.../functional/authentication/00-shared-fs.t | 2 +-
.../01-remote-suite-same-name.t | 4 +-
.../authentication/02-suite2-stop-suite1.t | 8 +-
tests/functional/cli/01-help.t | 1 -
tests/functional/cylc-diff/01-same.t | 2 +-
tests/functional/cylc-install/00-simple.t | 180 +++-----
tests/functional/cylc-install/02-failures.t | 104 +++++
.../cylc-install/03-file-transfer.t | 114 ++++++
tests/functional/deprecations/03-suiterc.t | 4 +-
tests/functional/lib/bash/test_header | 8 +-
tests/functional/registration/00-simple.t | 1 -
tests/functional/validate/48-reg-then-pwd.t | 6 +-
tests/integration/test_scan_api.py | 4 +-
.../unit/test_install.py | 28 +-
tests/unit/test_pathutil.py | 20 +-
tests/unit/test_suite_files.py | 387 ++++++------------
24 files changed, 881 insertions(+), 568 deletions(-)
create mode 100644 tests/functional/cylc-install/02-failures.t
create mode 100644 tests/functional/cylc-install/03-file-transfer.t
rename cylc/flow/scripts/get_directory.py => tests/unit/test_install.py (54%)
mode change 100755 => 100644
diff --git a/cylc/flow/config.py b/cylc/flow/config.py
index f2617b7cb19..290fcced138 100644
--- a/cylc/flow/config.py
+++ b/cylc/flow/config.py
@@ -60,7 +60,7 @@
import cylc.flow.flags
from cylc.flow.graphnode import GraphNodeParser
from cylc.flow.pathutil import (
- get_suite_run_dir,
+ get_workflow_run_dir,
get_suite_run_log_dir,
get_suite_run_share_dir,
get_suite_run_work_dir,
@@ -157,7 +157,7 @@ def __init__(
self.suite = suite # suite name
self.fpath = fpath # suite definition
self.fdir = os.path.dirname(fpath)
- self.run_dir = run_dir or get_suite_run_dir(self.suite)
+ self.run_dir = run_dir or get_workflow_run_dir(self.suite)
self.log_dir = log_dir or get_suite_run_log_dir(self.suite)
self.share_dir = share_dir or get_suite_run_share_dir(self.suite)
self.work_dir = work_dir or get_suite_run_work_dir(self.suite)
diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index 25281a6ae04..56b508b7ba7 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -17,6 +17,7 @@
import os
from os.path import expandvars
+import re
from shutil import rmtree
from cylc.flow import LOG
@@ -46,11 +47,11 @@ def get_remote_suite_work_dir(platform, suite, *args):
)
-def get_suite_run_dir(suite, *args):
- """Return local suite run directory, join any extra args."""
+def get_workflow_run_dir(flow_name, *args):
+ """Return local workflow run directory, join any extra args."""
return expandvars(
os.path.join(
- get_platform()['run directory'], suite, *args
+ get_platform()['run directory'], flow_name, *args
)
)
@@ -58,35 +59,35 @@ def get_suite_run_dir(suite, *args):
def get_suite_run_job_dir(suite, *args):
"""Return suite run job (log) directory, join any extra args."""
return expandvars(
- get_suite_run_dir(suite, 'log', 'job', *args)
+ get_workflow_run_dir(suite, 'log', 'job', *args)
)
def get_suite_run_log_dir(suite, *args):
"""Return suite run log directory, join any extra args."""
- return expandvars(get_suite_run_dir(suite, 'log', 'suite', *args))
+ return expandvars(get_workflow_run_dir(suite, 'log', 'suite', *args))
def get_suite_run_log_name(suite):
"""Return suite run log file path."""
- path = get_suite_run_dir(suite, 'log', 'suite', 'log')
+ path = get_workflow_run_dir(suite, 'log', 'suite', 'log')
return expandvars(path)
def get_suite_file_install_log_name(suite):
"""Return suite file install log file path."""
- path = get_suite_run_dir(suite, 'log', 'suite', 'file-installation-log')
+ path = get_workflow_run_dir(suite, 'log', 'suite', 'file-installation-log')
return expandvars(path)
def get_suite_run_config_log_dir(suite, *args):
"""Return suite run flow.cylc log directory, join any extra args."""
- return expandvars(get_suite_run_dir(suite, 'log', 'flow-config', *args))
+ return expandvars(get_workflow_run_dir(suite, 'log', 'flow-config', *args))
def get_suite_run_pub_db_name(suite):
"""Return suite run public database file path."""
- return expandvars(get_suite_run_dir(suite, 'log', 'db'))
+ return expandvars(get_workflow_run_dir(suite, 'log', 'db'))
def get_suite_run_share_dir(suite, *args):
@@ -105,7 +106,8 @@ def get_suite_run_work_dir(suite, *args):
def get_suite_test_log_name(suite):
"""Return suite run ref test log file path."""
- return expandvars(get_suite_run_dir(suite, 'log', 'suite', 'reftest.log'))
+ return expandvars(
+ get_workflow_run_dir(suite, 'log', 'suite', 'reftest.log'))
def make_suite_run_tree(suite):
@@ -113,7 +115,7 @@ def make_suite_run_tree(suite):
cfg = glbl_cfg().get()
# Roll archive
archlen = cfg['scheduler']['run directory rolling archive length']
- dir_ = os.path.expandvars(get_suite_run_dir(suite))
+ dir_ = os.path.expandvars(get_workflow_run_dir(suite))
for i in range(archlen, -1, -1): # archlen...0
if i > 0:
dpath = f'{dir_}.{i}'
@@ -128,7 +130,7 @@ def make_suite_run_tree(suite):
os.rename(dpath, f'{dir_}.{i + 1}')
# Create
for dir_ in (
- get_suite_run_dir(suite),
+ get_workflow_run_dir(suite),
get_suite_run_log_dir(suite),
get_suite_run_job_dir(suite),
get_suite_run_config_log_dir(suite),
@@ -141,10 +143,9 @@ def make_suite_run_tree(suite):
LOG.debug(f'{dir_}: directory created')
-def make_localhost_symlinks(suite):
+def make_localhost_symlinks(rund, flow_name, log_type=LOG):
"""Creates symlinks for any configured symlink dirs from glbl_cfg."""
- dirs_to_symlink = get_dirs_to_symlink('localhost', suite)
- rund = get_suite_run_dir(suite)
+ dirs_to_symlink = get_dirs_to_symlink('localhost', flow_name)
for key, value in dirs_to_symlink.items():
if key == 'run':
dst = rund
@@ -156,10 +157,12 @@ def make_localhost_symlinks(suite):
f'Unable to create symlink to {src}.'
f' \'{value}\' contains an invalid environment variable.'
' Please check configuration.')
+ if log_type:
+ log_type.info(f"Creating symlink from {src} to {dst}")
make_symlink(src, dst)
-def get_dirs_to_symlink(install_target, suite):
+def get_dirs_to_symlink(install_target, flow_name):
"""Returns dictionary of directories to symlink from glbcfg."""
dirs_to_symlink = {}
symlink_conf = glbl_cfg().get(['symlink dirs'])
@@ -168,12 +171,12 @@ def get_dirs_to_symlink(install_target, suite):
return dirs_to_symlink
base_dir = symlink_conf[install_target]['run']
if base_dir is not None:
- dirs_to_symlink['run'] = os.path.join(base_dir, 'cylc-run', suite)
+ dirs_to_symlink['run'] = os.path.join(base_dir, 'cylc-run', flow_name)
for dir_ in ['log', 'share', 'share/cycle', 'work']:
link = symlink_conf[install_target][dir_]
if link is None or link == base_dir:
continue
- dirs_to_symlink[dir_] = os.path.join(link, 'cylc-run', suite, dir_)
+ dirs_to_symlink[dir_] = os.path.join(link, 'cylc-run', flow_name, dir_)
return dirs_to_symlink
@@ -231,3 +234,15 @@ def remove_dir(path):
else:
LOG.debug(f'Removing directory: {path}')
rmtree(path)
+ raise WorkflowFilesError(f"Error when symlinking '{exc}'")
+
+
+def get_next_rundir_number(run_path):
+ """Return the new run number"""
+ run_n_path = os.path.expanduser(os.path.join(run_path, "runN"))
+ try:
+ old_run_path = os.readlink(run_n_path)
+ last_run_num = re.search(r'(?:run)(\d*$)', old_run_path).group(1)
+ return int(last_run_num) + 1
+ except OSError:
+ return 1
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index c98df73438b..7a5e2fe2b38 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -63,7 +63,7 @@
from cylc.flow.parsec.util import printcfg
from cylc.flow.parsec.validate import DurationFloat
from cylc.flow.pathutil import (
- get_suite_run_dir,
+ get_workflow_run_dir,
get_suite_run_log_dir,
get_suite_run_config_log_dir,
get_suite_run_share_dir,
@@ -261,9 +261,8 @@ def __init__(self, reg, options, is_restart=False):
)
# directory information
- self.suite_dir = suite_files.get_suite_source_dir(self.suite)
self.flow_file = suite_files.get_flow_file(self.suite)
- self.suite_run_dir = get_suite_run_dir(self.suite)
+ self.suite_run_dir = get_workflow_run_dir(self.suite)
self.suite_work_dir = get_suite_run_work_dir(self.suite)
self.suite_share_dir = get_suite_run_share_dir(self.suite)
self.suite_log_dir = get_suite_run_log_dir(self.suite)
@@ -280,18 +279,13 @@ def __init__(self, reg, options, is_restart=False):
async def install(self):
"""Get the filesystem in the right state to run the flow.
- * Install.
* Install authentication files.
* Build the directory tree.
* Copy Python files.
"""
- # Install
- try:
- suite_files.get_suite_source_dir(self.suite)
- except SuiteServiceFileError:
- # Source path is assumed to be the run directory
- suite_files.install(self.suite, get_suite_run_dir(self.suite))
+
+ make_suite_run_tree(self.suite)
# Create ZMQ keys
key_housekeeping(self.suite, platform=self.options.host or 'localhost')
@@ -301,21 +295,18 @@ async def install(self):
suite_files.get_suite_srv_dir(self.suite),
['etc/job.sh'])
- make_suite_run_tree(self.suite)
# Copy local python modules from source to run directory
for sub_dir in ["python", os.path.join("lib", "python")]:
# TODO - eventually drop the deprecated "python" sub-dir.
- suite_py = os.path.join(self.suite_dir, sub_dir)
- if (os.path.realpath(self.suite_dir) !=
- os.path.realpath(self.suite_run_dir) and
- os.path.isdir(suite_py)):
+ suite_py = os.path.join(self.suite_run_dir, sub_dir) # change to rundir
+ if os.path.isdir(suite_py)):
suite_run_py = os.path.join(self.suite_run_dir, sub_dir)
try:
rmtree(suite_run_py)
except OSError:
pass
copytree(suite_py, suite_run_py)
- sys.path.append(os.path.join(self.suite_dir, sub_dir))
+ sys.path.append(os.path.join(self.suite_run_dir, sub_dir))
async def initialise(self):
"""Initialise the components and sub-systems required to run the flow.
@@ -376,7 +367,7 @@ async def initialise(self):
proc_pool=self.proc_pool,
suite_run_dir=self.suite_run_dir,
suite_share_dir=self.suite_share_dir,
- suite_source_dir=self.suite_dir
+ suite_source_dir=self.suite_run_dir
)
self.task_events_mgr = TaskEventsManager(
@@ -422,10 +413,8 @@ async def configure(self):
# Copy local python modules from source to run directory
for sub_dir in ["python", os.path.join("lib", "python")]:
# TODO - eventually drop the deprecated "python" sub-dir.
- suite_py = os.path.join(self.suite_dir, sub_dir)
- if (os.path.realpath(self.suite_dir) !=
- os.path.realpath(self.suite_run_dir) and
- os.path.isdir(suite_py)):
+ suite_py = os.path.join(self.suite_run_dir, sub_dir)
+ if os.path.isdir(suite_py)):
suite_run_py = os.path.join(self.suite_run_dir, sub_dir)
try:
rmtree(suite_run_py)
diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py
index 452c9205035..f9c983c0d1e 100644
--- a/cylc/flow/scheduler_cli.py
+++ b/cylc/flow/scheduler_cli.py
@@ -32,9 +32,9 @@
Options
)
from cylc.flow.pathutil import (
- get_suite_run_dir,
+ get_workflow_run_dir,
get_suite_run_log_name,
- get_suite_file_install_log_name, make_localhost_symlinks)
+ get_suite_file_install_log_name)
from cylc.flow.remote import _remote_cylc_cmd
from cylc.flow.scheduler import Scheduler, SchedulerError
from cylc.flow.scripts import cylc_header
@@ -261,19 +261,6 @@ def get_option_parser(is_restart, add_std_opts=False):
get_option_parser(is_restart=True, add_std_opts=True), DEFAULT_OPTS)
-def _auto_install():
- """Install a suite installed in the cylc-run directory."""
- try:
- reg = suite_files.install()
- except SuiteServiceFileError as exc:
- sys.exit(exc)
- # Replace this process with "cylc run REG ..." for 'ps -f'.
- os.execv(
- sys.argv[0],
- [sys.argv[0]] + sys.argv[1:] + [reg]
- )
-
-
def _open_logs(reg, no_detach):
"""Open Cylc log handlers for a flow run."""
if not no_detach:
@@ -320,8 +307,7 @@ def scheduler_cli(parser, options, args, is_restart=False):
suite_files.detect_old_contact_file(reg)
except SuiteServiceFileError as exc:
sys.exit(exc)
- make_localhost_symlinks(reg)
- _check_registration(reg)
+ _check_installation(reg)
# re-execute on another host if required
_distribute(options.host, is_restart)
@@ -367,8 +353,8 @@ def scheduler_cli(parser, options, args, is_restart=False):
def _check_installation(reg):
- """Ensure the flow is installed."""
- suite_run_dir = get_suite_run_dir(reg)
+ """Check the flow is installed."""
+ suite_run_dir = get_workflow_run_dir(reg)
if not os.path.exists(suite_run_dir):
sys.stderr.write(f'suite service directory not found '
f'at: {suite_run_dir}\n')
@@ -435,6 +421,7 @@ def restart(parser, options, *args):
@cli_function(partial(get_option_parser, is_restart=False))
def run(parser, options, *args):
"""Implement cylc run."""
+
if not args:
_auto_install()
if options.startcp:
diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py
index 517324e71bd..f67fa3d6b12 100755
--- a/cylc/flow/scripts/install.py
+++ b/cylc/flow/scripts/install.py
@@ -18,36 +18,43 @@
"""cylc install [OPTIONS] ARGS
-Install a new suite.
+Install a new workflow.
+Install the name REG. The workflow server program can then be started, stopped,
+and targeted by name REG. (Note that "cylc run" can also install workflows on
+the fly).
-Install the name REG for the suite definition in PATH. The suite server
-program can then be started, stopped, and targeted by name REG. (Note that
-"cylc run" can also install suites on the fly).
+Installation creates a workflow run directory "~/cylc-run/REG/", with a run
+directory "~/cylc-run/REG/run1" containing a "_cylc-install/source" symlink to
+the source directory.
+Any files or directories (excluding .git, .svn) from the source directory are
+copied to the new run directory.
+A .service directory will also be created and used for server authentication
+files at run time.
-Installation creates a suite run directory "~/cylc-run/REG/" containing a
-".service/source" symlink to the suite definition PATH. The .service directory
-will also be used for server authentication files at run time.
-Suite names can be hierarchical, corresponding to the path under ~/cylc-run.
+Workflow names can be hierarchical, corresponding to the path under ~/cylc-run.
Examples:
- # Register PATH/flow.cylc as dogs/fido
- # (with run directory ~/cylc-run/dogs/fido)
- $ cylc install dogs/fido PATH
-
- # Install $PWD/flow.cylc as dogs/fido.
+ # Install workflow dogs/fido from $PWD
+ # (with run directory ~/cylc-run/dogs/fido/run1)
+ # (if "run1" exists this will increment)
$ cylc install dogs/fido
- # Install $PWD/flow.cylc as the parent directory
- # name: $(basename $PWD).
- $ cylc install
+ # Install $PWD/flow.cylc with specified flow name: fido
+ # (with run directory ~/cylc-run/fido/run1)
+ $ cylc install --flow-name=fido
+
+ # Install PATH/TO/FLOW/flow.cylc
+ $ cylc install --directory=PATH/TO/FLOW
-The same suite can be installed with multiple names; this results in multiple
-suite run directories that link to the same suite definition.
+ # Install cats/flow.cylc
+ # (with run directory ~/cylc-run/cats/paws)
+ # overriding the run1, run2, run3 etc structure.
+ $ cylc install --run-name=paws
-To "unregister" a suite, delete or rename its run directory (renaming it under
-~/cylc-run effectively re-registers the original suite with the new name).
+The same workflow can be installed with multiple names; this results in
+multiple workflow run directories that link to the same suite definition.
"""
@@ -59,35 +66,71 @@
from cylc.flow.exceptions import PluginError
from cylc.flow.option_parsers import CylcOptionParser as COP
from cylc.flow.pathutil import get_suite_run_dir
-from cylc.flow.suite_files import parse_suite_arg, install
+from cylc.flow.suite_files import install_workflow
from cylc.flow.terminal import cli_function
def get_option_parser():
parser = COP(
__doc__, comms=True, prep=True,
- argdoc=[("[REG]", "Workflow name"),
- ("[PATH]", "Workflow definition directory (defaults to $PWD)")
+ argdoc=[("[REG]", "Workflow name")
])
parser.add_option(
- "--redirect", help="Allow an existing suite name and run directory"
- " to be used with another suite.",
- action="store_true", default=False, dest="redirect")
+ "--flow-name",
+ help="Install into ~/cylc-run/flow-name/runN ",
+ action="store",
+ metavar="MY_FLOW",
+ default=None,
+ dest="flow_name")
+
+ parser.add_option(
+ "--directory", "-C",
+ help=(
+ "Install the workflow found in path specfied."
+ " This defaults to $PWD."),
+ action="store",
+ metavar="PATH/TO/FLOW",
+ default=None,
+ dest="source")
parser.add_option(
- "--run-name", help="Name the run ",
- action="store", metavar="RUNDIR", default=None, dest="rundir")
+ "--run-name",
+ help="Name the run.",
+ action="store",
+ metavar="RUN_NAME",
+ default=None,
+ dest="run_name")
parser.add_option(
- "--run-dir", help="Symlink $HOME/cylc-run/REG to RUNDIR/REG.",
- action="store", metavar="RUNDIR", default=None, dest="rundir")
+ "--no-run-name",
+ help="Install the workflow directly into ~/cylc-run/$(basename $PWD)",
+ action="store_true",
+ default=False,
+ dest="no_run_name")
+
+ parser.add_option(
+ "--no-symlink-dirs",
+ help="Use this option to override creating default local symlinks.",
+ action="store_true",
+ default=False,
+ dest="no_symlinks")
return parser
@cli_function(get_option_parser)
def main(parser, opts, flow_name=None, src=None):
+ if opts.no_run_name and opts.run_name:
+ parser.error(
+ """options --no-run-name and --run-name are mutually exclusive.
+ Use one or the other""")
+ flow_name = install_workflow(
+ flow_name=opts.flow_name,
+ source=opts.source,
+ run_name=opts.run_name,
+ no_run_name=opts.no_run_name,
+ no_symlinks=opts.no_symlinks)
for entry_point in pkg_resources.iter_entry_points(
'cylc.pre_configure'
):
@@ -102,8 +145,6 @@ def main(parser, opts, flow_name=None, src=None):
exc
) from None
- flow_name = install(reg, src, redirect=opts.redirect, rundir=opts.rundir)
-
for entry_point in pkg_resources.iter_entry_points(
'cylc.post_install'
):
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index 152a2c7df10..5799f5140ac 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -16,6 +16,8 @@
"""Suite service files management."""
+import aiofiles
+from enum import Enum
import os
from pathlib import Path
from random import shuffle
@@ -25,15 +27,19 @@
import time
import zmq.auth
-import aiofiles
-
from cylc.flow import LOG
from cylc.flow.exceptions import (
- CylcError, PlatformLookupError, SuiteServiceFileError, TaskRemoteMgmtError,
+ CylcError,
+ PlatformLookupError,
+ SuiteServiceFileError,
+ TaskRemoteMgmtError,
WorkflowFilesError)
import cylc.flow.flags
from cylc.flow.pathutil import (
- get_suite_run_dir, make_localhost_symlinks, remove_dir)
+ get_workflow_run_dir,
+ make_localhost_symlinks,
+ remove_dir,
+ get_next_rundir_number)
from cylc.flow.platforms import (
get_platform, get_install_target_to_platforms_map)
from cylc.flow.hostuserutil import (
@@ -44,8 +50,8 @@
from cylc.flow.remote import construct_ssh_cmd
from cylc.flow.suite_db_mgr import SuiteDatabaseManager
from cylc.flow.unicode_rules import SuiteNameValidator
-
-from enum import Enum
+from cylc.flow.loggingutil import CylcLogFormatter
+from cylc.flow.wallclock import get_current_time_string
class KeyType(Enum):
@@ -142,6 +148,9 @@ class SuiteFiles:
SUITE_RC = 'suite.rc'
"""Deprecated workflow configuration file."""
+ SOURCE = 'source'
+ """Symlink to the workflow source directory (For workflow dir)"""
+
class Service:
"""The directory containing Cylc system files."""
@@ -149,14 +158,11 @@ class Service:
"""The name of this directory."""
CONTACT = 'contact'
- """Contains settings for the running suite.
+ """Contains settings for the running workflow.
For details of the fields see ``ContactFileFields``.
"""
- SOURCE = 'source'
- """Symlink to the suite definition (suite dir)."""
-
PUBLIC_FILE_EXTENSION = '.key'
PRIVATE_FILE_EXTENSION = '.key_secret'
"""Keyword identifiers used to form the certificate names.
@@ -164,6 +170,15 @@ class Service:
be renamed, but we hard-code them since they can't be extracted easily.
"""
+ class Install:
+ """The directory containing install source link."""
+
+ DIRNAME = '_cylc-install'
+ """The name of this directory."""
+
+ SOURCE = 'source'
+ """Symlink to the workflow definition (For run dir)."""
+
class ContactFileFields:
"""Field names present in ``SuiteFiles.Service.CONTACT``.
@@ -236,6 +251,12 @@ class ContactFileFields:
* ssh -n "%(host)s" kill %(pid)s # final brute force!
"""
+INSTALL_LOG = logging.getLogger('cylc-install')
+INSTALL_LOG.addHandler(logging.NullHandler())
+INSTALL_LOG.setLevel(logging.INFO)
+
+FORBIDDEN_SOURCE_DIR = ['log', 'share', 'work', SuiteFiles.Install.DIRNAME]
+
def detect_old_contact_file(reg, check_host_port=None):
"""Detect old suite contact file.
@@ -341,36 +362,10 @@ def get_contact_file(reg):
get_suite_srv_dir(reg), SuiteFiles.Service.CONTACT)
-def get_flow_file(reg, suite_owner=None):
+def get_flow_file(reg):
"""Return the path of a suite's flow.cylc file."""
return os.path.join(
- get_suite_source_dir(reg, suite_owner), SuiteFiles.FLOW_FILE)
-
-
-def get_suite_source_dir(reg, suite_owner=None):
- """Return the source directory path of a suite.
-
- Will install un-installed suites located in the cylc run dir.
- """
- srv_d = get_suite_srv_dir(reg, suite_owner)
- fname = os.path.join(srv_d, SuiteFiles.Service.SOURCE)
- try:
- source = os.readlink(fname)
- except OSError:
- suite_d = os.path.dirname(srv_d)
- if os.path.exists(suite_d) and not is_remote_user(suite_owner):
- # suite exists but is not yet installed
- install(flow_name=reg, source=suite_d)
- return suite_d
- raise SuiteServiceFileError(f"Suite not found: {reg}")
- else:
- if not os.path.isabs(source):
- source = os.path.normpath(os.path.join(srv_d, source))
- flow_file_path = os.path.join(source, SuiteFiles.FLOW_FILE)
- if not os.path.exists(flow_file_path):
- # suite exists but is probably using deprecated suite.rc
- install(flow_name=reg, source=source)
- return source
+ get_workflow_run_dir(reg), SuiteFiles.FLOW_FILE)
def get_suite_srv_dir(reg, suite_owner=None):
@@ -383,7 +378,7 @@ def get_suite_srv_dir(reg, suite_owner=None):
or os.getenv("CYLC_SUITE_NAME") != reg
or os.getenv("CYLC_SUITE_OWNER") != suite_owner
):
- run_d = get_suite_run_dir(reg)
+ run_d = get_workflow_run_dir(reg)
return os.path.join(run_d, SuiteFiles.Service.DIRNAME)
@@ -437,7 +432,7 @@ def parse_suite_arg(options, arg):
if arg == '.':
arg = os.getcwd()
try:
- path = get_flow_file(arg, options.suite_owner)
+ path = get_flow_file(arg)
name = arg
except SuiteServiceFileError:
arg = os.path.abspath(arg)
@@ -449,11 +444,12 @@ def parse_suite_arg(options, arg):
path = os.path.join(arg, SuiteFiles.SUITE_RC)
if not os.path.exists(path):
raise SuiteServiceFileError(
- f'no flow.cylc or suite.rc in {arg}')
+ f'no {SuiteFiles.FLOW_FILE} or {SuiteFiles.SUITE_RC}'
+ f' in {arg}')
else:
LOG.warning(
f'The filename "{SuiteFiles.SUITE_RC}" is deprecated '
- f'in favor of "{SuiteFiles.FLOW_FILE}".')
+ f'in favour of "{SuiteFiles.FLOW_FILE}".')
else:
path = arg
name = os.path.basename(os.path.dirname(arg))
@@ -885,12 +881,13 @@ def _validate_reg(reg):
f'suite name cannot be an absolute path: {reg}')
-def check_nested_run_dirs(reg):
+def check_nested_run_dirs(run_dir, flow_name):
"""Disallow nested run dirs e.g. trying to install foo/bar where foo is
- already a valid suite directory.
+ already a valid workflow directory.
Args:
- reg (str): suite name
+ run_dir (path): run directory path
+ flow_name (str): workflow name
Raise:
WorkflowFilesError:
@@ -899,23 +896,23 @@ def check_nested_run_dirs(reg):
depth)
"""
exc_msg = (
- 'Nested run directories not allowed - cannot install suite name '
+ 'Nested run directories not allowed - cannot install workflow name '
'"%s" as "%s" is already a valid run directory.')
def _check_child_dirs(path, depth_count=1):
for result in os.scandir(path):
if result.is_dir() and not result.is_symlink():
if is_valid_run_dir(result.path):
- raise WorkflowFilesError(exc_msg % (reg, result.path))
+ raise WorkflowFilesError(exc_msg % (flow_name, result.path))
if depth_count < MAX_SCAN_DEPTH:
_check_child_dirs(result.path, depth_count + 1)
- reg_path = os.path.normpath(reg)
+ reg_path = os.path.normpath(run_dir)
parent_dir = os.path.dirname(reg_path)
- while parent_dir != '':
+ while parent_dir not in ['', '/']:
if is_valid_run_dir(parent_dir):
raise WorkflowFilesError(
- exc_msg % (reg, get_cylc_run_abs_path(parent_dir)))
+ exc_msg % (parent_dir, get_cylc_run_abs_path(parent_dir)))
parent_dir = os.path.dirname(parent_dir)
reg_path = get_cylc_run_abs_path(reg_path)
@@ -945,4 +942,233 @@ def get_cylc_run_abs_path(path):
"""
if os.path.isabs(path):
return path
- return get_suite_run_dir(path)
+ return get_workflow_run_dir(path)
+
+
+def _open_install_log(reg, rund, is_reload=False):
+ """Open Cylc log handlers for an install."""
+ time_str = get_current_time_string(
+ override_use_utc=True, use_basic_format=True,
+ display_sub_seconds=False
+ )
+ if is_reload:
+ load_type = "reload"
+ else:
+ load_type = "install"
+ rund = Path(rund).expanduser()
+ log_path = Path(
+ rund,
+ 'log',
+ 'install',
+ f"{time_str}-{load_type}.log")
+ log_parent_dir = log_path.parent
+ log_parent_dir.mkdir(exist_ok=True, parents=True)
+ handler = logging.FileHandler(log_path)
+ handler.setFormatter(CylcLogFormatter())
+ INSTALL_LOG.addHandler(handler)
+
+
+def _close_install_log():
+ """Close Cylc log handlers for a flow run."""
+ for handler in INSTALL_LOG.handlers:
+ try:
+ handler.close()
+ except IOError:
+ pass
+
+
+def get_rsync_rund_cmd(src, dst, restart=False):
+ """Create and return the rsync command used for cylc install/re-install.
+ Args:
+ src (str): file path location of source directory
+ dst (str): file path location of destination directory
+ restart (bool): indicate restart (--delete option added)
+
+ Return: rsync_cmd: command used for rsync.
+
+ """
+
+ rsync_cmd = ["rsync"]
+ rsync_cmd.append("-av")
+ if restart:
+ rsync_cmd.append('--delete')
+ ignore_dirs = ['.git', '.svn','.cylcignore']
+ for exclude in ignore_dirs:
+ if Path(src).joinpath(exclude).exists():
+ rsync_cmd.append(f"--exclude={exclude}")
+ if Path(src).joinpath('.cylcignore').exists():
+ rsync_cmd.append("--exclude-from=.cylcignore")
+ rsync_cmd.append(f"{src}/")
+ rsync_cmd.append(f"{dst}/")
+
+ return rsync_cmd
+
+
+def install_workflow(flow_name=None, source=None, run_name=None,
+ no_run_name=False, no_symlinks=False):
+ """Install a workflow, or renew its installation.
+
+ Create symlink to suite source location, creating any symlinks for run,
+ work, log, share, share/cycle directories.
+
+ Args:
+ flow_name (str): workflow name, default basename($PWD).
+ source (str): directory location of flow.cylc file, default $PWD.
+ run_name (str): name of the run, overides run1, run2, run 3 etc...
+ If specified, cylc install will not create runN
+ symlink.
+ rundir (str): for overriding the default cylc-run directory.
+
+ Return:
+ str: The installed suite name (which may be computed here).
+
+ Raise:
+ SuiteServiceFileError:
+ No flow.cylc file found in source location.
+ Illegal name (can look like a relative path, but not absolute).
+ Another suite already has this name (unless --redirect).
+ Trying to install a workflow that is nested inside of another.
+ """
+ if not source:
+ source = Path.cwd()
+ elif Path(source).name == SuiteFiles.FLOW_FILE:
+ source = Path(source).parent
+ source = Path(source).expanduser()
+ if not flow_name:
+ flow_name = (Path.cwd().stem)
+ validate_flow_name(flow_name)
+ if run_name == '_cylc-install':
+ raise SuiteServiceFileError(
+ 'Run name cannot be "_cylc-install".'
+ ' Please choose another run name.')
+ validate_source_dir(source)
+ run_path_base = Path(get_workflow_run_dir(flow_name)).expanduser()
+ relink = False
+ if no_run_name:
+ rundir = run_path_base
+ elif run_name:
+ rundir = run_path_base.joinpath(run_name)
+ else:
+ run_n = Path(run_path_base, 'runN').expanduser()
+ run_num = get_next_rundir_number(run_path_base)
+ rundir = Path(run_path_base, f'run{run_num}')
+ if run_num == 1 and rundir.exists():
+ SuiteServiceFileError(
+ f"This path: {rundir} exists. Try using --run-name")
+ unlink_runN(run_n)
+ relink = True
+ check_nested_run_dirs(rundir, flow_name)
+ try:
+ rundir.mkdir(exist_ok=True)
+ except OSError as e:
+ if e.strerror == "File exists":
+ raise SuiteServiceFileError(f"Run directory already exists : {e}")
+ _open_install_log(flow_name, rundir)
+ # create source symlink to be used as the basis of ensuring runs are
+ # from a constistent source dir.
+ base_source_link = run_path_base.joinpath(SuiteFiles.Install.SOURCE)
+ if not base_source_link.exists():
+ run_path_base.joinpath(SuiteFiles.Install.SOURCE).symlink_to(source)
+ if relink:
+ link_runN(rundir)
+ if not no_symlinks:
+ make_localhost_symlinks(rundir, flow_name, log_type=INSTALL_LOG)
+ create_workflow_srv_dir(rundir)
+ # flow.cylc must exist so we can detect accidentally reversed args.
+ flow_file_path = source.joinpath(SuiteFiles.FLOW_FILE)
+ if not flow_file_path.is_file():
+ # If using deprecated suite.rc, symlink it into flow.cylc:
+ suite_rc_path = source.joinpath(SuiteFiles.SUITE_RC)
+ if suite_rc_path.is_file():
+ flow_file_path.symlink_to(suite_rc_path)
+ INSTALL_LOG.warning(
+ f'The filename "{SuiteFiles.SUITE_RC}" is deprecated in favour'
+ f' of "{SuiteFiles.FLOW_FILE}". Symlink created.')
+ else:
+ raise SuiteServiceFileError(
+ f'no {SuiteFiles.FLOW_FILE} or {SuiteFiles.SUITE_RC}'
+ f' in {source}')
+ rsync_cmd = get_rsync_rund_cmd(source, rundir)
+ proc = Popen(rsync_cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True)
+ stdout, stderr = proc.communicate()
+ INSTALL_LOG.info(f"Copying files from {source} to {rundir}")
+ INSTALL_LOG.info(f"{stdout}")
+ if stderr:
+ INSTALL_LOG.warning(
+ f"An error occurred when copying files from {source} to {rundir}")
+ INSTALL_LOG.warning(f" Error: {stderr}")
+ cylc_install = Path(rundir, SuiteFiles.Install.DIRNAME)
+ cylc_install.mkdir(parents=True)
+ source_link = cylc_install.joinpath(SuiteFiles.Install.SOURCE)
+ # check source link matches the source symlink from workflow dir.
+ if os.readlink(base_source_link) == str(source):
+ INSTALL_LOG.info(f"Creating symlink from {source_link}")
+ source_link.symlink_to(source)
+ else:
+ raise SuiteServiceFileError(
+ "Source directory between runs are not consistent")
+ INSTALL_LOG.info(f'INSTALLED {flow_name} from {source} -> {rundir}')
+ print(f'INSTALLED {flow_name} from {source} -> {rundir}')
+ _close_install_log()
+ return flow_name
+
+
+def create_workflow_srv_dir(rundir=None, source=None):
+ """Create suite service directory"""
+
+ workflow_srv_d = rundir.joinpath(SuiteFiles.Service.DIRNAME)
+ workflow_srv_d.mkdir(exist_ok=True, parents=True)
+
+
+def validate_flow_name(flow_name):
+ is_valid, message = SuiteNameValidator.validate(flow_name)
+ if not is_valid:
+ raise SuiteServiceFileError(f'Invalid workflow name - {message}')
+ if Path.is_absolute(Path(flow_name)):
+ raise SuiteServiceFileError(
+ f'Workflow name cannot be an absolute path: {flow_name}')
+
+
+def validate_source_dir(source):
+ """Ensure the source directory is valid.
+
+ Args:
+ source (path): Path to source directory
+ Raises:
+ SuiteServiceFileError:
+ If log, share, work or _cylc-install directories exist in the
+ source directory.
+ Cylc installing from within the cylc-run dir
+ """
+ # Ensure source dir does not contain log, share, work, _cylc-install
+ for dir_ in FORBIDDEN_SOURCE_DIR:
+ path_to_check = Path(source, dir_)
+ if path_to_check.exists():
+ raise SuiteServiceFileError(
+ f'Installation failed. - {dir_} exists in source directory.')
+ cylc_run_dir = Path(
+ get_platform()['run directory'].replace('$HOME', '~')
+ ).expanduser()
+ if os.path.abspath(os.path.realpath(cylc_run_dir)
+ ) in os.path.abspath(os.path.realpath(source)):
+ raise SuiteServiceFileError(
+ f'Installation failed. Source directory should not be in'
+ f' {cylc_run_dir}')
+
+
+def unlink_runN(run_n):
+ """Remove symlink runN"""
+ try:
+ Path(run_n).unlink()
+ except OSError:
+ pass
+
+
+def link_runN(latest_run):
+ """Create symlink runN, pointing at the latest run"""
+ latest_run = Path(latest_run).expanduser()
+ run_n = Path(latest_run.parent, 'runN')
+ try:
+ run_n.symlink_to(latest_run)
+ except OSError:
+ pass
diff --git a/cylc/flow/task_remote_mgr.py b/cylc/flow/task_remote_mgr.py
index 2d8568495ca..c06fbeb09e8 100644
--- a/cylc/flow/task_remote_mgr.py
+++ b/cylc/flow/task_remote_mgr.py
@@ -36,7 +36,7 @@
from cylc.flow.pathutil import (
get_remote_suite_run_dir,
get_dirs_to_symlink,
- get_suite_run_dir)
+ get_workflow_run_dir)
from cylc.flow.remote import construct_rsync_over_ssh_cmd
from cylc.flow.subprocctx import SubProcContext
from cylc.flow.suite_files import (
@@ -293,6 +293,35 @@ def _remote_init_callback(
pass
install_target = platform['install target']
if proc_ctx.ret_code == 0:
+ if REMOTE_INIT_DONE in proc_ctx.out:
+ src_path = get_workflow_run_dir(self.suite)
+ dst_path = get_remote_suite_run_dir(platform, self.suite)
+ try:
+ process = procopen(construct_rsync_over_ssh_cmd(
+ src_path,
+ dst_path,
+ platform,
+ self.rsync_includes),
+ stdoutpipe=True,
+ stderrpipe=True,
+ universal_newlines=True)
+
+ out, err = process.communicate(timeout=600)
+ install_target = platform['install target']
+ if out:
+ RSYNC_LOG.info(
+ 'File installation information for '
+ f'{install_target}:\n {out}')
+ LOG.info("File installation complete.")
+ if err:
+ LOG.error(
+ 'File installation error on '
+ f'{install_target}:\n {err}')
+ except Exception as ex:
+ LOG.error(f"Problem during rsync: {ex}")
+ self.remote_init_map[self.install_target] = (
+ REMOTE_INIT_FAILED)
+ return
if "KEYSTART" in proc_ctx.out:
regex_result = re.search(
'KEYSTART((.|\n|\r)*)KEYEND', proc_ctx.out)
diff --git a/setup.cfg b/setup.cfg
index 1203feb79ad..2e75878ad03 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -79,7 +79,6 @@ cylc.command =
ext-trigger = cylc.flow.scripts.ext_trigger:main
extract-resources = cylc.flow.scripts.extract_resources:main
function-run = cylc.flow.scripts.function_run:main
- get-directory = cylc.flow.scripts.get_directory:main
get-site-config = cylc.flow.scripts.get_site_config:main
get-suite-config = cylc.flow.scripts.get_suite_config:main
get-suite-contact = cylc.flow.scripts.get_suite_contact:main
diff --git a/tests/functional/authentication/00-shared-fs.t b/tests/functional/authentication/00-shared-fs.t
index 688c45daa34..d451d0bf5b3 100755
--- a/tests/functional/authentication/00-shared-fs.t
+++ b/tests/functional/authentication/00-shared-fs.t
@@ -28,7 +28,7 @@ SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BA
SUITE_RUN_DIR="$RUN_DIR/${SUITE_NAME}"
mkdir -p "$(dirname "${SUITE_RUN_DIR}")"
cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}" "${SUITE_RUN_DIR}"
-cylc install "${SUITE_NAME}" "${SUITE_RUN_DIR}" 2>'/dev/null'
+cylc install --flow-name=${SUITE_NAME} --no-run-name >&2
run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
diff --git a/tests/functional/authentication/01-remote-suite-same-name.t b/tests/functional/authentication/01-remote-suite-same-name.t
index 45854c1648f..3727d294121 100755
--- a/tests/functional/authentication/01-remote-suite-same-name.t
+++ b/tests/functional/authentication/01-remote-suite-same-name.t
@@ -34,8 +34,8 @@ scp ${SSH_OPTS} -pqr "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/"* \
# shellcheck disable=SC2086
run_ok "${TEST_NAME_BASE}-register" \
ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" \
- CYLC_VERSION="$(cylc version)" cylc install "${SUITE_NAME}" \
- "cylc-run/${SUITE_NAME}"
+ CYLC_VERSION="$(cylc version)" cylc install --flow-name="${SUITE_NAME}" \
+ --no-run-name --directory="cylc-run/${SUITE_NAME}"
suite_run_ok "${TEST_NAME_BASE}" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
diff --git a/tests/functional/authentication/02-suite2-stop-suite1.t b/tests/functional/authentication/02-suite2-stop-suite1.t
index bcc44fa9086..dd8a1778120 100755
--- a/tests/functional/authentication/02-suite2-stop-suite1.t
+++ b/tests/functional/authentication/02-suite2-stop-suite1.t
@@ -26,7 +26,7 @@ NAME2="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}-2
SUITE1_RUND="${RUND}/${NAME1}"
mkdir -p "${SUITE1_RUND}"
cp -p "${TEST_SOURCE_DIR}/basic/flow.cylc" "${SUITE1_RUND}"
-cylc install "${NAME1}" "${SUITE1_RUND}"
+cylc install --flow-name="${NAME1}" --no-run-name --directory="${SUITE1_RUND}"
SUITE2_RUND="${RUND}/${NAME2}"
mkdir -p "${SUITE2_RUND}"
cat >"${SUITE2_RUND}/flow.cylc" <<__FLOW_CONFIG__
@@ -39,11 +39,11 @@ cat >"${SUITE2_RUND}/flow.cylc" <<__FLOW_CONFIG__
[[t1]]
script=cylc shutdown "${NAME1}"
__FLOW_CONFIG__
-cylc install "${NAME2}" "${SUITE2_RUND}"
+cylc install --flow-name="${NAME2}" --directory="${SUITE2_RUND}" --no-run-name
cylc run --no-detach "${NAME1}" 1>'1.out' 2>&1 &
SUITE_RUN_DIR="${SUITE1_RUND}" poll_suite_running
run_ok "${TEST_NAME_BASE}" cylc run --no-detach --abort-if-any-task-fails "${NAME2}"
cylc shutdown "${NAME1}" --max-polls=20 --interval=1 1>'/dev/null' 2>&1 || true
-purge "${NAME1}"
-purge "${NAME2}"
+# purge "${NAME1}"
+# purge "${NAME2}"
exit
diff --git a/tests/functional/cli/01-help.t b/tests/functional/cli/01-help.t
index 5da7cfc0a19..ddaa2faa98f 100755
--- a/tests/functional/cli/01-help.t
+++ b/tests/functional/cli/01-help.t
@@ -43,7 +43,6 @@ __STDERR__
run_fail "${TEST_NAME_BASE}-get" cylc get
cmp_ok "${TEST_NAME_BASE}-get.stderr" <<'__STDERR__'
cylc get: is ambiguous for:
- cylc get-directory
cylc get-site-config
cylc get-suite-config
cylc get-suite-contact
diff --git a/tests/functional/cylc-diff/01-same.t b/tests/functional/cylc-diff/01-same.t
index a8d9124abc1..3a14430486d 100755
--- a/tests/functional/cylc-diff/01-same.t
+++ b/tests/functional/cylc-diff/01-same.t
@@ -33,7 +33,7 @@ init_suite "${TEST_NAME_BASE}-1" "${PWD}/flow.cylc"
SUITE_NAME1="${SUITE_NAME}"
# shellcheck disable=SC2153
SUITE_NAME2="${SUITE_NAME1%1}2"
-cylc install "${SUITE_NAME2}" "${TEST_DIR}/${SUITE_NAME1}" 2>'/dev/null'
+cylc install --flow-name="${SUITE_NAME2}" --directory="${TEST_DIR}/${SUITE_NAME1}" --no-run-name 2>'/dev/null'
run_ok "${TEST_NAME_BASE}" cylc diff "${SUITE_NAME1}" "${SUITE_NAME2}"
cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__
diff --git a/tests/functional/cylc-install/00-simple.t b/tests/functional/cylc-install/00-simple.t
index 9aa239a2f7f..65f42bf4293 100755
--- a/tests/functional/cylc-install/00-simple.t
+++ b/tests/functional/cylc-install/00-simple.t
@@ -16,7 +16,7 @@
# along with this program. If not, see .
#------------------------------------------------------------------------------
-# Test suite registration
+# Test workflow installation
export RND_SUITE_NAME
export RND_SUITE_SOURCE
@@ -42,165 +42,111 @@ function purge_rnd_suite() {
}
. "$(dirname "$0")/test_header"
-set_test_number 33
+set_test_number 18
-# Use $SUITE_NAME and $SUITE_RUN_DIR defined by test_header
-#------------------------------
-# Test fail no suite source dir
-TEST_NAME="${TEST_NAME_BASE}-nodir"
-make_rnd_suite
-rm -rf "${RND_SUITE_SOURCE}"
-run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
-__ERR__
-purge_rnd_suite
-
-#---------------------------
-# Test fail no flow.cylc file
-TEST_NAME="${TEST_NAME_BASE}-nodir"
-make_rnd_suite
-rm -f "${RND_SUITE_SOURCE}/flow.cylc"
-run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
-__ERR__
-purge_rnd_suite
-
-#-------------------------------------------------------
-# Test default name: "cylc reg" (suite in $PWD, no args)
-TEST_NAME="${TEST_NAME_BASE}-pwd1"
+# Test default name: "cylc install" (suite in $PWD, no args)
+TEST_NAME="${TEST_NAME_BASE}-basic"
make_rnd_suite
pushd "${RND_SUITE_SOURCE}" || exit 1
-run_ok "${TEST_NAME}" cylc install
+run_ok "${TEST_NAME}" cylc install
+
contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED $RND_SUITE_NAME -> ${RND_SUITE_SOURCE}
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
__OUT__
popd || exit 1
purge_rnd_suite
-#--------------------------------------------------
-# Test default path: "cylc reg REG" (suite in $PWD)
+# Test default path: "cylc install REG" (flow in $PWD)
TEST_NAME="${TEST_NAME_BASE}-pwd2"
make_rnd_suite
pushd "${RND_SUITE_SOURCE}" || exit 1
run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}"
contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
__OUT__
popd || exit 1
purge_rnd_suite
-#-------------------------
-# Test "cylc reg REG PATH"
-TEST_NAME="${TEST_NAME_BASE}-normal"
+
+
+# Test default path: "cylc install REG" --no-run-name (flow in $PWD)
+TEST_NAME="${TEST_NAME_BASE}-pwd-no-run-name"
make_rnd_suite
-run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" --no-run-name
contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}
__OUT__
+popd || exit 1
purge_rnd_suite
-
-#--------------------------------------------------------------------
-# Test register existing run directory: "cylc reg REG ~/cylc-run/REG"
-TEST_NAME="${TEST_NAME_BASE}-reg-run-dir"
+# Test "cylc install REG" flow-name given (flow in $PWD)
+TEST_NAME="${TEST_NAME_BASE}-flow-name"
make_rnd_suite
-mkdir -p "${RND_SUITE_RUNDIR}"
-cp "${RND_SUITE_SOURCE}/flow.cylc" "${RND_SUITE_RUNDIR}"
-run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}-olaf"
contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
+INSTALLED ${RND_SUITE_NAME}-olaf from ${RND_SUITE_SOURCE} -> ${RUN_DIR}/${RND_SUITE_NAME}-olaf/run1
__OUT__
-SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")"
-run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}"
-# Run it twice
-run_ok "${TEST_NAME}-2" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
-contains_ok "${TEST_NAME}-2.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
-__OUT__
-SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")"
-run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}"
+popd || exit 1
+rm -rf ${RUN_DIR}/${RND_SUITE_NAME}-olaf
purge_rnd_suite
-#----------------------------------------------------------------
-# Test fail "cylc reg REG PATH" where REG already points to PATH2
-TEST_NAME="${TEST_NAME_BASE}-dup1"
+# Test "cylc install REG" flow-name given (flow in $PWD)
+TEST_NAME="${TEST_NAME_BASE}-flow-name-no-run-name"
make_rnd_suite
-run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-RND_SUITE_NAME1="${RND_SUITE_NAME}"
-RND_SUITE_SOURCE1="${RND_SUITE_SOURCE}"
-RND_SUITE_RUNDIR1="${RND_SUITE_RUNDIR}"
-make_rnd_suite
-TEST_NAME="${TEST_NAME_BASE}-dup2"
-run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: the name '${RND_SUITE_NAME1}' already points to ${RND_SUITE_SOURCE1}.
-Use --redirect to re-use an existing name and run directory.
-__ERR__
-# Now force it
-TEST_NAME="${TEST_NAME_BASE}-dup3"
-run_ok "${TEST_NAME}" cylc install --redirect "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
-sed -i 's/^\t//; s/^.* WARNING - /WARNING - /' "${TEST_NAME}.stderr"
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-WARNING - the name '${RND_SUITE_NAME1}' points to ${RND_SUITE_SOURCE1}.
-It will now be redirected to ${RND_SUITE_SOURCE}.
-Files in the existing ${RND_SUITE_NAME1} run directory will be overwritten.
-__ERR__
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}-olaf" --no-run-name
contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME1} -> ${RND_SUITE_SOURCE}
+INSTALLED ${RND_SUITE_NAME}-olaf from ${RND_SUITE_SOURCE} -> ${RUN_DIR}/${RND_SUITE_NAME}-olaf
__OUT__
-
-TEST_NAME="${TEST_NAME_BASE}-get-dir"
-run_ok "${TEST_NAME}" cylc get-directory "${RND_SUITE_NAME1}"
-contains_ok "${TEST_NAME}.stdout" <<__ERR__
-${RND_SUITE_SOURCE}
-__ERR__
-
+popd || exit 1
+rm -rf ${RUN_DIR}/${RND_SUITE_NAME}-olaf
purge_rnd_suite
-purge_rnd_suite "${RND_SUITE_SOURCE1}" "${RND_SUITE_RUNDIR1}"
-#-----------------------
-# Test alternate run dir
-# 1. Normal case.
-TEST_NAME="${TEST_NAME_BASE}-alt-run-dir"
+# Test "cylc install" --directory given (flow in --directory)
+TEST_NAME="${TEST_NAME_BASE}-option--directory"
make_rnd_suite
-ALT_RUN_DIR="${PWD}/alt"
-run_ok "${TEST_NAME}" \
- cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
+run_ok "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}" --directory="${RND_SUITE_SOURCE}"
contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
__OUT__
-run_ok "${TEST_NAME}-check-link" test -L "${RND_SUITE_RUNDIR}"
-run_ok "${TEST_NAME}-rm-link" rm "${RND_SUITE_RUNDIR}"
-run_ok "${TEST_NAME}-rm-alt-run-dir" rm -r "${ALT_RUN_DIR}"
purge_rnd_suite
-# 2. If reg already exists (as a directory).
-TEST_NAME="${TEST_NAME_BASE}-alt-exists1"
+# Test running cylc install twice increments run dirs correctly
+TEST_NAME="${TEST_NAME_BASE}-install-twice"
make_rnd_suite
-ALT_RUN_DIR="${PWD}/alt"
-mkdir -p "${RND_SUITE_RUNDIR}"
-run_fail "${TEST_NAME}" \
- cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__OUT__
-SuiteServiceFileError: Run directory '${RND_SUITE_RUNDIR}' already exists.
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
+__OUT__
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run2
__OUT__
+popd || exit 1
+
purge_rnd_suite
-# 3. If reg already exists (as a valid symlink).
-TEST_NAME="${TEST_NAME_BASE}-alt-exists2"
+
+# Test -C option
+TEST_NAME="${TEST_NAME_BASE}-option-C"
make_rnd_suite
-ALT_RUN_DIR="${PWD}/alt"
-TDIR=$(mktemp -d)
-mkdir -p "$(dirname "${RND_SUITE_RUNDIR}")"
-ln -s "${TDIR}" "${RND_SUITE_RUNDIR}"
-run_fail "${TEST_NAME}" \
- cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__OUT__
-SuiteServiceFileError: Symlink '${RND_SUITE_RUNDIR}' already points to ${TDIR}.
+run_ok "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}" -C "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
__OUT__
purge_rnd_suite
-rm -rf "${TDIR}"
+
+# things that still need testing:
+
+# symlink/--no-symlink-dirs option
+# workflow name not abs path:
+# SuiteServiceFileError: Workflow name cannot be an absolute path:
+# test by sending run_ok "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_RUNDIR}-olaf"
+# source dir contains forbidden
+
+
exit
diff --git a/tests/functional/cylc-install/02-failures.t b/tests/functional/cylc-install/02-failures.t
new file mode 100644
index 00000000000..21cfc183d58
--- /dev/null
+++ b/tests/functional/cylc-install/02-failures.t
@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+#------------------------------------------------------------------------------
+# Test workflow installation failures
+
+
+export RND_SUITE_NAME
+export RND_SUITE_SOURCE
+export RND_SUITE_RUNDIR
+
+function make_rnd_suite() {
+ # Create a randomly-named suite source directory.
+ # Define its run directory.
+ RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)
+ RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}"
+ mkdir -p "${RND_SUITE_SOURCE}"
+ touch "${RND_SUITE_SOURCE}/flow.cylc"
+ RND_SUITE_RUNDIR="${RUN_DIR}/${RND_SUITE_NAME}"
+}
+
+function purge_rnd_suite() {
+ # Remove the suite source created by make_rnd_suite().
+ # And remove its run-directory too.
+ RND_SUITE_SOURCE=${1:-$RND_SUITE_SOURCE}
+ RND_SUITE_RUNDIR=${2:-$RND_SUITE_RUNDIR}
+ rm -rf "${RND_SUITE_SOURCE}"
+ rm -rf "${RND_SUITE_RUNDIR}"
+}
+
+. "$(dirname "$0")/test_header"
+set_test_number 16
+
+# Test fail no suite source dir
+TEST_NAME="${TEST_NAME_BASE}-nodir"
+make_rnd_suite
+rm -rf "${RND_SUITE_SOURCE}"
+run_fail "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}" --no-run-name -C "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
+__ERR__
+purge_rnd_suite
+
+
+# Test fail no flow.cylc or suite.rc file
+TEST_NAME="${TEST_NAME_BASE}-no-flow-file"
+make_rnd_suite
+rm -f "${RND_SUITE_SOURCE}/flow.cylc"
+run_fail "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}" -C "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
+__ERR__
+purge_rnd_suite
+
+# Test cylc install fails when given flow-name that is an absolute path
+TEST_NAME="${TEST_NAME_BASE}-no-abs-path-flow-name"
+make_rnd_suite
+rm -f "${RND_SUITE_SOURCE}/flow.cylc"
+run_fail "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_SOURCE}" -C "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+SuiteServiceFileError: Workflow name cannot be an absolute path: ${RND_SUITE_SOURCE}
+__ERR__
+purge_rnd_suite
+
+
+# Test cylc install can not be run from within the cylc-run directory
+TEST_NAME="${TEST_NAME_BASE}-forbid-cylc-run-dir-install"
+BASE_NAME="cylctb-${CYLC_TEST_TIME_INIT}"
+mkdir -p ${RUN_DIR}/${BASE_NAME}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME} && cd $_
+touch flow.cylc
+run_fail "${TEST_NAME}" cylc install
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+SuiteServiceFileError: Installation failed. Source directory should not be in ${RUN_DIR}
+__ERR__
+rm -rf ${RUN_DIR}/${BASE_NAME}
+
+
+# Test source dir can not contain '_cylc-install, log, share, work' dirs
+for DIR in 'work' 'share' 'log' '_cylc-install'; do
+ TEST_NAME="${TEST_NAME_BASE}-${DIR}-forbidden-in-source"
+ make_rnd_suite
+ pushd "${RND_SUITE_SOURCE}" || exit 1
+ mkdir ${DIR}
+ run_fail "${TEST_NAME}" cylc install
+ contains_ok "${TEST_NAME}.stderr" <<__ERR__
+SuiteServiceFileError: Installation failed. - ${DIR} exists in source directory.
+__ERR__
+ popd || exit 1
+ purge_rnd_suite
+done
diff --git a/tests/functional/cylc-install/03-file-transfer.t b/tests/functional/cylc-install/03-file-transfer.t
new file mode 100644
index 00000000000..c17f0c35f30
--- /dev/null
+++ b/tests/functional/cylc-install/03-file-transfer.t
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+#------------------------------------------------------------------------------
+# Test rsync of workflow installation
+. "$(dirname "$0")/test_header"
+set_test_number 6
+
+export RND_SUITE_NAME
+export RND_SUITE_SOURCE
+export RND_SUITE_RUNDIR
+
+function make_rnd_suite() {
+ # Create a randomly-named suite source directory.
+ # Define its run directory.
+ RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)
+ RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}"
+ mkdir -p "${RND_SUITE_SOURCE}"
+ touch "${RND_SUITE_SOURCE}/flow.cylc"
+ RND_SUITE_RUNDIR="${RUN_DIR}/${RND_SUITE_NAME}"
+}
+
+function purge_rnd_suite() {
+ # Remove the suite source created by make_rnd_suite().
+ # And remove its run-directory too.
+ rm -rf "${RND_SUITE_SOURCE}"
+ rm -rf "${RND_SUITE_RUNDIR}"
+}
+
+# Test cylc install copies files to run dir successfully.
+TEST_NAME="${TEST_NAME_BASE}-basic"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+mkdir .git .svn dir1 dir2
+touch .git/file1 .svn/file1 dir1/file1 dir2/file1 file1 file2
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" --no-run-name
+
+tree -a -I '*.log|03-file-transfer*' "${RND_SUITE_RUNDIR}/" > 'basic-tree.out'
+cmp_ok 'basic-tree.out' <<__OUT__
+${RND_SUITE_RUNDIR}/
+├── _cylc-install
+│  └── source -> ${RND_SUITE_SOURCE}
+├── dir1
+│  └── file1
+├── dir2
+│  └── file1
+├── file1
+├── file2
+├── flow.cylc
+├── log
+│  └── install
+├── .service
+└── source -> ${RND_SUITE_SOURCE}
+
+8 directories, 5 files
+__OUT__
+
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}
+__OUT__
+popd || exit 1
+purge_rnd_suite
+
+
+# Test cylc install copies files to run dir successfully, exluding files from .cylcignore file.
+TEST_NAME="${TEST_NAME_BASE}-cylcignore-file"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+mkdir .git .svn dir1 dir2 extradir1 extradir2
+touch .git/file1 .svn/file1 dir1/file1 dir2/file1 extradir1/file1 extradir2/file1 file1 file2 .cylcignore
+cat > .cylcignore <<__END__
+dir*
+extradir*
+file2
+__END__
+
+run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" --no-run-name
+
+tree -a -I '*.log|03-file-transfer*' "${RND_SUITE_RUNDIR}/" > 'cylc-ignore-tree.out'
+cmp_ok 'cylc-ignore-tree.out' <<__OUT__
+${RND_SUITE_RUNDIR}/
+├── _cylc-install
+│  └── source -> ${RND_SUITE_SOURCE}
+├── file1
+├── flow.cylc
+├── log
+│  └── install
+├── .service
+└── source -> ${RND_SUITE_SOURCE}
+
+6 directories, 2 files
+__OUT__
+
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}
+__OUT__
+popd || exit 1
+purge_rnd_suite
+
+
diff --git a/tests/functional/deprecations/03-suiterc.t b/tests/functional/deprecations/03-suiterc.t
index e787b5aa0e1..182f2e8625f 100644
--- a/tests/functional/deprecations/03-suiterc.t
+++ b/tests/functional/deprecations/03-suiterc.t
@@ -38,8 +38,8 @@ __FLOW__
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate .
-TEST_NAME="${TEST_NAME_BASE}-register"
-run_ok "${TEST_NAME}" cylc install "${SUITE_NAME}"
+TEST_NAME="${TEST_NAME_BASE}-install"
+run_ok "${TEST_NAME}" cylc install --flow-name=${SUITE_NAME} --no-run-name
exists_ok "flow.cylc"
diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header
index 3d3d6e749be..ee8309ae999 100644
--- a/tests/functional/lib/bash/test_header
+++ b/tests/functional/lib/bash/test_header
@@ -424,7 +424,7 @@ init_suite() {
SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}"
mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
cat "${FLOW_CONFIG}" >"${TEST_DIR}/${SUITE_NAME}/flow.cylc"
- cylc install "${SUITE_NAME}" "${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
+ cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
cd "${TEST_DIR}/${SUITE_NAME}"
}
@@ -433,10 +433,10 @@ install_suite() {
local TEST_SOURCE_BASE="${2:-${TEST_NAME}}"
SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}"
SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}"
- mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
+ mkdir -p "${TEST_DIR}/${SUITE_NAME}/" # make source dir
cp -r "${TEST_SOURCE_DIR}/${TEST_SOURCE_BASE}/"* "${TEST_DIR}/${SUITE_NAME}/"
- cylc install "${SUITE_NAME}" "${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
- cd "${TEST_DIR}/${SUITE_NAME}"
+ cylc install --no-run-name --flow-name"${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
+ cd "${TEST_DIR}/${SUITE_NAME}/"
}
log_scan () {
diff --git a/tests/functional/registration/00-simple.t b/tests/functional/registration/00-simple.t
index ed875dec81f..4d929adb557 100755
--- a/tests/functional/registration/00-simple.t
+++ b/tests/functional/registration/00-simple.t
@@ -152,7 +152,6 @@ REGISTERED ${RND_SUITE_NAME1} -> ${RND_SUITE_SOURCE}
__OUT__
TEST_NAME="${TEST_NAME_BASE}-get-dir"
-run_ok "${TEST_NAME}" cylc get-directory "${RND_SUITE_NAME1}"
contains_ok "${TEST_NAME}.stdout" <<__ERR__
${RND_SUITE_SOURCE}
__ERR__
diff --git a/tests/functional/validate/48-reg-then-pwd.t b/tests/functional/validate/48-reg-then-pwd.t
index 03ce3d54899..190df61bb92 100755
--- a/tests/functional/validate/48-reg-then-pwd.t
+++ b/tests/functional/validate/48-reg-then-pwd.t
@@ -15,7 +15,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#-------------------------------------------------------------------------------
-# Test validation order, registered suites before current working directory.
+# Test validation order, installed suites before current working directory.
. "$(dirname "$0")/test_header"
set_test_number 2
@@ -42,8 +42,8 @@ __FLOW_CONFIG__
# This should validate bad suite under current directory
run_fail "${TEST_NAME_BASE}" cylc validate "${SUITE_NAME}"
-# This should validate registered good suite
-cylc install "${SUITE_NAME}" "${PWD}/good"
+# This should validate installed good suite
+cylc install --flow-name="${SUITE_NAME}" -C "${PWD}/good"
run_ok "${TEST_NAME_BASE}" cylc validate "${SUITE_NAME}"
purge
diff --git a/tests/integration/test_scan_api.py b/tests/integration/test_scan_api.py
index 48bf691ceaa..858d0d10482 100644
--- a/tests/integration/test_scan_api.py
+++ b/tests/integration/test_scan_api.py
@@ -259,12 +259,12 @@ async def test_scan_cleans_stuck_contact_files(
schd = scheduler(reg)
srv_dir = Path(run_dir, reg, SuiteFiles.Service.DIRNAME)
tmp_dir = test_dir / 'srv'
- cont = srv_dir / SuiteFiles.Service.CONTACT
+ cont = run_dir / SuiteFiles.Service.CONTACT
# run the flow, copy the contact, stop the flow, copy back the contact
async with run(schd):
# remove the source symlink to avoid recursion
- (srv_dir / SuiteFiles.Service.SOURCE).unlink()
+ (run_dir / SuiteFiles.Install.SOURCE).unlink()
copytree(srv_dir, tmp_dir)
rmtree(srv_dir)
copytree(tmp_dir, srv_dir)
diff --git a/cylc/flow/scripts/get_directory.py b/tests/unit/test_install.py
old mode 100755
new mode 100644
similarity index 54%
rename from cylc/flow/scripts/get_directory.py
rename to tests/unit/test_install.py
index 70b5b432f0c..7423857685a
--- a/cylc/flow/scripts/get_directory.py
+++ b/tests/unit/test_install.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python3
-
# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
#
@@ -16,27 +14,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-"""cylc get-directory REG
-
-Retrieve and print the source directory location of suite REG.
-
-Here's an easy way to move to a suite source directory:
- $ cd $(cylc get-dir REG)."""
-
-from cylc.flow.option_parsers import CylcOptionParser as COP
-from cylc.flow.suite_files import get_suite_source_dir
-from cylc.flow.terminal import cli_function
-
-
-def get_option_parser():
- return COP(__doc__, prep=True)
+from cylc.flow.exceptions import SuiteServiceFileError
+from cylc.flow.suite_files import validate_source_dir
-@cli_function(get_option_parser)
-def main(parser, options, suite):
- """Implement "cylc get-directory"."""
- print(get_suite_source_dir(suite, options.suite_owner))
+# def test_validate_source_dir(source):
+# expected = raise(SuiteServiceFileError)
+# source_dir =
-if __name__ == "__main__":
- main()
+
\ No newline at end of file
diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py
index df555f07c72..4102e63ce5c 100644
--- a/tests/unit/test_pathutil.py
+++ b/tests/unit/test_pathutil.py
@@ -29,7 +29,7 @@
get_remote_suite_run_dir,
get_remote_suite_run_job_dir,
get_remote_suite_work_dir,
- get_suite_run_dir,
+ get_workflow_run_dir,
get_suite_run_job_dir,
get_suite_run_log_dir,
get_suite_run_log_name,
@@ -111,7 +111,7 @@ def test_get_suite_run_dirs(self, mocked_platform):
# args = extra *args
# tail2 = expected tail of return value from extra args
for func, tail1 in (
- (get_suite_run_dir, ''),
+ (get_workflow_run_dir, ''),
(get_suite_run_job_dir, '/log/job'),
(get_suite_run_log_dir, '/log/suite'),
(get_suite_run_config_log_dir, '/log/flow-config'),
@@ -247,22 +247,22 @@ def test_get_dirs_to_symlink(
@patch('os.path.expandvars')
-@patch('cylc.flow.pathutil.get_suite_run_dir')
+@patch('cylc.flow.pathutil.get_workflow_run_dir')
@patch('cylc.flow.pathutil.make_symlink')
@patch('cylc.flow.pathutil.get_dirs_to_symlink')
def test_make_localhost_symlinks_calls_make_symlink_for_each_key_value_dir(
mocked_dirs_to_symlink,
mocked_make_symlink,
- mocked_get_suite_run_dir, mocked_expandvars):
+ mocked_get_workflow_run_dir, mocked_expandvars):
mocked_dirs_to_symlink.return_value = {
'run': '$DOH/suite3',
'log': '$DEE/suite3/log',
'share': '$DEE/suite3/share'}
- mocked_get_suite_run_dir.return_value = "rund"
+ mocked_get_workflow_run_dir.return_value = "rund"
mocked_expandvars.return_value = "expanded"
- make_localhost_symlinks('suite')
+ make_localhost_symlinks('rund', 'suite')
mocked_make_symlink.assert_has_calls([
call('expanded', 'rund'),
@@ -272,23 +272,23 @@ def test_make_localhost_symlinks_calls_make_symlink_for_each_key_value_dir(
@patch('os.path.expandvars')
-@patch('cylc.flow.pathutil.get_suite_run_dir')
+@patch('cylc.flow.pathutil.get_workflow_run_dir')
@patch('cylc.flow.pathutil.make_symlink')
@patch('cylc.flow.pathutil.get_dirs_to_symlink')
def test_incorrect_environment_variables_raise_error(
mocked_dirs_to_symlink,
mocked_make_symlink,
- mocked_get_suite_run_dir, mocked_expandvars):
+ mocked_get_workflow_run_dir, mocked_expandvars):
mocked_dirs_to_symlink.return_value = {
'run': '$doh/cylc-run/test_workflow'}
- mocked_get_suite_run_dir.return_value = "rund"
+ mocked_get_workflow_run_dir.return_value = "rund"
mocked_expandvars.return_value = "$doh"
with pytest.raises(WorkflowFilesError, match=r"Unable to create symlink"
r" to \$doh. '\$doh/cylc-run/test_workflow' contains an"
" invalid environment variable. Please check "
"configuration."):
- make_localhost_symlinks('test_workflow')
+ make_localhost_symlinks('rund', 'test_workflow')
@pytest.mark.parametrize(
diff --git a/tests/unit/test_suite_files.py b/tests/unit/test_suite_files.py
index a369aaa4f56..72f5c32487a 100644
--- a/tests/unit/test_suite_files.py
+++ b/tests/unit/test_suite_files.py
@@ -27,223 +27,6 @@
from cylc.flow.suite_files import check_nested_run_dirs
-def get_register_test_cases():
- """Test cases for suite_files.register function."""
- return [
- # 1 no parameters provided, current directory is not a symlink,
- # and contains a valid flow.cylc
- (None, # reg
- None, # source
- False, # redirect,
- "/home/user/cylc-run/suite1", # cwd
- False, # isabs
- True, # isfile
- "/home/user/cylc-run/suite1/.service", # suite_srv_dir
- "/home/user/cylc-run/suite1", # readlink
- None, # expected symlink
- "suite1", # expected return value
- None, # expected exception
- None # expected part of exception message
- ),
- # 2 suite name provided, current directory is not a symlink,
- # and contains a valid flow.cylc
- ("super-suite-2", # reg
- None, # source
- False, # redirect,
- "/home/user/cylc-run/suite2", # cwd
- False, # isabs
- True, # isfile
- "/home/user/cylc-run/suite2/.service", # suite_srv_dir
- "/home/user/cylc-run/suite2", # readlink
- None, # expected symlink
- "super-suite-2", # expected return value
- None, # expected exception
- None # expected part of exception message
- ),
- # 3 suite name and directory location of flow.cylc provided,
- # current directory is not a symlink, and contains a valid flow.cylc
- ("suite3", # reg
- "/home/user/cylc-run/suite3/flow.cylc", # source
- False, # redirect,
- "/home/user/cylc-run/suite3", # cwd
- False, # isabs
- True, # isfile
- "/home/user/cylc-run/suite3/.service", # suite_srv_dir
- "/home/user/cylc-run/suite3", # readlink
- None, # expected symlink
- "suite3", # expected return value
- None, # expected exception
- None # expected part of exception message
- ),
- # 4 suite name and directory location of flow.cylc provided,
- # current directory is not a symlink, but the flow.cylc does not
- # exist
- ("suite4", # reg
- "/home/user/cylc-run/suite4/suite.txt", # source
- False, # redirect,
- "/home/user/cylc-run/suite4", # cwd
- False, # isabs
- False, # isfile
- "/home/user/cylc-run/suite4/.service", # suite_srv_dir
- "/home/user/cylc-run/suite4", # readlink
- None, # expected symlink
- "suite4", # expected return value
- SuiteServiceFileError, # expected exception
- "no flow.cylc" # expected part of exception message
- ),
- # 5 the source directory and the resolved symlink for $SOURCE in
- # $SOURCE/.service are not the same directory. No redirect
- # specified, so it must raise an error
- ("suite5", # reg
- "/home/user/cylc-run/suite5/suite.txt", # source
- False, # redirect,
- "/home/user/cylc-run/suite5", # cwd
- False, # isabs
- True, # isfile
- "/home/user/cylc-run/suite5/.service", # suite_srv_dir
- "/home/hercules/cylc-run/suite5", # readlink
- "/home/user/cylc-run/suite5", # expected symlink
- "suite5", # expected return value
- SuiteServiceFileError, # expected exception
- "already points to" # expected part of exception message
- ),
- # 6 the source directory and the resolved symlink for $SOURCE in
- # $SOURCE/.service are not the same directory. The redirect
- # flag is true, so it must simply delete the old source link
- ("suite6", # reg
- "/home/user/cylc-run/suite6/flow.cylc", # source
- True, # redirect,
- "/home/user/cylc-run/suite6", # cwd
- False, # isabs
- True, # isfile
- "/home/hercules/cylc-run/suite6/.service", # suite_srv_dir
- "/home/hercules/cylc-run/suite6", # readlink
- "/home/user/cylc-run/suite6", # expected symlink
- "suite6", # expected return value
- None, # expected exception
- None # expected part of exception message
- ),
- # 7 the source directory and the resolved symlink for $SOURCE in
- # $SOURCE/.service are not the same directory. The redirect
- # flag is true. But the resolved orig_source's parent directory,
- # is the source directory. So the symlink must be '..'
- ("suite7", # reg
- "/home/user/cylc-run/suite7/flow.cylc", # source
- True, # redirect,
- "/home/user/cylc-run/suite7", # cwd
- False, # isabs
- True, # isfile
- "/home/user/cylc-run/suite7/.service", # suite_srv_dir
- "/home/user/cylc-run/suites/suite7", # readlink
- "..", # expected symlink
- "suite7", # expected return value
- None, # expected exception
- None # expected part of exception message
- ),
- # 8 fails to readlink, resulting in a new symlink created
- ("suite8", # reg
- "/home/user/cylc-run/suite8/flow.cylc", # source
- False, # redirect,
- "/home/user/cylc-run/suite8", # cwd
- False, # isabs
- True, # isfile
- "/home/user/cylc-run/suite8/.service", # suite_srv_dir
- OSError, # readlink
- "..", # expected symlink
- "suite8", # expected return value
- None, # expected exception
- None # expected part of exception message
- ),
- # 9 the suite name is an absolute path
- ("/suite9/", # reg
- None, # source
- False, # redirect,
- None, # cwd
- True, # isabs
- True, # isfile
- None, # suite_srv_dir
- None, # readlink
- None, # expected symlink
- None, # expected return value
- SuiteServiceFileError, # expected exception
- "cannot be an absolute path" # expected part of exception message
- ),
- # 10 invalid suite name
- ("-foo", # reg
- None, # source
- False, # redirect,
- None, # cwd
- True, # isabs
- True, # isfile
- None, # suite_srv_dir
- None, # readlink
- None, # expected symlink
- None, # expected return value
- SuiteServiceFileError, # expected exception
- "cannot start with: ``.``, ``-``" # expected part of exception msg
- )
- ]
-
-
-@mock.patch('cylc.flow.suite_files.make_localhost_symlinks')
-@mock.patch('os.unlink')
-@mock.patch('os.makedirs')
-@mock.patch('os.symlink')
-@mock.patch('os.readlink')
-@mock.patch('os.path.isfile')
-@mock.patch('os.path.isabs')
-@mock.patch('os.getcwd')
-@mock.patch('os.path.abspath')
-@mock.patch('cylc.flow.suite_files.get_suite_srv_dir')
-@mock.patch('cylc.flow.suite_files.check_nested_run_dirs')
-def test_register(mocked_check_nested_run_dirs,
- mocked_get_suite_srv_dir,
- mocked_abspath,
- mocked_getcwd,
- mocked_isabs,
- mocked_isfile,
- mocked_readlink,
- mocked_symlink,
- mocked_makedirs,
- mocked_unlink,
- mocked_make_localhost_symlinks
- ):
- """Test the register function."""
- def mkdirs_standin(_, exist_ok=False):
- return True
-
- mocked_abspath.side_effect = lambda x: x
- # mocked_check_nested_run_dirs - no side effect as we're just ignoring it
-
- for (reg, source, redirect, cwd, isabs, isfile, suite_srv_dir,
- readlink, expected_symlink, expected, e_expected,
- e_message) in get_register_test_cases():
- mocked_getcwd.side_effect = lambda: cwd
- mocked_isabs.side_effect = lambda x: isabs
-
- mocked_isfile.side_effect = lambda x: isfile
- mocked_get_suite_srv_dir.return_value = str(suite_srv_dir)
- mocked_makedirs.return_value = True
- mocked_unlink.return_value = True
- if readlink == OSError:
- mocked_readlink.side_effect = readlink
- else:
- mocked_readlink.side_effect = lambda x: readlink
-
- if e_expected is None:
- reg = suite_files.install(reg, source, redirect)
- assert reg == expected
- if mocked_symlink.call_count > 0:
- # first argument, of the first call
- arg0 = mocked_symlink.call_args[0][0]
- assert arg0 == expected_symlink
- else:
- with pytest.raises(e_expected) as exc:
- suite_files.install(reg, source, redirect)
- if e_message is not None:
- assert e_message in str(exc.value)
-
-
@pytest.mark.parametrize(
'path, expected',
[('a/b/c', '/mock_cylc_dir/a/b/c'),
@@ -299,48 +82,116 @@ def test_nested_run_dirs_raise_error(direction, monkeypatch):
with pytest.raises(WorkflowFilesError) as exc:
suite_files.check_nested_run_dirs(path)
assert 'Nested run directories not allowed' in str(exc.value)
+@pytest.mark.parametrize(
+ 'run_dir',
+ [
+ ('bright/falls/light'),
+ ('bright/falls/light/dark')
+ ]
+)
+def test_rundir_parent_that_does_not_contain_workflow_no_error(
+ run_dir, monkeypatch):
+ """Test that a workflow raises no error when a parent directory is not also
+ a workflow directory."""
+
+ monkeypatch.setattr('cylc.flow.suite_files.os.path.isdir',
+ lambda x: False if x.find('.service') > 0
+ else True)
+ monkeypatch.setattr(
+ 'cylc.flow.suite_files.get_cylc_run_abs_path', lambda x: x)
+ monkeypatch.setattr(
+ 'cylc.flow.suite_files.os.scandir', lambda x: [])
- else:
- dirs = ['a', 'a/a', 'a/R', 'a/a/a', 'a/a/R',
- 'a/b', 'a/b/a', 'a/b/b',
- 'a/c', 'a/c/a', 'a/c/a/a', 'a/c/a/a/a', 'a/c/a/a/a/R',
- 'a/d', 'a/d/a', 'a/d/a/a', 'a/d/a/a/a', 'a/d/a/a/a/a',
- 'a/d/a/a/a/a/R']
- run_dirs = [d for d in dirs if 'R' in d]
-
- def mock_scandir(path):
- return [mock.Mock(path=d, is_dir=lambda: True,
- is_symlink=lambda: False) for d in dirs
- if (d.startswith(path) and len(d) == len(path) + 2)]
- monkeypatch.setattr('cylc.flow.suite_files.os.scandir', mock_scandir)
- monkeypatch.setattr('cylc.flow.suite_files.os.path.isdir',
- lambda x: x in dirs)
- monkeypatch.setattr('cylc.flow.suite_files.is_valid_run_dir',
- lambda x: x in run_dirs)
+ try:
+ suite_files.check_nested_run_dirs(run_dir, 'placeholder_flow')
+ except Exception:
+ pytest.fail("check_nested_run_dirs raised exception unexpectedly.")
- # No run dir nested below - ok:
- for path in ('a/a/a', 'a/b'):
- suite_files.check_nested_run_dirs(path)
- # Run dir nested below - bad:
- for path in ('a', 'a/a', 'a/c'):
- with pytest.raises(WorkflowFilesError) as exc:
- check_nested_run_dirs(path)
- assert 'Nested run directories not allowed' in str(exc.value)
- for func in (suite_files.check_nested_run_dirs, suite_files.install):
- for path in ('a', 'a/a', 'a/c'):
- with pytest.raises(SuiteServiceFileError) as exc:
- func(path)
- assert 'Nested run directories not allowed' in str(exc.value)
- # Run dir nested below max scan depth - not ideal but passes:
- suite_files.check_nested_run_dirs('a/d')
+@pytest.mark.parametrize(
+ 'run_dir, srv_dir',
+ [
+ ('bright/falls/light', 'bright/falls/.service'),
+ ('bright/falls/light/dark', 'bright/falls/light/.service')
+ ]
+)
+def test_rundir_parent_that_contains_workflow_raises_error(
+ run_dir, srv_dir, monkeypatch):
+ """Test that a workflow that contains another worfkflow raises error."""
+
+ monkeypatch.setattr(
+ 'cylc.flow.suite_files.os.path.isdir', lambda x: x == srv_dir)
+ monkeypatch.setattr(
+ 'cylc.flow.suite_files.get_cylc_run_abs_path', lambda x: x)
+ monkeypatch.setattr(
+ 'cylc.flow.suite_files.os.scandir', lambda x: [])
+
+ with pytest.raises(SuiteServiceFileError) as exc:
+ suite_files.check_nested_run_dirs(run_dir, 'placeholder_flow')
+ assert 'Nested run directories not allowed' in str(exc.value)
+
+
+@pytest.mark.parametrize(
+ 'run_dir',
+ [
+ ('a'),
+ ('d/a'),
+ ('z/d/a/a')
+ ]
+)
+def test_rundir_children_that_do_not_contain_workflows_no_error(
+ run_dir, monkeypatch):
+ """Test that a run directory that contains no other workflows does not
+ raise an error."""
+
+ monkeypatch.setattr('cylc.flow.suite_files.os.path.isdir',
+ lambda x: False if x.find('.service')
+ else True)
+ monkeypatch.setattr(
+ 'cylc.flow.suite_files.get_cylc_run_abs_path',
+ lambda x: x)
+ monkeypatch.setattr('cylc.flow.suite_files.os.scandir',
+ lambda x: [
+ mock.Mock(path=run_dir[0:len(x) + 2],
+ is_symlink=lambda: False)])
+ try:
+ suite_files.check_nested_run_dirs(run_dir, 'placeholder_flow')
+ except Exception:
+ pytest.fail("check_nested_run_dirs raised exception unexpectedly.")
@pytest.mark.parametrize(
- 'reg, expected_err, expected_msg',
- [('foo/bar/', None, None),
- ('/foo/bar', SuiteServiceFileError, "cannot be an absolute path"),
- ('$HOME/alone', SuiteServiceFileError, "invalid suite name")]
+ 'run_dir, srv_dir',
+ [
+ ('a', 'a/R/.service'),
+ ('d/a', 'd/a/a/R/.service'),
+ ('z/d/a/a', 'z/d/a/a/R/.service')
+ ]
+)
+def test_rundir_children_that_contain_workflows_raise_error(
+ run_dir, srv_dir, monkeypatch):
+ """Test that a workflow cannot be contained in a subdir of another
+ workflow."""
+ monkeypatch.setattr('cylc.flow.suite_files.os.path.isdir',
+ lambda x: False if (
+ x.find('.service') > 0 and x != srv_dir)
+ else True)
+ monkeypatch.setattr(
+ 'cylc.flow.suite_files.get_cylc_run_abs_path',
+ lambda x: x)
+ monkeypatch.setattr('cylc.flow.suite_files.os.scandir',
+ lambda x: [
+ mock.Mock(path=srv_dir[0:len(x) + 2],
+ is_symlink=lambda: False)])
+
+ with pytest.raises(SuiteServiceFileError) as exc:
+ check_nested_run_dirs(run_dir, 'placeholder_flow')
+ assert 'Nested run directories not allowed' in str(exc.value)
+
+@pytest.mark.parametrize(
+ 'reg, expected_err',
+ [('foo/bar/', None),
+ ('/foo/bar', SuiteServiceFileError)]
)
def test_validate_reg(reg, expected_err, expected_msg):
if expected_err:
@@ -757,3 +608,33 @@ def test_remove_empty_reg_parents(tmp_path):
suite_files._remove_empty_reg_parents(reg, path)
assert tmp_path.joinpath('foo').exists() is False
assert tmp_path.exists() is True
+
+@pytest.mark.parametrize(
+ 'run_dir, srv_dir',
+ [
+ ('a', 'a/R/.service'),
+ ('d/a', 'd/a/a/R/.service'),
+ ('z/d/a/a', 'z/d/a/a/R/.service')
+ ]
+)
+def test_symlinkrundir_children_that_contain_workflows_raise_error(
+ run_dir, srv_dir, monkeypatch):
+ """Test that a workflow cannot be contained in a subdir of another
+ workflow."""
+ monkeypatch.setattr('cylc.flow.suite_files.os.path.isdir',
+ lambda x: False if (
+ x.find('.service') > 0 and x != srv_dir)
+ else True)
+ monkeypatch.setattr(
+ 'cylc.flow.suite_files.get_cylc_run_abs_path',
+ lambda x: x)
+ monkeypatch.setattr('cylc.flow.suite_files.os.scandir',
+ lambda x: [
+ mock.Mock(path=srv_dir[0:len(x) + 2],
+ is_symlink=lambda: True)])
+
+ try:
+ check_nested_run_dirs(run_dir, 'placeholder_flow')
+ except SuiteServiceFileError:
+ pytest.fail("Unexpected SuiteServiceFileError, Check symlink logic.")
+
From ca205175f209d3c44c6f3a76e2a750d5107db4f7 Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Fri, 11 Dec 2020 19:40:52 +0000
Subject: [PATCH 04/13] cylc install tests
Add cylc-install to cylc-clean
---
cylc/flow/pathutil.py | 23 +-
cylc/flow/scheduler.py | 36 ++-
cylc/flow/scheduler_cli.py | 28 ++-
cylc/flow/scripts/cylc.py | 3 +-
cylc/flow/scripts/graph.py | 4 +-
cylc/flow/scripts/install.py | 19 +-
cylc/flow/scripts/validate.py | 2 -
cylc/flow/suite_files.py | 219 +++++++++---------
cylc/flow/task_remote_mgr.py | 31 +--
.../registration/02-on-the-fly.t | 98 --------
.../flakyfunctional/registration/test_header | 1 -
.../xtriggers/01-suite_state.t | 2 +-
.../functional/authentication/00-shared-fs.t | 2 +-
.../01-remote-suite-same-name.t | 10 +-
.../authentication/02-suite2-stop-suite1.t | 17 +-
tests/functional/cylc-clean/00-basic.t | 8 +
tests/functional/cylc-install/00-simple.t | 27 +--
tests/functional/cylc-install/01-symlinks.t | 139 +++++++++++
tests/functional/cylc-install/02-failures.t | 55 +++--
.../cylc-install/03-file-transfer.t | 10 +-
tests/functional/cylc-run/01-invalid-suite.t | 2 +-
tests/functional/deprecations/03-suiterc.t | 2 +-
tests/functional/events/47-long-output.t | 6 +-
tests/functional/events/suite/flow.cylc | 2 +-
tests/functional/lib/bash/test_header | 4 +-
tests/functional/registration/00-simple.t | 205 ----------------
tests/functional/rose-conf/03-fileinstall.t | 12 +-
tests/functional/suite-state/00-polling.t | 4 +-
tests/functional/suite-state/01-polling.t | 4 +-
tests/functional/validate/48-reg-then-pwd.t | 2 +-
tests/integration/test_scan_api.py | 4 +-
tests/unit/test_install.py | 26 ---
tests/unit/test_pathutil.py | 4 +-
tests/unit/test_suite_files.py | 27 +--
34 files changed, 426 insertions(+), 612 deletions(-)
delete mode 100755 tests/flakyfunctional/registration/02-on-the-fly.t
delete mode 120000 tests/flakyfunctional/registration/test_header
create mode 100644 tests/functional/cylc-install/01-symlinks.t
delete mode 100755 tests/functional/registration/00-simple.t
delete mode 100644 tests/unit/test_install.py
diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py
index 56b508b7ba7..fc30d5730ff 100644
--- a/cylc/flow/pathutil.py
+++ b/cylc/flow/pathutil.py
@@ -143,9 +143,19 @@ def make_suite_run_tree(suite):
LOG.debug(f'{dir_}: directory created')
-def make_localhost_symlinks(rund, flow_name, log_type=LOG):
- """Creates symlinks for any configured symlink dirs from glbl_cfg."""
- dirs_to_symlink = get_dirs_to_symlink('localhost', flow_name)
+def make_localhost_symlinks(rund, named_sub_dir):
+ """Creates symlinks for any configured symlink dirs from glbl_cfg.
+ Args:
+ rund: the entire run directory path
+ named_sub_dir: e.g flow_name/run1
+
+ Returns:
+ dict - A dictionary of Symlinks with sources as keys and
+ destinations as values: ``{source: destination}``
+
+ """
+ dirs_to_symlink = get_dirs_to_symlink('localhost', named_sub_dir)
+ symlinks_created = {}
for key, value in dirs_to_symlink.items():
if key == 'run':
dst = rund
@@ -157,9 +167,11 @@ def make_localhost_symlinks(rund, flow_name, log_type=LOG):
f'Unable to create symlink to {src}.'
f' \'{value}\' contains an invalid environment variable.'
' Please check configuration.')
- if log_type:
- log_type.info(f"Creating symlink from {src} to {dst}")
make_symlink(src, dst)
+ # symlink info returned for logging purposes, symlinks created
+ # before logs as this dir may be a symlink.
+ symlinks_created[src] = dst
+ return symlinks_created
def get_dirs_to_symlink(install_target, flow_name):
@@ -234,7 +246,6 @@ def remove_dir(path):
else:
LOG.debug(f'Removing directory: {path}')
rmtree(path)
- raise WorkflowFilesError(f"Error when symlinking '{exc}'")
def get_next_rundir_number(run_path):
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index 7a5e2fe2b38..da6c5643e95 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -42,7 +42,7 @@
from cylc.flow.cycling.loader import get_point
from cylc.flow.data_store_mgr import DataStoreMgr, parse_job_item
from cylc.flow.exceptions import (
- CylcError, SuiteConfigError, PlatformLookupError
+ CylcError, SuiteConfigError, PlatformLookupError, WorkflowFilesError
)
import cylc.flow.flags
from cylc.flow.host_select import select_suite_host
@@ -81,7 +81,6 @@
from cylc.flow.suite_db_mgr import SuiteDatabaseManager
from cylc.flow.suite_events import (
SuiteEventContext, SuiteEventHandler)
-from cylc.flow.exceptions import SuiteServiceFileError
from cylc.flow.suite_status import StopMode, AutoRestartMode
from cylc.flow import suite_files
from cylc.flow.taskdef import TaskDef
@@ -261,6 +260,7 @@ def __init__(self, reg, options, is_restart=False):
)
# directory information
+ self.suite_dir = suite_files.get_suite_source_dir(self.suite)
self.flow_file = suite_files.get_flow_file(self.suite)
self.suite_run_dir = get_workflow_run_dir(self.suite)
self.suite_work_dir = get_suite_run_work_dir(self.suite)
@@ -278,13 +278,25 @@ def __init__(self, reg, options, is_restart=False):
async def install(self):
"""Get the filesystem in the right state to run the flow.
-
+ * Validate flowfiles
* Install authentication files.
* Build the directory tree.
* Copy Python files.
"""
-
+ # Check if flow has been installed
+ if not suite_files.is_installed(self.suite_run_dir):
+ suite_files.register(self.suite, source=self.suite_run_dir)
+ # Install
+ try:
+ suite_files.get_suite_source_dir(self.suite)
+ except WorkflowFilesError:
+ # Source path is assumed to be the run directory
+ suite_files.register(
+ flow_name=self.suite,
+ source=get_workflow_run_dir(
+ self.suite))
+
make_suite_run_tree(self.suite)
# Create ZMQ keys
@@ -298,15 +310,17 @@ async def install(self):
# Copy local python modules from source to run directory
for sub_dir in ["python", os.path.join("lib", "python")]:
# TODO - eventually drop the deprecated "python" sub-dir.
- suite_py = os.path.join(self.suite_run_dir, sub_dir) # change to rundir
- if os.path.isdir(suite_py)):
+ suite_py = os.path.join(self.suite_dir, sub_dir)
+ if (os.path.realpath(self.suite_dir) !=
+ os.path.realpath(self.suite_run_dir) and
+ os.path.isdir(suite_py)):
suite_run_py = os.path.join(self.suite_run_dir, sub_dir)
try:
rmtree(suite_run_py)
except OSError:
pass
copytree(suite_py, suite_run_py)
- sys.path.append(os.path.join(self.suite_run_dir, sub_dir))
+ sys.path.append(os.path.join(self.suite_dir, sub_dir))
async def initialise(self):
"""Initialise the components and sub-systems required to run the flow.
@@ -367,7 +381,7 @@ async def initialise(self):
proc_pool=self.proc_pool,
suite_run_dir=self.suite_run_dir,
suite_share_dir=self.suite_share_dir,
- suite_source_dir=self.suite_run_dir
+ suite_source_dir=self.suite_dir
)
self.task_events_mgr = TaskEventsManager(
@@ -413,8 +427,10 @@ async def configure(self):
# Copy local python modules from source to run directory
for sub_dir in ["python", os.path.join("lib", "python")]:
# TODO - eventually drop the deprecated "python" sub-dir.
- suite_py = os.path.join(self.suite_run_dir, sub_dir)
- if os.path.isdir(suite_py)):
+ suite_py = os.path.join(self.suite_dir, sub_dir)
+ if (os.path.realpath(self.suite_dir) !=
+ os.path.realpath(self.suite_run_dir) and
+ os.path.isdir(suite_py)):
suite_run_py = os.path.join(self.suite_run_dir, sub_dir)
try:
rmtree(suite_run_py)
diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py
index f9c983c0d1e..6e77ad34555 100644
--- a/cylc/flow/scheduler_cli.py
+++ b/cylc/flow/scheduler_cli.py
@@ -21,6 +21,7 @@
import sys
from ansimarkup import parse as cparse
+from pathlib import Path
from cylc.flow import LOG, RSYNC_LOG
from cylc.flow.exceptions import SuiteServiceFileError
@@ -261,6 +262,19 @@ def get_option_parser(is_restart, add_std_opts=False):
get_option_parser(is_restart=True, add_std_opts=True), DEFAULT_OPTS)
+def _auto_install():
+ """Register workflow installed in the cylc-run directory"""
+ try:
+ reg = suite_files.register()
+ except SuiteServiceFileError as exc:
+ sys.exit(exc)
+ # Replace this process with "cylc run REG ..." for 'ps -f'.
+ os.execv(
+ sys.argv[0],
+ [sys.argv[0]] + sys.argv[1:] + [reg]
+ )
+
+
def _open_logs(reg, no_detach):
"""Open Cylc log handlers for a flow run."""
if not no_detach:
@@ -307,7 +321,7 @@ def scheduler_cli(parser, options, args, is_restart=False):
suite_files.detect_old_contact_file(reg)
except SuiteServiceFileError as exc:
sys.exit(exc)
- _check_installation(reg)
+ _check_srvd(reg)
# re-execute on another host if required
_distribute(options.host, is_restart)
@@ -352,12 +366,13 @@ def scheduler_cli(parser, options, args, is_restart=False):
sys.exit(ret)
-def _check_installation(reg):
- """Check the flow is installed."""
- suite_run_dir = get_workflow_run_dir(reg)
- if not os.path.exists(suite_run_dir):
+def _check_srvd(reg):
+ """Check the run dir contains .service dir"""
+ workflow_run_dir = get_workflow_run_dir(reg)
+ if not Path(workflow_run_dir,
+ suite_files.SuiteFiles.Service.DIRNAME).exists:
sys.stderr.write(f'suite service directory not found '
- f'at: {suite_run_dir}\n')
+ f'at: {workflow_run_dir}\n')
sys.exit(1)
@@ -421,7 +436,6 @@ def restart(parser, options, *args):
@cli_function(partial(get_option_parser, is_restart=False))
def run(parser, options, *args):
"""Implement cylc run."""
-
if not args:
_auto_install()
if options.startcp:
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 85e14b8f999..a2c17943a86 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -156,7 +156,8 @@ def get_version(long=False):
),
'jobscript': 'cylc jobscript has been removed',
'submit': 'cylc submit has been removed',
- 'register': 'cylc register had been removed, use cylc install or cylc run'
+ 'register': 'cylc register had been removed, use cylc install or cylc run',
+ 'get-directory': 'cylc get-directory has been removed.'
}
diff --git a/cylc/flow/scripts/graph.py b/cylc/flow/scripts/graph.py
index ce5f20ed109..385b955400d 100755
--- a/cylc/flow/scripts/graph.py
+++ b/cylc/flow/scripts/graph.py
@@ -26,7 +26,7 @@
from cylc.flow.config import SuiteConfig
from cylc.flow.cycling.loader import get_point
-from cylc.flow.exceptions import UserInputError, SuiteServiceFileError
+from cylc.flow.exceptions import UserInputError, WorkflowFilesError
from cylc.flow.option_parsers import CylcOptionParser as COP
from cylc.flow.suite_files import get_flow_file
from cylc.flow.templatevars import load_template_vars
@@ -170,7 +170,7 @@ def get_config(suite, opts, template_vars=None):
"""Return a SuiteConfig object for the provided reg / path."""
try:
flow_file = get_flow_file(suite)
- except SuiteServiceFileError:
+ except WorkflowFilesError:
# could not find suite, assume we have been given a path instead
flow_file = suite
suite = 'test'
diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py
index f67fa3d6b12..9fd01520461 100755
--- a/cylc/flow/scripts/install.py
+++ b/cylc/flow/scripts/install.py
@@ -61,11 +61,10 @@
import os
import pkg_resources
-from pathlib import Path
from cylc.flow.exceptions import PluginError
from cylc.flow.option_parsers import CylcOptionParser as COP
-from cylc.flow.pathutil import get_suite_run_dir
+from cylc.flow.pathutil import get_workflow_run_dir
from cylc.flow.suite_files import install_workflow
from cylc.flow.terminal import cli_function
@@ -125,12 +124,7 @@ def main(parser, opts, flow_name=None, src=None):
parser.error(
"""options --no-run-name and --run-name are mutually exclusive.
Use one or the other""")
- flow_name = install_workflow(
- flow_name=opts.flow_name,
- source=opts.source,
- run_name=opts.run_name,
- no_run_name=opts.no_run_name,
- no_symlinks=opts.no_symlinks)
+
for entry_point in pkg_resources.iter_entry_points(
'cylc.pre_configure'
):
@@ -145,6 +139,13 @@ def main(parser, opts, flow_name=None, src=None):
exc
) from None
+ flow_name = install_workflow(
+ flow_name=opts.flow_name,
+ source=opts.source,
+ run_name=opts.run_name,
+ no_run_name=opts.no_run_name,
+ no_symlinks=opts.no_symlinks)
+
for entry_point in pkg_resources.iter_entry_points(
'cylc.post_install'
):
@@ -152,7 +153,7 @@ def main(parser, opts, flow_name=None, src=None):
entry_point.resolve()(
dir_=os.getcwd(),
opts=opts,
- dest_root=get_suite_run_dir(flow_name)
+ dest_root=get_workflow_run_dir(flow_name)
)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py
index 3128c2b2004..ce14aea1c5d 100755
--- a/cylc/flow/scripts/validate.py
+++ b/cylc/flow/scripts/validate.py
@@ -86,13 +86,11 @@ def main(_, options, reg):
"""cylc validate CLI."""
profiler = Profiler(None, options.profile_mode)
profiler.start()
-
if not cylc.flow.flags.debug:
# for readability omit timestamps from logging unless in debug mode
for handler in LOG.handlers:
if isinstance(handler.formatter, CylcLogFormatter):
handler.formatter.configure(timestamp=False)
-
suite, flow_file = parse_suite_arg(options, reg)
cfg = SuiteConfig(
suite,
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index 5799f5140ac..a55a1bb2dbd 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -362,10 +362,35 @@ def get_contact_file(reg):
get_suite_srv_dir(reg), SuiteFiles.Service.CONTACT)
-def get_flow_file(reg):
+def get_flow_file(reg, suite_owner=None):
"""Return the path of a suite's flow.cylc file."""
return os.path.join(
- get_workflow_run_dir(reg), SuiteFiles.FLOW_FILE)
+ get_suite_source_dir(reg, suite_owner), SuiteFiles.FLOW_FILE)
+
+
+def get_suite_source_dir(reg, suite_owner=None):
+ """Return the source directory path of a suite.
+
+ Will register un-registered suites located in the cylc run dir.
+ """
+ srv_d = get_suite_srv_dir(reg, suite_owner)
+ fname = os.path.join(get_workflow_run_dir(reg), SuiteFiles.SOURCE)
+ try:
+ source = os.readlink(fname)
+ except OSError:
+ suite_d = os.path.dirname(srv_d)
+ if os.path.exists(suite_d) and not is_remote_user(suite_owner):
+ register(flow_name=reg, source=suite_d)
+ return suite_d
+ raise WorkflowFilesError(f"Suite not found: {reg}")
+ else:
+ if not os.path.isabs(source):
+ source = os.path.normpath(os.path.join(srv_d, source))
+ flow_file_path = os.path.join(source, SuiteFiles.FLOW_FILE)
+ if not os.path.exists(flow_file_path):
+ # suite exists but is probably using deprecated suite.rc
+ register(flow_name=reg, source=source)
+ return source
def get_suite_srv_dir(reg, suite_owner=None):
@@ -431,72 +456,73 @@ def parse_suite_arg(options, arg):
"""
if arg == '.':
arg = os.getcwd()
- try:
- path = get_flow_file(arg)
+ if os.path.isfile(arg):
+ name = os.path.dirname(arg)
+ path = os.path.abspath(arg)
+ else:
name = arg
- except SuiteServiceFileError:
- arg = os.path.abspath(arg)
- if os.path.isdir(arg):
- path = os.path.join(arg, SuiteFiles.FLOW_FILE)
- name = os.path.basename(arg)
- if not os.path.exists(path):
- # Probably using deprecated suite.rc
- path = os.path.join(arg, SuiteFiles.SUITE_RC)
+ try:
+ path = get_flow_file(arg)
+ except WorkflowFilesError:
+ arg = os.path.abspath(arg)
+ if os.path.isdir(arg):
+ path = os.path.join(arg, SuiteFiles.FLOW_FILE)
+ name = os.path.basename(arg)
if not os.path.exists(path):
- raise SuiteServiceFileError(
- f'no {SuiteFiles.FLOW_FILE} or {SuiteFiles.SUITE_RC}'
- f' in {arg}')
- else:
- LOG.warning(
- f'The filename "{SuiteFiles.SUITE_RC}" is deprecated '
- f'in favour of "{SuiteFiles.FLOW_FILE}".')
- else:
- path = arg
- name = os.path.basename(os.path.dirname(arg))
+ # Probably using deprecated suite.rc
+ path = os.path.join(arg, SuiteFiles.SUITE_RC)
+ if not os.path.exists(path):
+ raise SuiteServiceFileError(
+ f'no {SuiteFiles.FLOW_FILE} or '
+ f'{SuiteFiles.SUITE_RC} in {arg}')
+ else:
+ LOG.warning(
+ f'The filename "{SuiteFiles.SUITE_RC}" is '
+ f'deprecated in favour of '
+ f'"{SuiteFiles.FLOW_FILE}".')
+ else:
+ path = arg
+ name = os.path.basename(os.path.dirname(arg))
return name, path
-def install(flow_name=None, source=None, redirect=False, rundir=None):
- """Install a suite, or renew its installation.
+def register(flow_name=None, source=None):
+ """Set up workflow.
+ This completes some of the set up completed by cylc install.
+ Called only if running workflow that has not been installed.
- Create suite service directory and symlink to suite source location.
+ Validates workflow name.
+ Validates run directory structure.
+ Symlinks flow.cylc -> suite.rc.
+ Creates the .service directory.
Args:
flow_name (str): workflow name, default basename($PWD).
source (str): directory location of flow.cylc file, default $PWD.
- redirect (bool): allow reuse of existing name and run directory.
Return:
str: The installed suite name (which may be computed here).
Raise:
- SuiteServiceFileError:
+ WorkflowFilesError:
- No flow.cylc file found in source location.
- Illegal name (can look like a relative path, but not absolute).
- Another suite already has this name (unless --redirect).
- - Trying to install a workflow that is nested inside of another.
+ - Nested workflow run directories.
"""
if flow_name is None:
flow_name = (Path.cwd().stem)
- make_localhost_symlinks(flow_name)
-
is_valid, message = SuiteNameValidator.validate(flow_name)
if not is_valid:
- raise SuiteServiceFileError(f'Invalid workflow name - {message}')
-
+ raise WorkflowFilesError(f'Invalid workflow name - {message}')
if Path.is_absolute(Path(flow_name)):
- raise SuiteServiceFileError(
+ raise WorkflowFilesError(
f'Workflow name cannot be an absolute path: {flow_name}')
- check_nested_run_dirs(flow_name)
-
- make_localhost_symlinks(reg)
-
if source is not None:
if os.path.basename(source) == SuiteFiles.FLOW_FILE:
source = os.path.dirname(source)
else:
source = os.getcwd()
-
# flow.cylc must exist so we can detect accidentally reversed args.
source = os.path.abspath(source)
flow_file_path = os.path.join(source, SuiteFiles.FLOW_FILE)
@@ -509,49 +535,30 @@ def install(flow_name=None, source=None, redirect=False, rundir=None):
f'The filename "{SuiteFiles.SUITE_RC}" is deprecated in favor '
f'of "{SuiteFiles.FLOW_FILE}". Symlink created.')
else:
- raise SuiteServiceFileError(
+ raise WorkflowFilesError(
f'no flow.cylc or suite.rc in {source}')
-
+ symlinks_created = make_localhost_symlinks(
+ get_workflow_run_dir(flow_name), flow_name)
+ if bool(symlinks_created):
+ for src, dst in symlinks_created.items():
+ INSTALL_LOG.info(f"Symlink created from {src} to {dst}")
# Create service dir if necessary.
- srv_d = get_suite_srv_dir(reg)
+ srv_d = get_suite_srv_dir(flow_name)
os.makedirs(srv_d, exist_ok=True)
+ return flow_name
- # See if suite already has a source or not
- try:
- orig_source = os.readlink(
- os.path.join(srv_d, SuiteFiles.Service.SOURCE))
- except OSError:
- orig_source = None
- else:
- if not os.path.isabs(orig_source):
- orig_source = os.path.normpath(
- os.path.join(srv_d, orig_source))
- if orig_source is not None and source != orig_source:
- if not redirect:
- raise SuiteServiceFileError(
- f"the name '{flow_name}' already points to {orig_source}.\nUse "
- "--redirect to re-use an existing name and run directory.")
- LOG.warning(
- f"the name '{flow_name}' points to {orig_source}.\nIt will now be "
- f"redirected to {source}.\nFiles in the existing {flow_name} run "
- "directory will be overwritten.\n")
- # Remove symlink to the original suite.
- os.unlink(os.path.join(srv_d, SuiteFiles.Service.SOURCE))
-
- # Create symlink to the suite, if it doesn't already exist.
- if orig_source is None or source != orig_source:
- target = os.path.join(srv_d, SuiteFiles.Service.SOURCE)
- if (os.path.abspath(source) ==
- os.path.abspath(os.path.dirname(srv_d))):
- # If source happens to be the run directory,
- # create .service/source -> ..
- source_str = ".."
- else:
- source_str = source
- os.symlink(source_str, target)
- print(f'INSTALLED {flow_name} -> {source}')
- return flow_name
+def is_installed(path):
+ """Check to see if the path sent contains installed flow.
+
+ Checks for valid _cylc-install directory in current folder and checks
+ source link exists.
+ """
+ cylc_install_folder = Path(path, SuiteFiles.Install.DIRNAME)
+ source = Path(cylc_install_folder, SuiteFiles.Install.SOURCE)
+ if cylc_install_folder.exists and source.is_symlink():
+ return True
+ return False
def _clean_check(reg, run_dir):
@@ -765,17 +772,6 @@ def _remove_empty_reg_parents(reg, path):
break
-def start_install_log(reg, no_detach):
- if not no_detach:
- while LOG.handlers:
- LOG.handlers[0].close()
- LOG.removeHandler(LOG.handlers[0])
-
- install_log_path = get_install_log_name(reg)
- handler = TimestampRotatingFileHandler(install_log_path, no_detach)
- INSTALL_LOG.addHandler(handler)
-
-
def remove_keys_on_server(keys):
"""Removes server-held authentication keys"""
# WARNING, DESTRUCTIVE. Removes old keys if they already exist.
@@ -903,7 +899,9 @@ def _check_child_dirs(path, depth_count=1):
for result in os.scandir(path):
if result.is_dir() and not result.is_symlink():
if is_valid_run_dir(result.path):
- raise WorkflowFilesError(exc_msg % (flow_name, result.path))
+ raise WorkflowFilesError(
+ exc_msg %
+ (flow_name, result.path))
if depth_count < MAX_SCAN_DEPTH:
_check_child_dirs(result.path, depth_count + 1)
@@ -945,22 +943,18 @@ def get_cylc_run_abs_path(path):
return get_workflow_run_dir(path)
-def _open_install_log(reg, rund, is_reload=False):
+def _open_install_log(rund):
"""Open Cylc log handlers for an install."""
time_str = get_current_time_string(
override_use_utc=True, use_basic_format=True,
display_sub_seconds=False
)
- if is_reload:
- load_type = "reload"
- else:
- load_type = "install"
rund = Path(rund).expanduser()
log_path = Path(
rund,
'log',
'install',
- f"{time_str}-{load_type}.log")
+ f"{time_str}-install.log")
log_parent_dir = log_path.parent
log_parent_dir.mkdir(exist_ok=True, parents=True)
handler = logging.FileHandler(log_path)
@@ -992,7 +986,7 @@ def get_rsync_rund_cmd(src, dst, restart=False):
rsync_cmd.append("-av")
if restart:
rsync_cmd.append('--delete')
- ignore_dirs = ['.git', '.svn','.cylcignore']
+ ignore_dirs = ['.git', '.svn', '.cylcignore']
for exclude in ignore_dirs:
if Path(src).joinpath(exclude).exists():
rsync_cmd.append(f"--exclude={exclude}")
@@ -1023,7 +1017,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
str: The installed suite name (which may be computed here).
Raise:
- SuiteServiceFileError:
+ WorkflowFilesError:
No flow.cylc file found in source location.
Illegal name (can look like a relative path, but not absolute).
Another suite already has this name (unless --redirect).
@@ -1038,12 +1032,13 @@ def install_workflow(flow_name=None, source=None, run_name=None,
flow_name = (Path.cwd().stem)
validate_flow_name(flow_name)
if run_name == '_cylc-install':
- raise SuiteServiceFileError(
+ raise WorkflowFilesError(
'Run name cannot be "_cylc-install".'
' Please choose another run name.')
validate_source_dir(source)
run_path_base = Path(get_workflow_run_dir(flow_name)).expanduser()
relink = False
+ run_num = int()
if no_run_name:
rundir = run_path_base
elif run_name:
@@ -1053,17 +1048,25 @@ def install_workflow(flow_name=None, source=None, run_name=None,
run_num = get_next_rundir_number(run_path_base)
rundir = Path(run_path_base, f'run{run_num}')
if run_num == 1 and rundir.exists():
- SuiteServiceFileError(
+ WorkflowFilesError(
f"This path: {rundir} exists. Try using --run-name")
unlink_runN(run_n)
relink = True
check_nested_run_dirs(rundir, flow_name)
+ if not no_symlinks:
+ sub_dir = flow_name
+ if run_num:
+ sub_dir += '/' + f'run{run_num}'
+ symlinks_created = make_localhost_symlinks(rundir, sub_dir)
+ _open_install_log(rundir)
+ if not no_symlinks and bool(symlinks_created):
+ for src, dst in symlinks_created.items():
+ INSTALL_LOG.info(f"Symlink created from {src} to {dst}")
try:
rundir.mkdir(exist_ok=True)
except OSError as e:
if e.strerror == "File exists":
- raise SuiteServiceFileError(f"Run directory already exists : {e}")
- _open_install_log(flow_name, rundir)
+ raise WorkflowFilesError(f"Run directory already exists : {e}")
# create source symlink to be used as the basis of ensuring runs are
# from a constistent source dir.
base_source_link = run_path_base.joinpath(SuiteFiles.Install.SOURCE)
@@ -1071,8 +1074,6 @@ def install_workflow(flow_name=None, source=None, run_name=None,
run_path_base.joinpath(SuiteFiles.Install.SOURCE).symlink_to(source)
if relink:
link_runN(rundir)
- if not no_symlinks:
- make_localhost_symlinks(rundir, flow_name, log_type=INSTALL_LOG)
create_workflow_srv_dir(rundir)
# flow.cylc must exist so we can detect accidentally reversed args.
flow_file_path = source.joinpath(SuiteFiles.FLOW_FILE)
@@ -1085,7 +1086,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
f'The filename "{SuiteFiles.SUITE_RC}" is deprecated in favour'
f' of "{SuiteFiles.FLOW_FILE}". Symlink created.')
else:
- raise SuiteServiceFileError(
+ raise WorkflowFilesError(
f'no {SuiteFiles.FLOW_FILE} or {SuiteFiles.SUITE_RC}'
f' in {source}')
rsync_cmd = get_rsync_rund_cmd(source, rundir)
@@ -1105,7 +1106,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
INSTALL_LOG.info(f"Creating symlink from {source_link}")
source_link.symlink_to(source)
else:
- raise SuiteServiceFileError(
+ raise WorkflowFilesError(
"Source directory between runs are not consistent")
INSTALL_LOG.info(f'INSTALLED {flow_name} from {source} -> {rundir}')
print(f'INSTALLED {flow_name} from {source} -> {rundir}')
@@ -1123,9 +1124,9 @@ def create_workflow_srv_dir(rundir=None, source=None):
def validate_flow_name(flow_name):
is_valid, message = SuiteNameValidator.validate(flow_name)
if not is_valid:
- raise SuiteServiceFileError(f'Invalid workflow name - {message}')
+ raise WorkflowFilesError(f'Invalid workflow name - {message}')
if Path.is_absolute(Path(flow_name)):
- raise SuiteServiceFileError(
+ raise WorkflowFilesError(
f'Workflow name cannot be an absolute path: {flow_name}')
@@ -1135,7 +1136,7 @@ def validate_source_dir(source):
Args:
source (path): Path to source directory
Raises:
- SuiteServiceFileError:
+ WorkflowFilesError:
If log, share, work or _cylc-install directories exist in the
source directory.
Cylc installing from within the cylc-run dir
@@ -1144,14 +1145,14 @@ def validate_source_dir(source):
for dir_ in FORBIDDEN_SOURCE_DIR:
path_to_check = Path(source, dir_)
if path_to_check.exists():
- raise SuiteServiceFileError(
+ raise WorkflowFilesError(
f'Installation failed. - {dir_} exists in source directory.')
cylc_run_dir = Path(
get_platform()['run directory'].replace('$HOME', '~')
).expanduser()
if os.path.abspath(os.path.realpath(cylc_run_dir)
) in os.path.abspath(os.path.realpath(source)):
- raise SuiteServiceFileError(
+ raise WorkflowFilesError(
f'Installation failed. Source directory should not be in'
f' {cylc_run_dir}')
diff --git a/cylc/flow/task_remote_mgr.py b/cylc/flow/task_remote_mgr.py
index c06fbeb09e8..4c847bb5210 100644
--- a/cylc/flow/task_remote_mgr.py
+++ b/cylc/flow/task_remote_mgr.py
@@ -293,35 +293,6 @@ def _remote_init_callback(
pass
install_target = platform['install target']
if proc_ctx.ret_code == 0:
- if REMOTE_INIT_DONE in proc_ctx.out:
- src_path = get_workflow_run_dir(self.suite)
- dst_path = get_remote_suite_run_dir(platform, self.suite)
- try:
- process = procopen(construct_rsync_over_ssh_cmd(
- src_path,
- dst_path,
- platform,
- self.rsync_includes),
- stdoutpipe=True,
- stderrpipe=True,
- universal_newlines=True)
-
- out, err = process.communicate(timeout=600)
- install_target = platform['install target']
- if out:
- RSYNC_LOG.info(
- 'File installation information for '
- f'{install_target}:\n {out}')
- LOG.info("File installation complete.")
- if err:
- LOG.error(
- 'File installation error on '
- f'{install_target}:\n {err}')
- except Exception as ex:
- LOG.error(f"Problem during rsync: {ex}")
- self.remote_init_map[self.install_target] = (
- REMOTE_INIT_FAILED)
- return
if "KEYSTART" in proc_ctx.out:
regex_result = re.search(
'KEYSTART((.|\n|\r)*)KEYEND', proc_ctx.out)
@@ -371,7 +342,7 @@ def file_install(self, platform):
"""
install_target = platform['install target']
self.remote_init_map[install_target] = REMOTE_FILE_INSTALL_IN_PROGRESS
- src_path = get_suite_run_dir(self.suite)
+ src_path = get_workflow_run_dir(self.suite)
dst_path = get_remote_suite_run_dir(platform, self.suite)
install_target = platform['install target']
ctx = SubProcContext(
diff --git a/tests/flakyfunctional/registration/02-on-the-fly.t b/tests/flakyfunctional/registration/02-on-the-fly.t
deleted file mode 100755
index 90e3804a69b..00000000000
--- a/tests/flakyfunctional/registration/02-on-the-fly.t
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-# Test on-the-fly suite registration by "cylc run"
-#------------------------------------------------------------------------------
-# Test `cylc run` with no registration
-
-. "$(dirname "$0")/test_header"
-set_test_number 9
-
-TEST_NAME="${TEST_NAME_BASE}-pwd"
-
-TESTD="cylctb-cheese-${CYLC_TEST_TIME_INIT}"
-mkdir "${TESTD}"
-cat >> "${TESTD}/flow.cylc" <<'__FLOW_CONFIG__'
-[meta]
- title = the quick brown fox
-[scheduling]
- [[graph]]
- R1 = foo
-[runtime]
- [[foo]]
- script = true
-__FLOW_CONFIG__
-
-cd "${TESTD}" || exit 1
-run_ok "${TEST_NAME}-run" cylc run --hold
-contains_ok "${TEST_NAME}-run.stdout" <<__ERR__
-REGISTERED ${TESTD} -> ${PWD}
-__ERR__
-
-run_ok "${TEST_NAME}-stop" cylc stop --max-polls=10 --interval=2 "${TESTD}"
-
-purge "${TESTD}"
-#------------------------------------------------------------------------------
-# Test `cylc run` REG for an un-registered suite
-TESTD="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_NAME_BASE}"
-
-mkdir -p "${RUN_DIR}/${TESTD}"
-cat >> "${RUN_DIR}/${TESTD}/flow.cylc" <<'__FLOW_CONFIG__'
-[meta]
- title = the quick brown fox
-[scheduling]
- [[graph]]
- R1 = foo
-[runtime]
- [[foo]]
- script = true
-__FLOW_CONFIG__
-
-TEST_NAME="${TEST_NAME_BASE}-cylc-run-dir"
-run_ok "${TEST_NAME}-run" cylc run --hold "${TESTD}"
-contains_ok "${TEST_NAME}-run.stdout" <<__ERR__
-REGISTERED ${TESTD} -> ${RUN_DIR}/${TESTD}
-__ERR__
-
-run_ok "${TEST_NAME}-stop" cylc stop --max-polls=10 --interval=2 "${TESTD}"
-
-purge "${TESTD}"
-#------------------------------------------------------------------------------
-# Test `cylc run` REG for an un-registered suite
-mkdir -p "${RUN_DIR}/${TESTD}"
-cat >> "${RUN_DIR}/${TESTD}/flow.cylc" <<'__FLOW_CONFIG__'
-[meta]
- title = the quick brown fox
-[sched]
- [[graph]]
- R1 = foo
-[runtime]
- [[foo]]
- script = true
-__FLOW_CONFIG__
-
-TEST_NAME="${TEST_NAME_BASE}-cylc-run-dir-2"
-run_fail "${TEST_NAME}-validate" cylc validate "${TESTD}"
-contains_ok "${TEST_NAME}-validate.stdout" <<__OUT__
-REGISTERED ${TESTD} -> ${RUN_DIR}/${TESTD}
-__OUT__
-contains_ok "${TEST_NAME}-validate.stderr" <<__ERR__
-IllegalItemError: sched
-__ERR__
-
-purge "${TESTD}"
-
-exit
diff --git a/tests/flakyfunctional/registration/test_header b/tests/flakyfunctional/registration/test_header
deleted file mode 120000
index 0126592858e..00000000000
--- a/tests/flakyfunctional/registration/test_header
+++ /dev/null
@@ -1 +0,0 @@
-../../functional/lib/bash/test_header
\ No newline at end of file
diff --git a/tests/flakyfunctional/xtriggers/01-suite_state.t b/tests/flakyfunctional/xtriggers/01-suite_state.t
index 2d22fc3d697..36f91490047 100644
--- a/tests/flakyfunctional/xtriggers/01-suite_state.t
+++ b/tests/flakyfunctional/xtriggers/01-suite_state.t
@@ -26,7 +26,7 @@ install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
# Register and validate the upstream suite.
SUITE_NAME_UPSTREAM="${SUITE_NAME}-upstream"
-cylc reg "${SUITE_NAME_UPSTREAM}" "${TEST_DIR}/${SUITE_NAME}/upstream"
+cylc install --flow-name="${SUITE_NAME_UPSTREAM}" -C "${TEST_DIR}/${SUITE_NAME}/upstream" --no-run-name
run_ok "${TEST_NAME_BASE}-val-up" cylc val --debug "${SUITE_NAME_UPSTREAM}"
# Validate the downstream test suite.
diff --git a/tests/functional/authentication/00-shared-fs.t b/tests/functional/authentication/00-shared-fs.t
index d451d0bf5b3..1f0f3d4b0b3 100755
--- a/tests/functional/authentication/00-shared-fs.t
+++ b/tests/functional/authentication/00-shared-fs.t
@@ -28,7 +28,7 @@ SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BA
SUITE_RUN_DIR="$RUN_DIR/${SUITE_NAME}"
mkdir -p "$(dirname "${SUITE_RUN_DIR}")"
cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}" "${SUITE_RUN_DIR}"
-cylc install --flow-name=${SUITE_NAME} --no-run-name >&2
+cylc install --flow-name="${SUITE_NAME}" --no-run-name 2>'/dev/null'
run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
diff --git a/tests/functional/authentication/01-remote-suite-same-name.t b/tests/functional/authentication/01-remote-suite-same-name.t
index 3727d294121..502b7723282 100755
--- a/tests/functional/authentication/01-remote-suite-same-name.t
+++ b/tests/functional/authentication/01-remote-suite-same-name.t
@@ -27,18 +27,20 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
SSH_OPTS='-oBatchMode=yes -oConnectTimeout=5'
# shellcheck disable=SC2029,SC2086
-ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" mkdir -p "cylc-run/${SUITE_NAME}"
+ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" mkdir -p "cylctb-cylc-source/${SUITE_NAME}"
# shellcheck disable=SC2086
scp ${SSH_OPTS} -pqr "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/"* \
- "${CYLC_TEST_HOST}:cylc-run/${SUITE_NAME}"
+ "${CYLC_TEST_HOST}:cylctb-cylc-source/${SUITE_NAME}"
# shellcheck disable=SC2086
-run_ok "${TEST_NAME_BASE}-register" \
+run_ok "${TEST_NAME_BASE}-install" \
ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" \
CYLC_VERSION="$(cylc version)" cylc install --flow-name="${SUITE_NAME}" \
- --no-run-name --directory="cylc-run/${SUITE_NAME}"
+ --no-run-name --directory="cylctb-cylc-source/${SUITE_NAME}"
suite_run_ok "${TEST_NAME_BASE}" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
+ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" \
+ rm -rf cylctb-cylc-source
purge
exit
diff --git a/tests/functional/authentication/02-suite2-stop-suite1.t b/tests/functional/authentication/02-suite2-stop-suite1.t
index dd8a1778120..f8492fb9397 100755
--- a/tests/functional/authentication/02-suite2-stop-suite1.t
+++ b/tests/functional/authentication/02-suite2-stop-suite1.t
@@ -25,11 +25,15 @@ NAME1="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}-1
NAME2="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}-2"
SUITE1_RUND="${RUND}/${NAME1}"
mkdir -p "${SUITE1_RUND}"
-cp -p "${TEST_SOURCE_DIR}/basic/flow.cylc" "${SUITE1_RUND}"
-cylc install --flow-name="${NAME1}" --no-run-name --directory="${SUITE1_RUND}"
+RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)
+RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}"
+mkdir -p "${RND_SUITE_SOURCE}"
+cp -p "${TEST_SOURCE_DIR}/basic/flow.cylc" "${RND_SUITE_SOURCE}"
+cylc install --flow-name="${NAME1}" --no-run-name --directory="${RND_SUITE_SOURCE}"
SUITE2_RUND="${RUND}/${NAME2}"
mkdir -p "${SUITE2_RUND}"
-cat >"${SUITE2_RUND}/flow.cylc" <<__FLOW_CONFIG__
+rm "${RND_SUITE_SOURCE}/flow.cylc"
+cat >"${RND_SUITE_SOURCE}/flow.cylc" <<__FLOW_CONFIG__
[scheduler]
[[events]]
[scheduling]
@@ -39,11 +43,12 @@ cat >"${SUITE2_RUND}/flow.cylc" <<__FLOW_CONFIG__
[[t1]]
script=cylc shutdown "${NAME1}"
__FLOW_CONFIG__
-cylc install --flow-name="${NAME2}" --directory="${SUITE2_RUND}" --no-run-name
+cylc install --flow-name="${NAME2}" --directory="${RND_SUITE_SOURCE}" --no-run-name
cylc run --no-detach "${NAME1}" 1>'1.out' 2>&1 &
SUITE_RUN_DIR="${SUITE1_RUND}" poll_suite_running
run_ok "${TEST_NAME_BASE}" cylc run --no-detach --abort-if-any-task-fails "${NAME2}"
cylc shutdown "${NAME1}" --max-polls=20 --interval=1 1>'/dev/null' 2>&1 || true
-# purge "${NAME1}"
-# purge "${NAME2}"
+purge "${NAME1}"
+purge "${NAME2}"
+rm -rf "${RND_SUITE_SOURCE}"
exit
diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t
index a5c755e85f2..dd98333ff6e 100644
--- a/tests/functional/cylc-clean/00-basic.t
+++ b/tests/functional/cylc-clean/00-basic.t
@@ -54,6 +54,8 @@ readlink "$SUITE_RUN_DIR" > "${TEST_NAME}.stdout"
cmp_ok "${TEST_NAME}.stdout" <<< "${TEST_DIR}/${SYM_NAME}-run/cylc-run/${SUITE_NAME}"
+
+INSTALL_LOG_FILE=$(ls "${TEST_DIR}/${SYM_NAME}-log/cylc-run/${SUITE_NAME}/log/install")
TEST_NAME="test-dir-tree-pre-clean"
tree --noreport --charset=ascii "${TEST_DIR}/${SYM_NAME}-"* > "${TEST_NAME}.stdout"
# Note: backticks need to be escaped in the heredoc
@@ -73,6 +75,8 @@ ${TEST_DIR}/${SYM_NAME}-log
| \`-- cylc-clean
| \`-- ${TEST_NAME_BASE}
| \`-- log
+ | \`-- install
+ | \`-- ${INSTALL_LOG_FILE}
\`-- leave-me-alone
${TEST_DIR}/${SYM_NAME}-run
\`-- cylc-run
@@ -80,8 +84,12 @@ ${TEST_DIR}/${SYM_NAME}-run
\`-- ${FUNCTIONAL_DIR}
\`-- cylc-clean
\`-- ${TEST_NAME_BASE}
+ |-- _cylc-install
+ | \`-- source -> ${TEST_DIR}/${SUITE_NAME}
+ |-- flow.cylc
|-- log -> ${TEST_DIR}/${SYM_NAME}-log/cylc-run/${SUITE_NAME}/log
|-- share -> ${TEST_DIR}/${SYM_NAME}-share/cylc-run/${SUITE_NAME}/share
+ |-- source -> ${TEST_DIR}/${SUITE_NAME}
\`-- work -> ${TEST_DIR}/${SYM_NAME}-work/cylc-run/${SUITE_NAME}/work
${TEST_DIR}/${SYM_NAME}-share
\`-- cylc-run
diff --git a/tests/functional/cylc-install/00-simple.t b/tests/functional/cylc-install/00-simple.t
index 65f42bf4293..61c7ad67259 100755
--- a/tests/functional/cylc-install/00-simple.t
+++ b/tests/functional/cylc-install/00-simple.t
@@ -17,6 +17,8 @@
#------------------------------------------------------------------------------
# Test workflow installation
+. "$(dirname "$0")/test_header"
+set_test_number 18
export RND_SUITE_NAME
export RND_SUITE_SOURCE
@@ -35,16 +37,10 @@ function make_rnd_suite() {
function purge_rnd_suite() {
# Remove the suite source created by make_rnd_suite().
# And remove its run-directory too.
- RND_SUITE_SOURCE=${1:-$RND_SUITE_SOURCE}
- RND_SUITE_RUNDIR=${2:-$RND_SUITE_RUNDIR}
rm -rf "${RND_SUITE_SOURCE}"
rm -rf "${RND_SUITE_RUNDIR}"
}
-. "$(dirname "$0")/test_header"
-set_test_number 18
-
-
# Test default name: "cylc install" (suite in $PWD, no args)
TEST_NAME="${TEST_NAME_BASE}-basic"
make_rnd_suite
@@ -68,8 +64,6 @@ __OUT__
popd || exit 1
purge_rnd_suite
-
-
# Test default path: "cylc install REG" --no-run-name (flow in $PWD)
TEST_NAME="${TEST_NAME_BASE}-pwd-no-run-name"
make_rnd_suite
@@ -80,6 +74,7 @@ INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}
__OUT__
popd || exit 1
purge_rnd_suite
+
# Test "cylc install REG" flow-name given (flow in $PWD)
TEST_NAME="${TEST_NAME_BASE}-flow-name"
make_rnd_suite
@@ -89,7 +84,7 @@ contains_ok "${TEST_NAME}.stdout" <<__OUT__
INSTALLED ${RND_SUITE_NAME}-olaf from ${RND_SUITE_SOURCE} -> ${RUN_DIR}/${RND_SUITE_NAME}-olaf/run1
__OUT__
popd || exit 1
-rm -rf ${RUN_DIR}/${RND_SUITE_NAME}-olaf
+rm -rf "${RUN_DIR}/${RND_SUITE_NAME}-olaf"
purge_rnd_suite
# Test "cylc install REG" flow-name given (flow in $PWD)
@@ -101,7 +96,7 @@ contains_ok "${TEST_NAME}.stdout" <<__OUT__
INSTALLED ${RND_SUITE_NAME}-olaf from ${RND_SUITE_SOURCE} -> ${RUN_DIR}/${RND_SUITE_NAME}-olaf
__OUT__
popd || exit 1
-rm -rf ${RUN_DIR}/${RND_SUITE_NAME}-olaf
+rm -rf "${RUN_DIR}/${RND_SUITE_NAME}-olaf"
purge_rnd_suite
# Test "cylc install" --directory given (flow in --directory)
@@ -126,10 +121,8 @@ contains_ok "${TEST_NAME}.stdout" <<__OUT__
INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run2
__OUT__
popd || exit 1
-
purge_rnd_suite
-
# Test -C option
TEST_NAME="${TEST_NAME_BASE}-option-C"
make_rnd_suite
@@ -139,14 +132,4 @@ INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
__OUT__
purge_rnd_suite
-# things that still need testing:
-
-# symlink/--no-symlink-dirs option
-# workflow name not abs path:
-# SuiteServiceFileError: Workflow name cannot be an absolute path:
-# test by sending run_ok "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_RUNDIR}-olaf"
-# source dir contains forbidden
-
-
-
exit
diff --git a/tests/functional/cylc-install/01-symlinks.t b/tests/functional/cylc-install/01-symlinks.t
new file mode 100644
index 00000000000..27838de6dfd
--- /dev/null
+++ b/tests/functional/cylc-install/01-symlinks.t
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+#------------------------------------------------------------------------------
+# Test workflow installation symlinking localhost
+
+. "$(dirname "$0")/test_header"
+
+if [[ -z ${TMPDIR:-} || -z ${USER:-} || $TMPDIR/$USER == "$HOME" ]]; then
+ skip_all '"TMPDIR" or "USER" not defined or "TMPDIR"/"USER" is "HOME"'
+fi
+
+set_test_number 14
+
+create_test_global_config "" "
+[symlink dirs]
+ [[localhost]]
+ run = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_run_dir
+ share = \$TMPDIR/\$USER/test_cylc_symlink/
+ log = \$TMPDIR/\$USER/test_cylc_symlink/
+ share/cycle = \$TMPDIR/\$USER/test_cylc_symlink/cylctb_tmp_share_dir
+ work = \$TMPDIR/\$USER/test_cylc_symlink/
+"
+
+export RND_SUITE_NAME
+export RND_SUITE_SOURCE
+export RND_SUITE_RUNDIR
+
+function make_rnd_suite() {
+ # Create a randomly-named suite source directory.
+ # Define its run directory.
+ RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)
+ RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}"
+ mkdir -p "${RND_SUITE_SOURCE}"
+ touch "${RND_SUITE_SOURCE}/flow.cylc"
+ RND_SUITE_RUNDIR="${RUN_DIR}/${RND_SUITE_NAME}"
+}
+
+function purge_rnd_suite() {
+ # Remove the suite source created by make_rnd_suite().
+ # And remove its run-directory too.
+ rm -rf "${RND_SUITE_SOURCE}"
+ rm -rf "${RND_SUITE_RUNDIR}"
+}
+
+# Test "cylc install" ensure symlinks are created
+TEST_NAME="${TEST_NAME_BASE}-symlinks-created"
+make_rnd_suite
+run_ok "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}" --directory="${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
+__OUT__
+
+TEST_SYM="${TEST_NAME_BASE}-run-symlink-exists-ok"
+
+if [[ $(readlink "$HOME/cylc-run/${RND_SUITE_NAME}/run1") == \
+ "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_SUITE_NAME}/run1" ]]; then
+ ok "$TEST_SYM"
+else
+ fail "$TEST_SYM"
+fi
+
+
+
+TEST_SYM="${TEST_NAME_BASE}-share/cycle-symlink-exists-ok"
+if [[ $(readlink "$HOME/cylc-run/${RND_SUITE_NAME}/run1/share/cycle") == \
+"$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_SUITE_NAME}/run1/share/cycle" ]]; then
+ ok "$TEST_SYM"
+else
+ fail "$TEST_SYM"
+fi
+
+for DIR in 'work' 'share' 'log'; do
+ TEST_SYM="${TEST_NAME_BASE}-${DIR}-symlink-exists-ok"
+ if [[ $(readlink "$HOME/cylc-run/${RND_SUITE_NAME}/run1/${DIR}") == \
+ "$TMPDIR/${USER}/test_cylc_symlink/cylc-run/${RND_SUITE_NAME}/run1/${DIR}" ]]; then
+ ok "$TEST_SYM"
+ else
+ fail "$TEST_SYM"
+ fi
+done
+rm -rf "${TMPDIR}/${USER}/test_cylc_symlink/"
+purge_rnd_suite
+
+
+
+# Test "cylc install" --no-symlink-dirs
+TEST_NAME="${TEST_NAME_BASE}-no-symlinks-created"
+make_rnd_suite
+run_ok "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}" --no-symlink-dirs --directory="${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
+__OUT__
+
+
+TEST_SYM="${TEST_NAME_BASE}-run-symlink-exists-ok"
+
+if [[ $(readlink "$HOME/cylc-run/${RND_SUITE_NAME}/run1") == \
+ "$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_run_dir/cylc-run/${RND_SUITE_NAME}/run1" ]]; then
+ fail "$TEST_SYM"
+else
+ ok "$TEST_SYM"
+fi
+
+
+
+TEST_SYM="${TEST_NAME_BASE}-share/cycle-symlink-not-exists-ok"
+if [[ $(readlink "$HOME/cylc-run/${RND_SUITE_NAME}/run1/share/cycle") == \
+"$TMPDIR/${USER}/test_cylc_symlink/cylctb_tmp_share_dir/cylc-run/${RND_SUITE_NAME}/share/cycle" ]]; then
+ fail "$TEST_SYM"
+else
+ ok "$TEST_SYM"
+fi
+
+for DIR in 'work' 'share' 'log'; do
+ TEST_SYM="${TEST_NAME_BASE}-${DIR}-symlink-not-exists-ok"
+ if [[ $(readlink "$HOME/cylc-run/${RND_SUITE_NAME}/run1/${DIR}") == \
+ "$TMPDIR/${USER}/test_cylc_symlink/cylc-run/${RND_SUITE_NAME}/${DIR}" ]]; then
+ fail "$TEST_SYM"
+ else
+ ok "$TEST_SYM"
+ fi
+done
+rm -rf "${TMPDIR}/${USER}/test_cylc_symlink/"
+purge_rnd_suite
diff --git a/tests/functional/cylc-install/02-failures.t b/tests/functional/cylc-install/02-failures.t
index 21cfc183d58..0dc0426b98e 100644
--- a/tests/functional/cylc-install/02-failures.t
+++ b/tests/functional/cylc-install/02-failures.t
@@ -36,14 +36,12 @@ function make_rnd_suite() {
function purge_rnd_suite() {
# Remove the suite source created by make_rnd_suite().
# And remove its run-directory too.
- RND_SUITE_SOURCE=${1:-$RND_SUITE_SOURCE}
- RND_SUITE_RUNDIR=${2:-$RND_SUITE_RUNDIR}
rm -rf "${RND_SUITE_SOURCE}"
rm -rf "${RND_SUITE_RUNDIR}"
}
. "$(dirname "$0")/test_header"
-set_test_number 16
+set_test_number 18
# Test fail no suite source dir
TEST_NAME="${TEST_NAME_BASE}-nodir"
@@ -51,44 +49,30 @@ make_rnd_suite
rm -rf "${RND_SUITE_SOURCE}"
run_fail "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}" --no-run-name -C "${RND_SUITE_SOURCE}"
contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
+WorkflowFilesError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
__ERR__
purge_rnd_suite
-
# Test fail no flow.cylc or suite.rc file
TEST_NAME="${TEST_NAME_BASE}-no-flow-file"
make_rnd_suite
rm -f "${RND_SUITE_SOURCE}/flow.cylc"
run_fail "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_NAME}" -C "${RND_SUITE_SOURCE}"
contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
+WorkflowFilesError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
__ERR__
purge_rnd_suite
# Test cylc install fails when given flow-name that is an absolute path
TEST_NAME="${TEST_NAME_BASE}-no-abs-path-flow-name"
make_rnd_suite
-rm -f "${RND_SUITE_SOURCE}/flow.cylc"
run_fail "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_SOURCE}" -C "${RND_SUITE_SOURCE}"
contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: Workflow name cannot be an absolute path: ${RND_SUITE_SOURCE}
+WorkflowFilesError: Workflow name cannot be an absolute path: ${RND_SUITE_SOURCE}
__ERR__
purge_rnd_suite
-# Test cylc install can not be run from within the cylc-run directory
-TEST_NAME="${TEST_NAME_BASE}-forbid-cylc-run-dir-install"
-BASE_NAME="cylctb-${CYLC_TEST_TIME_INIT}"
-mkdir -p ${RUN_DIR}/${BASE_NAME}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME} && cd $_
-touch flow.cylc
-run_fail "${TEST_NAME}" cylc install
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: Installation failed. Source directory should not be in ${RUN_DIR}
-__ERR__
-rm -rf ${RUN_DIR}/${BASE_NAME}
-
-
# Test source dir can not contain '_cylc-install, log, share, work' dirs
for DIR in 'work' 'share' 'log' '_cylc-install'; do
TEST_NAME="${TEST_NAME_BASE}-${DIR}-forbidden-in-source"
@@ -97,8 +81,35 @@ for DIR in 'work' 'share' 'log' '_cylc-install'; do
mkdir ${DIR}
run_fail "${TEST_NAME}" cylc install
contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: Installation failed. - ${DIR} exists in source directory.
+WorkflowFilesError: Installation failed. - ${DIR} exists in source directory.
__ERR__
- popd || exit 1
purge_rnd_suite
+ popd || exit 1
done
+
+# Test cylc install can not be run from within the cylc-run directory
+TEST_NAME="${TEST_NAME_BASE}-forbid-cylc-run-dir-install"
+BASE_NAME="test-install-${CYLC_TEST_TIME_INIT}"
+mkdir -p "${RUN_DIR}/${BASE_NAME}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}" && cd "$_" || exit
+touch flow.cylc
+run_fail "${TEST_NAME}" cylc install
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+WorkflowFilesError: Installation failed. Source directory should not be in ${RUN_DIR}
+__ERR__
+cd "${RUN_DIR}" || exit
+rm -rf "${BASE_NAME}"
+purge_rnd_suite
+
+# Test --run-name and --no-run-name options are mutually exclusive
+
+TEST_NAME="${TEST_NAME_BASE}--no-run-name-and--run-name-forbidden"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_fail "${TEST_NAME}" cylc install --run-name="${RND_SUITE_NAME}" --no-run-name
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+cylc: error: options --no-run-name and --run-name are mutually exclusive.
+__ERR__
+purge_rnd_suite
+popd || exit 1
+
+exit
diff --git a/tests/functional/cylc-install/03-file-transfer.t b/tests/functional/cylc-install/03-file-transfer.t
index c17f0c35f30..03b2cb93587 100644
--- a/tests/functional/cylc-install/03-file-transfer.t
+++ b/tests/functional/cylc-install/03-file-transfer.t
@@ -49,9 +49,10 @@ mkdir .git .svn dir1 dir2
touch .git/file1 .svn/file1 dir1/file1 dir2/file1 file1 file2
run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" --no-run-name
-tree -a -I '*.log|03-file-transfer*' "${RND_SUITE_RUNDIR}/" > 'basic-tree.out'
+tree -a -v -I '*.log|03-file-transfer*' "${RND_SUITE_RUNDIR}/" > 'basic-tree.out'
cmp_ok 'basic-tree.out' <<__OUT__
${RND_SUITE_RUNDIR}/
+├── .service
├── _cylc-install
│  └── source -> ${RND_SUITE_SOURCE}
├── dir1
@@ -63,7 +64,6 @@ ${RND_SUITE_RUNDIR}/
├── flow.cylc
├── log
│  └── install
-├── .service
└── source -> ${RND_SUITE_SOURCE}
8 directories, 5 files
@@ -90,16 +90,16 @@ __END__
run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" --no-run-name
-tree -a -I '*.log|03-file-transfer*' "${RND_SUITE_RUNDIR}/" > 'cylc-ignore-tree.out'
+tree -a -v -I '*.log|03-file-transfer*' "${RND_SUITE_RUNDIR}/" > 'cylc-ignore-tree.out'
cmp_ok 'cylc-ignore-tree.out' <<__OUT__
${RND_SUITE_RUNDIR}/
+├── .service
├── _cylc-install
│  └── source -> ${RND_SUITE_SOURCE}
├── file1
├── flow.cylc
├── log
│  └── install
-├── .service
└── source -> ${RND_SUITE_SOURCE}
6 directories, 2 files
@@ -110,5 +110,3 @@ INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}
__OUT__
popd || exit 1
purge_rnd_suite
-
-
diff --git a/tests/functional/cylc-run/01-invalid-suite.t b/tests/functional/cylc-run/01-invalid-suite.t
index cb661cbd441..f5cf3cf7838 100644
--- a/tests/functional/cylc-run/01-invalid-suite.t
+++ b/tests/functional/cylc-run/01-invalid-suite.t
@@ -23,7 +23,7 @@ set_test_number 3
#-------------------------------------------------------------------------------
INVALID_SUITE_NAME="broken-parachute-8877-mp5"
run_fail "${TEST_NAME_BASE}-run" cylc run "${INVALID_SUITE_NAME}"
-grep_ok "suite service directory not found at" "${TEST_NAME_BASE}-run.stderr"
+grep_ok "WorkflowFilesError: Suite not found: ${INVALID_SUITE_NAME}" "${TEST_NAME_BASE}-run.stderr"
exists_fail "${HOME}/cylc-run/${INVALID_SUITE_NAME}"
exit
diff --git a/tests/functional/deprecations/03-suiterc.t b/tests/functional/deprecations/03-suiterc.t
index 182f2e8625f..e4a3418b8fd 100644
--- a/tests/functional/deprecations/03-suiterc.t
+++ b/tests/functional/deprecations/03-suiterc.t
@@ -39,7 +39,7 @@ TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate .
TEST_NAME="${TEST_NAME_BASE}-install"
-run_ok "${TEST_NAME}" cylc install --flow-name=${SUITE_NAME} --no-run-name
+run_ok "${TEST_NAME}" cylc install --flow-name="${SUITE_NAME}" --no-run-name
exists_ok "flow.cylc"
diff --git a/tests/functional/events/47-long-output.t b/tests/functional/events/47-long-output.t
index 5d499005565..1ebf8078cde 100755
--- a/tests/functional/events/47-long-output.t
+++ b/tests/functional/events/47-long-output.t
@@ -63,11 +63,11 @@ run_ok "log-event-handler-00-out" \
grep -qF "[(('event-handler-00', 'succeeded'), 1) out]" 'log'
run_ok "log-event-handler-ret-code" \
grep -qF "[(('event-handler-00', 'succeeded'), 1) ret_code] 0" 'log'
-
+
purge
-
+# Forcibly remove log directory
+rm -rf "${TEST_DIR}/${SUITE_NAME}/log"
# REPEAT: Long STDERR output
-
init_suite "${TEST_NAME_BASE}" <<__FLOW_CONFIG__
[scheduling]
[[graph]]
diff --git a/tests/functional/events/suite/flow.cylc b/tests/functional/events/suite/flow.cylc
index ae36b4304e1..7b557366e0d 100644
--- a/tests/functional/events/suite/flow.cylc
+++ b/tests/functional/events/suite/flow.cylc
@@ -4,7 +4,7 @@
[runtime]
[[common]]
script = """
-cylc reg $REG $DEF
+cylc install --flow-name=$REG -C $DEF --no-run-name
echo "Sub-suite log file is: $PWD/$LOG"
if cylc run --debug --no-detach $REG > $LOG 2>&1; then
echo "ERROR: sub-suite did not abort as planned"
diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header
index ee8309ae999..0db52028b93 100644
--- a/tests/functional/lib/bash/test_header
+++ b/tests/functional/lib/bash/test_header
@@ -424,7 +424,7 @@ init_suite() {
SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}"
mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
cat "${FLOW_CONFIG}" >"${TEST_DIR}/${SUITE_NAME}/flow.cylc"
- cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
+ cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}"
cd "${TEST_DIR}/${SUITE_NAME}"
}
@@ -435,7 +435,7 @@ install_suite() {
SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}"
mkdir -p "${TEST_DIR}/${SUITE_NAME}/" # make source dir
cp -r "${TEST_SOURCE_DIR}/${TEST_SOURCE_BASE}/"* "${TEST_DIR}/${SUITE_NAME}/"
- cylc install --no-run-name --flow-name"${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}" 2>'/dev/null'
+ cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}"
cd "${TEST_DIR}/${SUITE_NAME}/"
}
diff --git a/tests/functional/registration/00-simple.t b/tests/functional/registration/00-simple.t
deleted file mode 100755
index 4d929adb557..00000000000
--- a/tests/functional/registration/00-simple.t
+++ /dev/null
@@ -1,205 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-#------------------------------------------------------------------------------
-# Test suite registration
-
-export RND_SUITE_NAME
-export RND_SUITE_SOURCE
-export RND_SUITE_RUNDIR
-
-function make_rnd_suite() {
- # Create a randomly-named suite source directory.
- # Define its run directory.
- RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)
- RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}"
- mkdir -p "${RND_SUITE_SOURCE}"
- touch "${RND_SUITE_SOURCE}/flow.cylc"
- RND_SUITE_RUNDIR="${RUN_DIR}/${RND_SUITE_NAME}"
-}
-
-function purge_rnd_suite() {
- # Remove the suite source created by make_rnd_suite().
- # And remove its run-directory too.
- RND_SUITE_SOURCE=${1:-$RND_SUITE_SOURCE}
- RND_SUITE_RUNDIR=${2:-$RND_SUITE_RUNDIR}
- rm -rf "${RND_SUITE_SOURCE}"
- rm -rf "${RND_SUITE_RUNDIR}"
-}
-
-. "$(dirname "$0")/test_header"
-set_test_number 24
-
-# Use $SUITE_NAME and $SUITE_RUN_DIR defined by test_header
-
-#------------------------------
-# Test fail no suite source dir
-TEST_NAME="${TEST_NAME_BASE}-nodir"
-make_rnd_suite
-rm -rf "${RND_SUITE_SOURCE}"
-run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
-__ERR__
-purge_rnd_suite
-
-#---------------------------
-# Test fail no flow.cylc file
-TEST_NAME="${TEST_NAME_BASE}-nodir"
-make_rnd_suite
-rm -f "${RND_SUITE_SOURCE}/flow.cylc"
-run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: no flow.cylc or suite.rc in ${RND_SUITE_SOURCE}
-__ERR__
-purge_rnd_suite
-
-#-------------------------------------------------------
-# Test default name: "cylc reg" (suite in $PWD, no args)
-TEST_NAME="${TEST_NAME_BASE}-pwd1"
-make_rnd_suite
-pushd "${RND_SUITE_SOURCE}" || exit 1
-run_ok "${TEST_NAME}" cylc install
-contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED $RND_SUITE_NAME -> ${RND_SUITE_SOURCE}
-__OUT__
-popd || exit 1
-purge_rnd_suite
-
-#--------------------------------------------------
-# Test default path: "cylc reg REG" (suite in $PWD)
-TEST_NAME="${TEST_NAME_BASE}-pwd2"
-make_rnd_suite
-pushd "${RND_SUITE_SOURCE}" || exit 1
-run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}"
-contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
-__OUT__
-popd || exit 1
-purge_rnd_suite
-
-#-------------------------
-# Test "cylc reg REG PATH"
-TEST_NAME="${TEST_NAME_BASE}-normal"
-make_rnd_suite
-run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
-__OUT__
-purge_rnd_suite
-
-#--------------------------------------------------------------------
-# Test register existing run directory: "cylc reg REG ~/cylc-run/REG"
-TEST_NAME="${TEST_NAME_BASE}-reg-run-dir"
-make_rnd_suite
-mkdir -p "${RND_SUITE_RUNDIR}"
-cp "${RND_SUITE_SOURCE}/flow.cylc" "${RND_SUITE_RUNDIR}"
-run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
-contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
-__OUT__
-SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")"
-run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}"
-# Run it twice
-run_ok "${TEST_NAME}-2" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_RUNDIR}"
-contains_ok "${TEST_NAME}-2.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_RUNDIR}
-__OUT__
-SOURCE="$(readlink "${RND_SUITE_RUNDIR}/.service/source")"
-run_ok "${TEST_NAME}-source" test '..' = "${SOURCE}"
-purge_rnd_suite
-
-#----------------------------------------------------------------
-# Test fail "cylc reg REG PATH" where REG already points to PATH2
-TEST_NAME="${TEST_NAME_BASE}-dup1"
-make_rnd_suite
-run_ok "${TEST_NAME}" cylc install "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-RND_SUITE_NAME1="${RND_SUITE_NAME}"
-RND_SUITE_SOURCE1="${RND_SUITE_SOURCE}"
-RND_SUITE_RUNDIR1="${RND_SUITE_RUNDIR}"
-make_rnd_suite
-TEST_NAME="${TEST_NAME_BASE}-dup2"
-run_fail "${TEST_NAME}" cylc install "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-SuiteServiceFileError: the name '${RND_SUITE_NAME1}' already points to ${RND_SUITE_SOURCE1}.
-Use --redirect to re-use an existing name and run directory.
-__ERR__
-# Now force it
-TEST_NAME="${TEST_NAME_BASE}-dup3"
-run_ok "${TEST_NAME}" cylc install --redirect "${RND_SUITE_NAME1}" "${RND_SUITE_SOURCE}"
-sed -i 's/^\t//; s/^.* WARNING - /WARNING - /' "${TEST_NAME}.stderr"
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-WARNING - the name '${RND_SUITE_NAME1}' points to ${RND_SUITE_SOURCE1}.
-It will now be redirected to ${RND_SUITE_SOURCE}.
-Files in the existing ${RND_SUITE_NAME1} run directory will be overwritten.
-__ERR__
-contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME1} -> ${RND_SUITE_SOURCE}
-__OUT__
-
-TEST_NAME="${TEST_NAME_BASE}-get-dir"
-contains_ok "${TEST_NAME}.stdout" <<__ERR__
-${RND_SUITE_SOURCE}
-__ERR__
-
-purge_rnd_suite
-purge_rnd_suite "${RND_SUITE_SOURCE1}" "${RND_SUITE_RUNDIR1}"
-
-#-----------------------
-# Test alternate run dir
-# 1. Normal case.
-TEST_NAME="${TEST_NAME_BASE}-alt-run-dir"
-make_rnd_suite
-ALT_RUN_DIR="${PWD}/alt"
-run_ok "${TEST_NAME}" \
- cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stdout" <<__OUT__
-REGISTERED ${RND_SUITE_NAME} -> ${RND_SUITE_SOURCE}
-__OUT__
-run_ok "${TEST_NAME}-check-link" test -L "${RND_SUITE_RUNDIR}"
-run_ok "${TEST_NAME}-rm-link" rm "${RND_SUITE_RUNDIR}"
-run_ok "${TEST_NAME}-rm-alt-run-dir" rm -r "${ALT_RUN_DIR}"
-purge_rnd_suite
-
-# 2. If reg already exists (as a directory).
-TEST_NAME="${TEST_NAME_BASE}-alt-exists1"
-make_rnd_suite
-ALT_RUN_DIR="${PWD}/alt"
-mkdir -p "${RND_SUITE_RUNDIR}"
-run_fail "${TEST_NAME}" \
- cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__OUT__
-SuiteServiceFileError: Run directory '${RND_SUITE_RUNDIR}' already exists.
-__OUT__
-purge_rnd_suite
-
-# 3. If reg already exists (as a valid symlink).
-TEST_NAME="${TEST_NAME_BASE}-alt-exists2"
-make_rnd_suite
-ALT_RUN_DIR="${PWD}/alt"
-TDIR=$(mktemp -d)
-mkdir -p "$(dirname "${RND_SUITE_RUNDIR}")"
-ln -s "${TDIR}" "${RND_SUITE_RUNDIR}"
-run_fail "${TEST_NAME}" \
- cylc install --run-dir="${ALT_RUN_DIR}" "${RND_SUITE_NAME}" "${RND_SUITE_SOURCE}"
-contains_ok "${TEST_NAME}.stderr" <<__OUT__
-SuiteServiceFileError: Symlink '${RND_SUITE_RUNDIR}' already points to ${TDIR}.
-__OUT__
-purge_rnd_suite
-rm -rf "${TDIR}"
-
-exit
diff --git a/tests/functional/rose-conf/03-fileinstall.t b/tests/functional/rose-conf/03-fileinstall.t
index c9e6b1980ac..3fd20bae9f1 100755
--- a/tests/functional/rose-conf/03-fileinstall.t
+++ b/tests/functional/rose-conf/03-fileinstall.t
@@ -23,12 +23,18 @@ python -c "import cylc.rose" > /dev/null 2>&1 ||
skip_all "cylc.rose not installed in environment."
set_test_number 3
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-sed -i "s@REPLACE_THIS@${CYLC_REPO_DIR}/tests/functional/rose-conf/fileinstall_data@g" rose-suite.conf
+# make new source dir
+SOURCE_DIR="${PWD}/cylc-source-dir"
+mkdir "${SOURCE_DIR}"
+cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}" "${SOURCE_DIR}/03-fileinstall"
+# cp -r "${CYLC_REPO_DIR}/tests/functional/rose-conf/fileinstall_data/" "${SOURCE_DIR}/03-fileinstall"
+sed -i "s@REPLACE_THIS@${CYLC_REPO_DIR}/tests/functional/rose-conf/fileinstall_data@g" "${SOURCE_DIR}/03-fileinstall/rose-suite.conf"
+SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}"
+SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}/03-fileinstall"
-run_ok "${TEST_NAME_BASE}-validate" cylc install "${SUITE_NAME}"
+run_ok "{TEST_NAME_BASE}-install" cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${SOURCE_DIR}/03-fileinstall"
# Test that data files have been concatenated.
DATA_INSTALLED_PATH="${SUITE_RUN_DIR}/data"
diff --git a/tests/functional/suite-state/00-polling.t b/tests/functional/suite-state/00-polling.t
index 64a6aef5502..b95bfe7e831 100644
--- a/tests/functional/suite-state/00-polling.t
+++ b/tests/functional/suite-state/00-polling.t
@@ -24,11 +24,11 @@ set_test_number 5
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" 'polling'
#-------------------------------------------------------------------------------
-# copy the upstream suite to the test directory and register it
+# copy the upstream suite to the test directory and install it
cp -r "${TEST_SOURCE_DIR}/upstream" "${TEST_DIR}/"
# use full range of characters in the suite-to-be-polled name:
UPSTREAM="${SUITE_NAME}-up_stre.am"
-cylc reg "${UPSTREAM}" "${TEST_DIR}/upstream"
+cylc install --flow-name="${UPSTREAM}" -C "${TEST_DIR}/upstream" --no-run-name
#-------------------------------------------------------------------------------
# validate both suites as tests
TEST_NAME="${TEST_NAME_BASE}-validate-upstream"
diff --git a/tests/functional/suite-state/01-polling.t b/tests/functional/suite-state/01-polling.t
index dfdc83bd036..1623f1f82ed 100644
--- a/tests/functional/suite-state/01-polling.t
+++ b/tests/functional/suite-state/01-polling.t
@@ -24,11 +24,11 @@ set_test_number 5
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" 'polling'
#-------------------------------------------------------------------------------
-# copy the upstream suite to the test directory and register it
+# copy the upstream suite to the test directory and install it
cp -r "${TEST_SOURCE_DIR}/upstream" "${TEST_DIR}/"
# this version uses a simple Rose-style suite name [\w-]
UPSTREAM="${SUITE_NAME}-upstream"
-cylc reg "${UPSTREAM}" "${TEST_DIR}/upstream"
+cylc install --flow-name="${UPSTREAM}" -C "${TEST_DIR}/upstream" --no-run-name
#-------------------------------------------------------------------------------
# validate both suites as tests
TEST_NAME="${TEST_NAME_BASE}-validate-upstream"
diff --git a/tests/functional/validate/48-reg-then-pwd.t b/tests/functional/validate/48-reg-then-pwd.t
index 190df61bb92..450030751ba 100755
--- a/tests/functional/validate/48-reg-then-pwd.t
+++ b/tests/functional/validate/48-reg-then-pwd.t
@@ -43,7 +43,7 @@ __FLOW_CONFIG__
run_fail "${TEST_NAME_BASE}" cylc validate "${SUITE_NAME}"
# This should validate installed good suite
-cylc install --flow-name="${SUITE_NAME}" -C "${PWD}/good"
+cylc install --flow-name="${SUITE_NAME}" -C "${PWD}/good" --no-run-name
run_ok "${TEST_NAME_BASE}" cylc validate "${SUITE_NAME}"
purge
diff --git a/tests/integration/test_scan_api.py b/tests/integration/test_scan_api.py
index 858d0d10482..f296b49928a 100644
--- a/tests/integration/test_scan_api.py
+++ b/tests/integration/test_scan_api.py
@@ -259,12 +259,10 @@ async def test_scan_cleans_stuck_contact_files(
schd = scheduler(reg)
srv_dir = Path(run_dir, reg, SuiteFiles.Service.DIRNAME)
tmp_dir = test_dir / 'srv'
- cont = run_dir / SuiteFiles.Service.CONTACT
+ cont = srv_dir / SuiteFiles.Service.CONTACT
# run the flow, copy the contact, stop the flow, copy back the contact
async with run(schd):
- # remove the source symlink to avoid recursion
- (run_dir / SuiteFiles.Install.SOURCE).unlink()
copytree(srv_dir, tmp_dir)
rmtree(srv_dir)
copytree(tmp_dir, srv_dir)
diff --git a/tests/unit/test_install.py b/tests/unit/test_install.py
deleted file mode 100644
index 7423857685a..00000000000
--- a/tests/unit/test_install.py
+++ /dev/null
@@ -1,26 +0,0 @@
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-
-from cylc.flow.exceptions import SuiteServiceFileError
-from cylc.flow.suite_files import validate_source_dir
-
-
-# def test_validate_source_dir(source):
-
-# expected = raise(SuiteServiceFileError)
-# source_dir =
-
-
\ No newline at end of file
diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py
index 4102e63ce5c..8cd91645306 100644
--- a/tests/unit/test_pathutil.py
+++ b/tests/unit/test_pathutil.py
@@ -98,7 +98,7 @@ class TestPathutil(TestCase):
failed.
"""
@patch('cylc.flow.pathutil.get_platform')
- def test_get_suite_run_dirs(self, mocked_platform):
+ def test_get_workflow_run_dirs(self, mocked_platform):
"""Usage of get_suite_run_*dir."""
homedir = os.getenv("HOME")
mocked = MagicMock()
@@ -261,9 +261,7 @@ def test_make_localhost_symlinks_calls_make_symlink_for_each_key_value_dir(
'share': '$DEE/suite3/share'}
mocked_get_workflow_run_dir.return_value = "rund"
mocked_expandvars.return_value = "expanded"
-
make_localhost_symlinks('rund', 'suite')
-
mocked_make_symlink.assert_has_calls([
call('expanded', 'rund'),
call('expanded', 'rund/log'),
diff --git a/tests/unit/test_suite_files.py b/tests/unit/test_suite_files.py
index 72f5c32487a..79dddaecf96 100644
--- a/tests/unit/test_suite_files.py
+++ b/tests/unit/test_suite_files.py
@@ -64,24 +64,6 @@ def test_is_valid_run_dir(path, expected, is_abs_path, monkeypatch):
f'Is "{path}" a valid run dir?')
-@pytest.mark.parametrize('direction', ['parents', 'children'])
-def test_nested_run_dirs_raise_error(direction, monkeypatch):
- """Test that a suite cannot be contained in a subdir of another suite."""
- monkeypatch.setattr('cylc.flow.suite_files.get_cylc_run_abs_path',
- lambda x: x)
- if direction == "parents":
- monkeypatch.setattr('cylc.flow.suite_files.os.scandir', lambda x: [])
- monkeypatch.setattr('cylc.flow.suite_files.is_valid_run_dir',
- lambda x: x == os.path.join('bright', 'falls'))
- # Not nested in run dir - ok:
- suite_files.check_nested_run_dirs('alan/wake')
- # It is itself a run dir - ok:
- suite_files.check_nested_run_dirs('bright/falls')
- # Nested in a run dir - bad:
- for path in ('bright/falls/light', 'bright/falls/light/and/power'):
- with pytest.raises(WorkflowFilesError) as exc:
- suite_files.check_nested_run_dirs(path)
- assert 'Nested run directories not allowed' in str(exc.value)
@pytest.mark.parametrize(
'run_dir',
[
@@ -126,7 +108,7 @@ def test_rundir_parent_that_contains_workflow_raises_error(
monkeypatch.setattr(
'cylc.flow.suite_files.os.scandir', lambda x: [])
- with pytest.raises(SuiteServiceFileError) as exc:
+ with pytest.raises(WorkflowFilesError) as exc:
suite_files.check_nested_run_dirs(run_dir, 'placeholder_flow')
assert 'Nested run directories not allowed' in str(exc.value)
@@ -184,10 +166,11 @@ def test_rundir_children_that_contain_workflows_raise_error(
mock.Mock(path=srv_dir[0:len(x) + 2],
is_symlink=lambda: False)])
- with pytest.raises(SuiteServiceFileError) as exc:
+ with pytest.raises(WorkflowFilesError) as exc:
check_nested_run_dirs(run_dir, 'placeholder_flow')
assert 'Nested run directories not allowed' in str(exc.value)
+
@pytest.mark.parametrize(
'reg, expected_err',
[('foo/bar/', None),
@@ -416,7 +399,7 @@ def mocked_detect_old_contact_file(reg):
monkeypatch.setattr('cylc.flow.suite_files.detect_old_contact_file',
mocked_detect_old_contact_file)
- monkeypatch.setattr('cylc.flow.suite_files.get_suite_run_dir',
+ monkeypatch.setattr('cylc.flow.suite_files.get_workflow_run_dir',
lambda x: tmp_path.joinpath('cylc-run', x))
# --- The actual test ---
@@ -609,6 +592,7 @@ def test_remove_empty_reg_parents(tmp_path):
assert tmp_path.joinpath('foo').exists() is False
assert tmp_path.exists() is True
+
@pytest.mark.parametrize(
'run_dir, srv_dir',
[
@@ -637,4 +621,3 @@ def test_symlinkrundir_children_that_contain_workflows_raise_error(
check_nested_run_dirs(run_dir, 'placeholder_flow')
except SuiteServiceFileError:
pytest.fail("Unexpected SuiteServiceFileError, Check symlink logic.")
-
From f57b5aeb1cd2b3c7a0b2773829e54584554f0a7b Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Tue, 5 Jan 2021 15:32:13 +0000
Subject: [PATCH 05/13] Rose-cylc install fix
Add functional tests for cylc-install
---
CHANGES.md | 7 +++++--
cylc/flow/scripts/install.py | 11 +++++------
cylc/flow/suite_files.py | 7 +++++--
dockerfiles/bash/Dockerfile | 1 +
.../authentication/01-remote-suite-same-name.t | 2 +-
tests/functional/cylc-install/00-simple.t | 15 ++++++++++++++-
tests/functional/cylc-install/02-failures.t | 10 +++++++++-
tests/functional/rose-conf/03-fileinstall.t | 5 +----
tests/functional/rose-conf/04-opts-set-from-env.t | 4 +---
tests/functional/rose-conf/06-jinja2.thorough.t | 3 +--
.../functional/rose-conf/fileinstall_data/lion.py | 1 +
11 files changed, 44 insertions(+), 22 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index b413ae69d1f..4f4217b6d51 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -59,14 +59,17 @@ The filenames `suite.rc` and `global.rc` are now deprecated in favour of
compatibility, the `cylc run` command will automatically symlink an existing
`suite.rc` file to `flow.cylc`.
-Remove cylc register's option `--run-dir=DIR`, which created a run directory
-symlink to `DIR` (see #3884).
+Remove `cylc register` command
+([#4000](https://github.com/cylc/cylc-flow/pull/4000)).
### Enhancements
[#4014](https://github.com/cylc/cylc-flow/pull/4014) - Rename "ready" task
state to "preparing".
+[#4000](https://github.com/cylc/cylc-flow/pull/4000) - Cylc install command
+added. Install workflows into cylc run directory from arbitrary locations.
+
[#3992](https://github.com/cylc/cylc-flow/pull/3992) - Rename
batch system to job runner.
diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py
index 9fd01520461..f0c17a60437 100755
--- a/cylc/flow/scripts/install.py
+++ b/cylc/flow/scripts/install.py
@@ -59,12 +59,10 @@
"""
-import os
import pkg_resources
from cylc.flow.exceptions import PluginError
from cylc.flow.option_parsers import CylcOptionParser as COP
-from cylc.flow.pathutil import get_workflow_run_dir
from cylc.flow.suite_files import install_workflow
from cylc.flow.terminal import cli_function
@@ -139,21 +137,22 @@ def main(parser, opts, flow_name=None, src=None):
exc
) from None
- flow_name = install_workflow(
+ source_dir, rundir, _flow_name = install_workflow(
flow_name=opts.flow_name,
source=opts.source,
run_name=opts.run_name,
no_run_name=opts.no_run_name,
- no_symlinks=opts.no_symlinks)
+ no_symlinks=opts.no_symlinks
+ )
for entry_point in pkg_resources.iter_entry_points(
'cylc.post_install'
):
try:
entry_point.resolve()(
- dir_=os.getcwd(),
+ dir_=source_dir,
opts=opts,
- dest_root=get_workflow_run_dir(flow_name)
+ dest_root=str(rundir)
)
except Exception as exc:
# NOTE: except Exception (purposefully vague)
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index a55a1bb2dbd..46c330210de 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -1002,6 +1002,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
no_run_name=False, no_symlinks=False):
"""Install a workflow, or renew its installation.
+ Install workflow into new run directory.
Create symlink to suite source location, creating any symlinks for run,
work, log, share, share/cycle directories.
@@ -1014,7 +1015,9 @@ def install_workflow(flow_name=None, source=None, run_name=None,
rundir (str): for overriding the default cylc-run directory.
Return:
- str: The installed suite name (which may be computed here).
+ source (Path): The source direcory.
+ rundir (Path): The directory the workflow has been installed into.
+ flow_name (str): The installed suite name (which may be computed here).
Raise:
WorkflowFilesError:
@@ -1111,7 +1114,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
INSTALL_LOG.info(f'INSTALLED {flow_name} from {source} -> {rundir}')
print(f'INSTALLED {flow_name} from {source} -> {rundir}')
_close_install_log()
- return flow_name
+ return source, rundir, flow_name
def create_workflow_srv_dir(rundir=None, source=None):
diff --git a/dockerfiles/bash/Dockerfile b/dockerfiles/bash/Dockerfile
index ff0d51b4084..2396924a1d9 100644
--- a/dockerfiles/bash/Dockerfile
+++ b/dockerfiles/bash/Dockerfile
@@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y \
python3.7-dev \
sqlite3 \
wget \
+ rsync \
&& python3.7 -m pip install -U pip setuptools \
&& rm -rf /var/lib/apt/lists/*
diff --git a/tests/functional/authentication/01-remote-suite-same-name.t b/tests/functional/authentication/01-remote-suite-same-name.t
index 502b7723282..385b5cdfca2 100755
--- a/tests/functional/authentication/01-remote-suite-same-name.t
+++ b/tests/functional/authentication/01-remote-suite-same-name.t
@@ -39,7 +39,7 @@ run_ok "${TEST_NAME_BASE}-install" \
suite_run_ok "${TEST_NAME_BASE}" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-
+# shellcheck disable=SC2086
ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" \
rm -rf cylctb-cylc-source
purge
diff --git a/tests/functional/cylc-install/00-simple.t b/tests/functional/cylc-install/00-simple.t
index 61c7ad67259..b12cb729100 100755
--- a/tests/functional/cylc-install/00-simple.t
+++ b/tests/functional/cylc-install/00-simple.t
@@ -18,7 +18,7 @@
#------------------------------------------------------------------------------
# Test workflow installation
. "$(dirname "$0")/test_header"
-set_test_number 18
+set_test_number 20
export RND_SUITE_NAME
export RND_SUITE_SOURCE
@@ -53,6 +53,19 @@ __OUT__
popd || exit 1
purge_rnd_suite
+# Test default name: "cylc install" (suite in $PWD, flow.cylc given as arg)
+TEST_NAME="${TEST_NAME_BASE}-basic"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}" cylc install flow.cylc
+
+contains_ok "${TEST_NAME}.stdout" <<__OUT__
+INSTALLED $RND_SUITE_NAME from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
+__OUT__
+popd || exit 1
+purge_rnd_suite
+
+
# Test default path: "cylc install REG" (flow in $PWD)
TEST_NAME="${TEST_NAME_BASE}-pwd2"
make_rnd_suite
diff --git a/tests/functional/cylc-install/02-failures.t b/tests/functional/cylc-install/02-failures.t
index 0dc0426b98e..8e973c94e6f 100644
--- a/tests/functional/cylc-install/02-failures.t
+++ b/tests/functional/cylc-install/02-failures.t
@@ -41,7 +41,7 @@ function purge_rnd_suite() {
}
. "$(dirname "$0")/test_header"
-set_test_number 18
+set_test_number 20
# Test fail no suite source dir
TEST_NAME="${TEST_NAME_BASE}-nodir"
@@ -72,6 +72,14 @@ WorkflowFilesError: Workflow name cannot be an absolute path: ${RND_SUITE_SOURCE
__ERR__
purge_rnd_suite
+# Test cylc install fails when given run-name _cylc-install
+TEST_NAME="${TEST_NAME_BASE}-run-name-cylc-install-forbidden"
+make_rnd_suite
+run_fail "${TEST_NAME}" cylc install --run-name=_cylc-install -C "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+WorkflowFilesError: Run name cannot be "_cylc-install". Please choose another run name.
+__ERR__
+purge_rnd_suite
# Test source dir can not contain '_cylc-install, log, share, work' dirs
for DIR in 'work' 'share' 'log' '_cylc-install'; do
diff --git a/tests/functional/rose-conf/03-fileinstall.t b/tests/functional/rose-conf/03-fileinstall.t
index 3fd20bae9f1..c114680b874 100755
--- a/tests/functional/rose-conf/03-fileinstall.t
+++ b/tests/functional/rose-conf/03-fileinstall.t
@@ -28,13 +28,10 @@ set_test_number 3
SOURCE_DIR="${PWD}/cylc-source-dir"
mkdir "${SOURCE_DIR}"
cp -r "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}" "${SOURCE_DIR}/03-fileinstall"
-# cp -r "${CYLC_REPO_DIR}/tests/functional/rose-conf/fileinstall_data/" "${SOURCE_DIR}/03-fileinstall"
-
sed -i "s@REPLACE_THIS@${CYLC_REPO_DIR}/tests/functional/rose-conf/fileinstall_data@g" "${SOURCE_DIR}/03-fileinstall/rose-suite.conf"
SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}"
SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}/03-fileinstall"
-
-run_ok "{TEST_NAME_BASE}-install" cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${SOURCE_DIR}/03-fileinstall"
+run_ok "{TEST_NAME_BASE}-install" cylc install --no-run-name --flow-name="${SUITE_NAME}/03-fileinstall" --directory="${SOURCE_DIR}/03-fileinstall"
# Test that data files have been concatenated.
DATA_INSTALLED_PATH="${SUITE_RUN_DIR}/data"
diff --git a/tests/functional/rose-conf/04-opts-set-from-env.t b/tests/functional/rose-conf/04-opts-set-from-env.t
index 2c83c48411f..9d8fbfafa64 100755
--- a/tests/functional/rose-conf/04-opts-set-from-env.t
+++ b/tests/functional/rose-conf/04-opts-set-from-env.t
@@ -23,10 +23,8 @@ python -c "import cylc.rose" > /dev/null 2>&1 ||
skip_all "cylc.rose not installed in environment."
set_test_number 2
-install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
export ROSE_SUITE_OPT_CONF_KEYS=Gaelige
-
+install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
cylc view -p --stdout "${SUITE_NAME}" > processed.conf.test
diff --git a/tests/functional/rose-conf/06-jinja2.thorough.t b/tests/functional/rose-conf/06-jinja2.thorough.t
index f588ce14926..6270d803d1b 100755
--- a/tests/functional/rose-conf/06-jinja2.thorough.t
+++ b/tests/functional/rose-conf/06-jinja2.thorough.t
@@ -22,10 +22,9 @@ python -c "import cylc.rose" > /dev/null 2>&1 ||
skip_all "cylc.rose not installed in environment."
set_test_number 2
-
+export XYZ=xyz
install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-export XYZ=xyz
run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
XYZ=xyz cylc view -p --stdout "${SUITE_NAME}" > processed.conf.test 2> /dev/null
diff --git a/tests/functional/rose-conf/fileinstall_data/lion.py b/tests/functional/rose-conf/fileinstall_data/lion.py
index 4f386f35783..02067143c24 100755
--- a/tests/functional/rose-conf/fileinstall_data/lion.py
+++ b/tests/functional/rose-conf/fileinstall_data/lion.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
+
class TerriblePunException(Exception):
pass
From 74f0d61b4ed5f3a458a270674b32bf4acf0450b5 Mon Sep 17 00:00:00 2001
From: Melanie Hall <37735232+datamel@users.noreply.github.com>
Date: Fri, 8 Jan 2021 10:20:38 +0000
Subject: [PATCH 06/13] Update cylc/flow/suite_files.py
Co-authored-by: Oliver Sanders
---
cylc/flow/suite_files.py | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index 46c330210de..e8698186899 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -973,12 +973,14 @@ def _close_install_log():
def get_rsync_rund_cmd(src, dst, restart=False):
"""Create and return the rsync command used for cylc install/re-install.
- Args:
- src (str): file path location of source directory
- dst (str): file path location of destination directory
- restart (bool): indicate restart (--delete option added)
- Return: rsync_cmd: command used for rsync.
+ Args:
+ src (str): file path location of source directory
+ dst (str): file path location of destination directory
+ restart (bool): indicate restart (--delete option added)
+
+ Return:
+ list: command to use for rsync.
"""
From 0f6f89b6ebe02fb5ab2c36743537ec00fe0a8457 Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Sun, 10 Jan 2021 15:02:20 +0000
Subject: [PATCH 07/13] Ammend directory structure for cylc-install
Fix functional test for cylc-install
---
cylc/flow/suite_files.py | 25 +++++++------
tests/functional/cylc-diff/00-basic.t | 4 +--
tests/functional/cylc-diff/01-same.t | 4 +--
tests/functional/cylc-diff/02-identical.t | 4 +--
tests/functional/cylc-diff/03-icp.t | 4 +--
tests/functional/cylc-diff/04-icp-2.t | 4 +--
tests/functional/cylc-install/02-failures.t | 25 ++++++++++++-
.../cylc-install/03-file-transfer.t | 14 ++++----
tests/functional/events/00-suite.t | 6 ++--
.../17-task-event-job-logs-retrieve-command.t | 7 ++--
tests/functional/events/suite/flow.cylc | 35 ++++++++++---------
.../12-tidy-submits-of-prev-run.t | 4 +--
tests/functional/reload/17-graphing-change.t | 6 ++--
tests/functional/restart/01-broadcast.t | 11 +++---
tests/functional/restart/broadcast/flow.cylc | 11 ++++--
.../restart/lib/flow-runtime-restart.cylc | 2 +-
tests/functional/shutdown/19-log-reference.t | 2 +-
17 files changed, 101 insertions(+), 67 deletions(-)
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index e8698186899..d78de09d173 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -988,7 +988,7 @@ def get_rsync_rund_cmd(src, dst, restart=False):
rsync_cmd.append("-av")
if restart:
rsync_cmd.append('--delete')
- ignore_dirs = ['.git', '.svn', '.cylcignore']
+ ignore_dirs = ['.git', '.svn', '.cylcignore', SuiteFiles.Install.DIRNAME]
for exclude in ignore_dirs:
if Path(src).joinpath(exclude).exists():
rsync_cmd.append(f"--exclude={exclude}")
@@ -1015,6 +1015,9 @@ def install_workflow(flow_name=None, source=None, run_name=None,
If specified, cylc install will not create runN
symlink.
rundir (str): for overriding the default cylc-run directory.
+ no_run_name (bool): Flag as True to install workflow into
+ ~/cylc-run/$(basename $PWD)
+ no_symlinks (bool): Flag as True to skip making localhost symlink dirs
Return:
source (Path): The source direcory.
@@ -1028,6 +1031,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
Another suite already has this name (unless --redirect).
Trying to install a workflow that is nested inside of another.
"""
+
if not source:
source = Path.cwd()
elif Path(source).name == SuiteFiles.FLOW_FILE:
@@ -1043,7 +1047,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
validate_source_dir(source)
run_path_base = Path(get_workflow_run_dir(flow_name)).expanduser()
relink = False
- run_num = int()
+ run_num = 0
if no_run_name:
rundir = run_path_base
elif run_name:
@@ -1072,11 +1076,6 @@ def install_workflow(flow_name=None, source=None, run_name=None,
except OSError as e:
if e.strerror == "File exists":
raise WorkflowFilesError(f"Run directory already exists : {e}")
- # create source symlink to be used as the basis of ensuring runs are
- # from a constistent source dir.
- base_source_link = run_path_base.joinpath(SuiteFiles.Install.SOURCE)
- if not base_source_link.exists():
- run_path_base.joinpath(SuiteFiles.Install.SOURCE).symlink_to(source)
if relink:
link_runN(rundir)
create_workflow_srv_dir(rundir)
@@ -1103,16 +1102,20 @@ def install_workflow(flow_name=None, source=None, run_name=None,
INSTALL_LOG.warning(
f"An error occurred when copying files from {source} to {rundir}")
INSTALL_LOG.warning(f" Error: {stderr}")
- cylc_install = Path(rundir, SuiteFiles.Install.DIRNAME)
- cylc_install.mkdir(parents=True)
+ cylc_install = Path(rundir.parent, SuiteFiles.Install.DIRNAME)
+ if no_run_name:
+ cylc_install = Path(rundir, SuiteFiles.Install.DIRNAME)
source_link = cylc_install.joinpath(SuiteFiles.Install.SOURCE)
- # check source link matches the source symlink from workflow dir.
- if os.readlink(base_source_link) == str(source):
+ cylc_install.mkdir(parents=True, exist_ok=True)
+ if not source_link.exists():
INSTALL_LOG.info(f"Creating symlink from {source_link}")
source_link.symlink_to(source)
+ elif source_link.exists() and (os.readlink(source_link) == str(source)):
+ INSTALL_LOG.info("Symlink from {source_link} to {source} in place.")
else:
raise WorkflowFilesError(
"Source directory between runs are not consistent")
+ # check source link matches the source symlink from workflow dir.
INSTALL_LOG.info(f'INSTALLED {flow_name} from {source} -> {rundir}')
print(f'INSTALLED {flow_name} from {source} -> {rundir}')
_close_install_log()
diff --git a/tests/functional/cylc-diff/00-basic.t b/tests/functional/cylc-diff/00-basic.t
index 5c888a4ccca..98c01af4b72 100755
--- a/tests/functional/cylc-diff/00-basic.t
+++ b/tests/functional/cylc-diff/00-basic.t
@@ -43,8 +43,8 @@ SUITE_NAME2="${SUITE_NAME}"
run_ok "${TEST_NAME_BASE}" cylc diff "${SUITE_NAME1}" "${SUITE_NAME2}"
cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__
-Parsing ${SUITE_NAME1} (${TEST_DIR}/${SUITE_NAME1}/flow.cylc)
-Parsing ${SUITE_NAME2} (${TEST_DIR}/${SUITE_NAME2}/flow.cylc)
+Parsing ${SUITE_NAME1} (${RUN_DIR}/${SUITE_NAME1}/flow.cylc)
+Parsing ${SUITE_NAME2} (${RUN_DIR}/${SUITE_NAME2}/flow.cylc)
Suite definitions ${SUITE_NAME1} and ${SUITE_NAME2} differ
2 items only in ${SUITE_NAME1} (<)
diff --git a/tests/functional/cylc-diff/01-same.t b/tests/functional/cylc-diff/01-same.t
index 3a14430486d..1b8c45c0dc4 100755
--- a/tests/functional/cylc-diff/01-same.t
+++ b/tests/functional/cylc-diff/01-same.t
@@ -37,8 +37,8 @@ cylc install --flow-name="${SUITE_NAME2}" --directory="${TEST_DIR}/${SUITE_NAME1
run_ok "${TEST_NAME_BASE}" cylc diff "${SUITE_NAME1}" "${SUITE_NAME2}"
cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__
-Parsing ${SUITE_NAME1} (${TEST_DIR}/${SUITE_NAME1}/flow.cylc)
-Parsing ${SUITE_NAME2} (${TEST_DIR}/${SUITE_NAME1}/flow.cylc)
+Parsing ${SUITE_NAME1} (${RUN_DIR}/${SUITE_NAME1}/flow.cylc)
+Parsing ${SUITE_NAME2} (${RUN_DIR}/${SUITE_NAME2}/flow.cylc)
Suite definitions ${SUITE_NAME1} and ${SUITE_NAME2} are identical
__OUT__
cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null'
diff --git a/tests/functional/cylc-diff/02-identical.t b/tests/functional/cylc-diff/02-identical.t
index f5cf4f9b399..ba2b973ca44 100755
--- a/tests/functional/cylc-diff/02-identical.t
+++ b/tests/functional/cylc-diff/02-identical.t
@@ -43,8 +43,8 @@ SUITE_NAME2="${SUITE_NAME}"
run_ok "${TEST_NAME_BASE}" cylc diff "${SUITE_NAME1}" "${SUITE_NAME2}"
cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__
-Parsing ${SUITE_NAME1} (${TEST_DIR}/${SUITE_NAME1}/flow.cylc)
-Parsing ${SUITE_NAME2} (${TEST_DIR}/${SUITE_NAME2}/flow.cylc)
+Parsing ${SUITE_NAME1} (${RUN_DIR}/${SUITE_NAME1}/flow.cylc)
+Parsing ${SUITE_NAME2} (${RUN_DIR}/${SUITE_NAME2}/flow.cylc)
Suite definitions ${SUITE_NAME1} and ${SUITE_NAME2} are identical
__OUT__
cmp_ok "${TEST_NAME_BASE}.stderr" <'/dev/null'
diff --git a/tests/functional/cylc-diff/03-icp.t b/tests/functional/cylc-diff/03-icp.t
index 32569a2ec31..97f91367e10 100755
--- a/tests/functional/cylc-diff/03-icp.t
+++ b/tests/functional/cylc-diff/03-icp.t
@@ -48,8 +48,8 @@ SUITE_NAME2="${SUITE_NAME}"
run_ok "${TEST_NAME_BASE}" \
cylc diff --icp=2020 "${SUITE_NAME1}" "${SUITE_NAME2}"
cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__
-Parsing ${SUITE_NAME1} (${TEST_DIR}/${SUITE_NAME1}/flow.cylc)
-Parsing ${SUITE_NAME2} (${TEST_DIR}/${SUITE_NAME2}/flow.cylc)
+Parsing ${SUITE_NAME1} (${RUN_DIR}/${SUITE_NAME1}/flow.cylc)
+Parsing ${SUITE_NAME2} (${RUN_DIR}/${SUITE_NAME2}/flow.cylc)
Suite definitions ${SUITE_NAME1} and ${SUITE_NAME2} differ
2 items only in ${SUITE_NAME1} (<)
diff --git a/tests/functional/cylc-diff/04-icp-2.t b/tests/functional/cylc-diff/04-icp-2.t
index 5298ca70f2c..191dfd0a9ae 100755
--- a/tests/functional/cylc-diff/04-icp-2.t
+++ b/tests/functional/cylc-diff/04-icp-2.t
@@ -49,8 +49,8 @@ SUITE_NAME2="${SUITE_NAME}"
run_ok "${TEST_NAME_BASE}" \
cylc diff --icp=2020 "${SUITE_NAME1}" "${SUITE_NAME2}"
cmp_ok "${TEST_NAME_BASE}.stdout" <<__OUT__
-Parsing ${SUITE_NAME1} (${TEST_DIR}/${SUITE_NAME1}/flow.cylc)
-Parsing ${SUITE_NAME2} (${TEST_DIR}/${SUITE_NAME2}/flow.cylc)
+Parsing ${SUITE_NAME1} (${RUN_DIR}/${SUITE_NAME1}/flow.cylc)
+Parsing ${SUITE_NAME2} (${RUN_DIR}/${SUITE_NAME2}/flow.cylc)
Suite definitions ${SUITE_NAME1} and ${SUITE_NAME2} differ
2 items only in ${SUITE_NAME1} (<)
diff --git a/tests/functional/cylc-install/02-failures.t b/tests/functional/cylc-install/02-failures.t
index 8e973c94e6f..1723d8aa9d3 100644
--- a/tests/functional/cylc-install/02-failures.t
+++ b/tests/functional/cylc-install/02-failures.t
@@ -41,7 +41,30 @@ function purge_rnd_suite() {
}
. "$(dirname "$0")/test_header"
-set_test_number 20
+set_test_number 23
+
+# Test source directory between runs that are not consistent result in error
+
+TEST_NAME="${TEST_NAME_BASE}-forbid-inconsistent-source-dir-between-runs"
+SOURCE_DIR_1="test-install-${CYLC_TEST_TIME_INIT}/${TEST_NAME_BASE}"
+mkdir -p "${PWD}/${SOURCE_DIR_1}"
+pushd "${SOURCE_DIR_1}" || exit 1
+touch flow.cylc
+
+run_ok "${TEST_NAME}" cylc install
+popd || exit 1
+SOURCE_DIR_2="test-install-${CYLC_TEST_TIME_INIT}2/${TEST_NAME_BASE}"
+mkdir -p "${PWD}/${SOURCE_DIR_2}"
+pushd "${SOURCE_DIR_2}" || exit 1
+touch flow.cylc
+run_fail "${TEST_NAME}" cylc install
+
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+WorkflowFilesError: Source directory between runs are not consistent
+__ERR__
+rm -rf "${PWD:?}/${SOURCE_DIR_1}" "${PWD:?}/${SOURCE_DIR_2}"
+rm -rf "${RUN_DIR:?}/${TEST_NAME_BASE}"
+popd || exit
# Test fail no suite source dir
TEST_NAME="${TEST_NAME_BASE}-nodir"
diff --git a/tests/functional/cylc-install/03-file-transfer.t b/tests/functional/cylc-install/03-file-transfer.t
index 03b2cb93587..524f1c3d638 100644
--- a/tests/functional/cylc-install/03-file-transfer.t
+++ b/tests/functional/cylc-install/03-file-transfer.t
@@ -62,11 +62,10 @@ ${RND_SUITE_RUNDIR}/
├── file1
├── file2
├── flow.cylc
-├── log
-│  └── install
-└── source -> ${RND_SUITE_SOURCE}
+└── log
+ └── install
-8 directories, 5 files
+7 directories, 5 files
__OUT__
contains_ok "${TEST_NAME}.stdout" <<__OUT__
@@ -98,11 +97,10 @@ ${RND_SUITE_RUNDIR}/
│  └── source -> ${RND_SUITE_SOURCE}
├── file1
├── flow.cylc
-├── log
-│  └── install
-└── source -> ${RND_SUITE_SOURCE}
+└── log
+ └── install
-6 directories, 2 files
+5 directories, 2 files
__OUT__
contains_ok "${TEST_NAME}.stdout" <<__OUT__
diff --git a/tests/functional/events/00-suite.t b/tests/functional/events/00-suite.t
index d4d1be4a6f2..3ce1a31d3f1 100755
--- a/tests/functional/events/00-suite.t
+++ b/tests/functional/events/00-suite.t
@@ -20,9 +20,11 @@
set_test_number 2
install_suite "${TEST_NAME_BASE}" 'suite'
-run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
+run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" \
+ -s "SUITE_SRC_DIR='${TEST_DIR}/${SUITE_NAME}'"
suite_run_ok "${TEST_NAME_BASE}-run" \
- cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
+ cylc run --reference-test --debug --no-detach "${SUITE_NAME}" \
+ -s "SUITE_SRC_DIR='${TEST_DIR}/${SUITE_NAME}'"
for SUFFIX in '' '-shutdown' '-startup' '-timeout'; do
purge "${SUITE_NAME}${SUFFIX}"
diff --git a/tests/functional/events/17-task-event-job-logs-retrieve-command.t b/tests/functional/events/17-task-event-job-logs-retrieve-command.t
index 8a4ee1f5137..e8ae721d976 100755
--- a/tests/functional/events/17-task-event-job-logs-retrieve-command.t
+++ b/tests/functional/events/17-task-event-job-logs-retrieve-command.t
@@ -30,15 +30,14 @@ create_test_global_config "" "
OPT_SET='-s GLOBALCFG=True'
install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-
-mkdir -p "${TEST_DIR}/${SUITE_NAME}/bin"
-cat >"${TEST_DIR}/${SUITE_NAME}/bin/my-rsync" <<'__BASH__'
+mkdir -p "${SUITE_RUN_DIR}/bin"
+cat >"${SUITE_RUN_DIR}/bin/my-rsync" <<'__BASH__'
#!/usr/bin/env bash
set -eu
echo "$@" >>"${CYLC_SUITE_LOG_DIR}/my-rsync.log"
exec rsync -a "$@"
__BASH__
-chmod +x "${TEST_DIR}/${SUITE_NAME}/bin/my-rsync"
+chmod +x "${SUITE_RUN_DIR}/bin/my-rsync"
# shellcheck disable=SC2086
run_ok "${TEST_NAME_BASE}-validate" \
diff --git a/tests/functional/events/suite/flow.cylc b/tests/functional/events/suite/flow.cylc
index 7b557366e0d..d0fc492de12 100644
--- a/tests/functional/events/suite/flow.cylc
+++ b/tests/functional/events/suite/flow.cylc
@@ -1,26 +1,29 @@
+#!Jinja2
+
[scheduling]
[[graph]]
R1 = "startup => timeout => shutdown"
+
[runtime]
[[common]]
- script = """
-cylc install --flow-name=$REG -C $DEF --no-run-name
-echo "Sub-suite log file is: $PWD/$LOG"
-if cylc run --debug --no-detach $REG > $LOG 2>&1; then
- echo "ERROR: sub-suite did not abort as planned"
- exit 1
-else
- if grep "$GREP" $LOG; then
- echo "Sub-suite aborted as planned"
- else
- echo "ERROR: sub-suite did not abort as planned"
- exit 1
- fi
-fi
-"""
platform = localhost
+ script = """
+ cylc install --flow-name=$REG -C $DEF --no-run-name
+ echo "Sub-suite log file is: $PWD/$LOG"
+ if cylc run --debug --no-detach $REG > $LOG 2>&1; then
+ echo "ERROR: sub-suite did not abort as planned"
+ exit 1
+ else
+ if grep "$GREP" $LOG; then
+ echo "Sub-suite aborted as planned"
+ else
+ echo "ERROR: sub-suite did not abort as planned"
+ exit 1
+ fi
+ fi
+ """
[[[environment]]]
- DEF = $CYLC_SUITE_DEF_PATH/hidden/${CYLC_TASK_NAME}
+ DEF = {{ SUITE_SRC_DIR }}/hidden/${CYLC_TASK_NAME}
REG = ${CYLC_SUITE_NAME}-${CYLC_TASK_NAME}
LOG = ${CYLC_TASK_NAME}.log
GREP = "ERROR - ${CYLC_TASK_NAME} EVENT HANDLER FAILED"
diff --git a/tests/functional/job-submission/12-tidy-submits-of-prev-run.t b/tests/functional/job-submission/12-tidy-submits-of-prev-run.t
index faa3fd06592..c26e4cc1ea7 100755
--- a/tests/functional/job-submission/12-tidy-submits-of-prev-run.t
+++ b/tests/functional/job-submission/12-tidy-submits-of-prev-run.t
@@ -28,8 +28,8 @@ LOGD1="$RUN_DIR/${SUITE_NAME}/log/job/1/t1/01"
LOGD2="$RUN_DIR/${SUITE_NAME}/log/job/1/t1/02"
exists_ok "${LOGD1}"
exists_ok "${LOGD2}"
-sed -i 's/script =.*$/script = true/' "flow.cylc"
-sed -i -n '1,/triggered off/p' "reference.log"
+sed -i 's/script =.*$/script = true/' "$RUN_DIR/$SUITE_NAME/flow.cylc"
+sed -i -n '1,/triggered off/p' "$RUN_DIR/$SUITE_NAME/reference.log"
suite_run_ok "${TEST_NAME_BASE}-run" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
exists_ok "${LOGD1}"
diff --git a/tests/functional/reload/17-graphing-change.t b/tests/functional/reload/17-graphing-change.t
index f0fbf0b3c4d..ee47585a9dd 100755
--- a/tests/functional/reload/17-graphing-change.t
+++ b/tests/functional/reload/17-graphing-change.t
@@ -31,7 +31,7 @@ run_ok "${TEST_NAME_BASE}-add-run" cylc run --debug --hold "${SUITE_NAME}"
# change the flow.cylc file
cp "${TEST_SOURCE_DIR}/graphing-change/flow-1.cylc" \
- "${TEST_DIR}/${SUITE_NAME}/flow.cylc"
+ "${RUN_DIR}/${SUITE_NAME}/flow.cylc"
# reload suite
run_ok "${TEST_NAME_BASE}-add-reload" cylc reload "${SUITE_NAME}"
@@ -46,7 +46,7 @@ grep_ok "Added task: 'one'" "${LOG_FILE}"
# change the flow.cylc file
cp "${TEST_SOURCE_DIR}/graphing-change/flow.cylc" \
- "${TEST_DIR}/${SUITE_NAME}/flow.cylc"
+ "${RUN_DIR}/${SUITE_NAME}/flow.cylc"
# reload suite
run_ok "${TEST_NAME_BASE}-remove-reload" cylc reload "${SUITE_NAME}"
@@ -61,7 +61,7 @@ grep_ok "Removed task: 'one'" "${LOG_FILE}"
# change the flow.cylc file
cp "${TEST_SOURCE_DIR}/graphing-change/flow-2.cylc" \
- "${TEST_DIR}/${SUITE_NAME}/flow.cylc"
+ "${RUN_DIR}/${SUITE_NAME}/flow.cylc"
cylc set-outputs "${SUITE_NAME}" foo.1
cylc set-outputs "${SUITE_NAME}" baz.1
diff --git a/tests/functional/restart/01-broadcast.t b/tests/functional/restart/01-broadcast.t
index 0f1b6501901..6d60070d9bd 100755
--- a/tests/functional/restart/01-broadcast.t
+++ b/tests/functional/restart/01-broadcast.t
@@ -23,8 +23,7 @@ fi
set_test_number 8
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" broadcast
-cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$TEST_DIR/${SUITE_NAME}/"
-export TEST_DIR
+cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$RUN_DIR/${SUITE_NAME}/"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
@@ -37,14 +36,14 @@ TEST_NAME="${TEST_NAME_BASE}-restart-run"
suite_run_ok "${TEST_NAME}" cylc restart --no-detach --abort-if-any-task-fails "${SUITE_NAME}"
#-------------------------------------------------------------------------------
grep_ok "send_a_broadcast_task|20130923T0000Z|1|1|succeeded" \
- "${TEST_DIR}/pre-restart-db"
-contains_ok "${TEST_DIR}/post-restart-db" <<'__DB_DUMP__'
+ "${SUITE_RUN_DIR}/pre-restart-db"
+contains_ok "${SUITE_RUN_DIR}/post-restart-db" <<'__DB_DUMP__'
send_a_broadcast_task|20130923T0000Z|1|1|succeeded
shutdown|20130923T0000Z|1|1|succeeded
__DB_DUMP__
"${TEST_SOURCE_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
- > "${TEST_DIR}/db"
-contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__'
+ > "${SUITE_RUN_DIR}/db"
+contains_ok "${RUN_DIR}/db" <<'__DB_DUMP__'
broadcast_task|20130923T0000Z|1|1|succeeded
finish|20130923T0000Z|1|1|succeeded
output_states|20130923T0000Z|1|1|succeeded
diff --git a/tests/functional/restart/broadcast/flow.cylc b/tests/functional/restart/broadcast/flow.cylc
index 0d509f26603..380e1ea7041 100644
--- a/tests/functional/restart/broadcast/flow.cylc
+++ b/tests/functional/restart/broadcast/flow.cylc
@@ -1,10 +1,11 @@
#!jinja2
-{%- set TEST_DIR = environ['TEST_DIR'] %}
+
[scheduler]
UTC mode = True
[[events]]
abort on timeout = True
timeout = PT1M
+
[scheduling]
initial cycle point = 20130923T00
final cycle point = 20130923T00
@@ -15,10 +16,15 @@
output_states => broadcast_task
broadcast_task => finish
"""
+
[runtime]
[[send_a_broadcast_task]]
script = """
- cylc broadcast -n broadcast_task -p $CYLC_TASK_CYCLE_POINT -s "[environment]MY_VALUE='something'" $CYLC_SUITE_NAME
+ cylc broadcast \
+ -n broadcast_task \
+ -p $CYLC_TASK_CYCLE_POINT \
+ -s "[environment]MY_VALUE='something'" \
+ $CYLC_SUITE_NAME
cylc broadcast -d $CYLC_SUITE_NAME
"""
[[[meta]]]
@@ -35,4 +41,5 @@
description = "Broadcast-recipient task (runs after restart)"
[[[environment]]]
MY_VALUE=nothing
+
{% include 'flow-runtime-restart.cylc' %}
diff --git a/tests/functional/restart/lib/flow-runtime-restart.cylc b/tests/functional/restart/lib/flow-runtime-restart.cylc
index 3b5c9a262de..811f6e81f82 100644
--- a/tests/functional/restart/lib/flow-runtime-restart.cylc
+++ b/tests/functional/restart/lib/flow-runtime-restart.cylc
@@ -5,7 +5,7 @@
for i in {0..10}; do
ctb-select-task-states \
"${CYLC_SUITE_RUN_DIR}" "${CYLC_TASK_NAME}" \
- > {{ TEST_DIR }}/$OUTPUT_SUFFIX-db && break
+ > "${CYLC_SUITE_RUN_DIR}/$OUTPUT_SUFFIX-db" && break
sleep 1
done
"""
diff --git a/tests/functional/shutdown/19-log-reference.t b/tests/functional/shutdown/19-log-reference.t
index ade641bf78b..adb1779ccfa 100755
--- a/tests/functional/shutdown/19-log-reference.t
+++ b/tests/functional/shutdown/19-log-reference.t
@@ -36,7 +36,7 @@ __FLOW_CONFIG__
#-------------------------------------------------------------------------------
suite_run_ok "${TEST_NAME_BASE}-run-reflog" \
cylc run --debug --no-detach --reference-log "${SUITE_NAME}"
-exists_ok 'reference.log'
+exists_ok "${HOME}/cylc-run/${SUITE_NAME}/reference.log"
suite_run_ok "${TEST_NAME_BASE}-run-reftest" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
#-------------------------------------------------------------------------------
From ba860e4431da67b17f73068dfea9b14864e64d4e Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Thu, 14 Jan 2021 14:00:52 +0000
Subject: [PATCH 08/13] Cylc install - get suite_source_dir adapted, fix
functional tests
Fix get_flow_file
Fix unit test
---
cylc/flow/etc/cylc-bash-completion | 2 +-
cylc/flow/scheduler.py | 52 +++----------
cylc/flow/scripts/graph.py | 9 +--
cylc/flow/scripts/validate.py | 2 +
cylc/flow/subprocctx.py | 4 +-
cylc/flow/suite_files.py | 73 +++++++++----------
cylc/flow/xtrigger_mgr.py | 7 +-
tests/flakyfunctional/restart/14-multicycle.t | 5 +-
.../restart/14-multicycle/flow.cylc | 2 +-
.../flakyfunctional/restart/21-task-elapsed.t | 4 +-
.../functional/cyclers/20-multidaily_local.t | 2 +-
tests/functional/cylc-clean/00-basic.t | 1 -
.../05-host-bool-override.t | 1 -
tests/functional/cylc-install/02-failures.t | 5 +-
.../cylc-install/03-file-transfer.t | 3 +
.../cylc-ping/05-check-keys-sharedfs.t | 5 +-
tests/functional/cylc-run/01-invalid-suite.t | 29 --------
.../cylc-run/{04-profiler.t => 01-profiler.t} | 0
.../functional/cylc-run/03-remote-run-host.t | 2 +-
tests/functional/cylc-search/00-basic.t | 1 +
.../03-clock-triggered-non-utc-mode.t | 1 +
.../17-task-event-job-logs-retrieve-command.t | 8 +-
tests/functional/events/47-long-output.t | 22 +++---
.../13-tidy-submits-of-prev-run-remote-host.t | 9 ++-
...s-of-prev-run-remote-host-with-shared-fs.t | 4 +-
tests/functional/lib/bash/test_header | 6 +-
.../reload/22-remove-task-cycling.t | 2 +-
tests/functional/restart/01-broadcast.t | 14 ++--
tests/functional/restart/02-failed.t | 11 ++-
tests/functional/restart/05-submit-failed.t | 9 +--
tests/functional/restart/06-succeeded.t | 9 +--
tests/functional/restart/07-waiting.t | 7 +-
.../restart/34-auto-restart-basic.t | 2 +-
.../restart/35-auto-restart-recovery.t | 2 +-
.../restart/37-auto-restart-delay.t | 2 +-
.../restart/41-auto-restart-local-jobs.t | 3 +-
.../restart/42-auto-restart-ping-pong.t | 4 +-
.../43-auto-restart-force-override-normal.t | 2 +-
tests/functional/restart/44-stop-point.t | 2 +-
tests/functional/restart/broadcast/flow.cylc | 11 +--
tests/functional/xtriggers/02-persistence.t | 5 +-
tests/unit/test_suite_files.py | 11 +--
42 files changed, 144 insertions(+), 211 deletions(-)
delete mode 100644 tests/functional/cylc-run/01-invalid-suite.t
rename tests/functional/cylc-run/{04-profiler.t => 01-profiler.t} (100%)
diff --git a/cylc/flow/etc/cylc-bash-completion b/cylc/flow/etc/cylc-bash-completion
index ceab4cacdbf..a278eceb471 100644
--- a/cylc/flow/etc/cylc-bash-completion
+++ b/cylc/flow/etc/cylc-bash-completion
@@ -38,7 +38,7 @@ _cylc() {
cur="${COMP_WORDS[COMP_CWORD]}"
sec="${COMP_WORDS[1]}"
opts="$(cylc scan -t name 2>/dev/null)"
- suite_cmds="broadcast|bcast|cat-log|check-versions|clean|compare|diff|dump|edit|ext-trigger|external-trigger|get-directory|get-suite-config|get-config|get-suite-version|get-cylc-version|graph|graph-diff|hold|insert|install|kill|list|log|ls|tui|ping|poll|print|release|unhold|reload|remove|report-timings|reset|restart|run|start|scan|search|grep|set-verbosity|show|set-outputs|stop|shutdown|single|suite-state|test-battery|trigger|validate|view|warranty"
+ suite_cmds="broadcast|bcast|cat-log|check-versions|clean|compare|diff|dump|edit|ext-trigger|external-trigger|get-suite-config|get-config|get-suite-version|get-cylc-version|graph|graph-diff|hold|insert|install|kill|list|log|ls|tui|ping|poll|print|release|unhold|reload|remove|report-timings|reset|restart|run|start|scan|search|grep|set-verbosity|show|set-outputs|stop|shutdown|single|suite-state|test-battery|trigger|validate|view|warranty"
if [[ ${COMP_CWORD} -eq 1 ]]; then
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index da6c5643e95..65ebcf31e93 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -23,7 +23,6 @@
import os
from queue import Empty, Queue
from shlex import quote
-from shutil import copytree, rmtree
from subprocess import Popen, PIPE, DEVNULL
import sys
from threading import Barrier
@@ -42,7 +41,7 @@
from cylc.flow.cycling.loader import get_point
from cylc.flow.data_store_mgr import DataStoreMgr, parse_job_item
from cylc.flow.exceptions import (
- CylcError, SuiteConfigError, PlatformLookupError, WorkflowFilesError
+ CylcError, SuiteConfigError, PlatformLookupError
)
import cylc.flow.flags
from cylc.flow.host_select import select_suite_host
@@ -260,7 +259,6 @@ def __init__(self, reg, options, is_restart=False):
)
# directory information
- self.suite_dir = suite_files.get_suite_source_dir(self.suite)
self.flow_file = suite_files.get_flow_file(self.suite)
self.suite_run_dir = get_workflow_run_dir(self.suite)
self.suite_work_dir = get_suite_run_work_dir(self.suite)
@@ -284,18 +282,12 @@ async def install(self):
* Copy Python files.
"""
- # Check if flow has been installed
- if not suite_files.is_installed(self.suite_run_dir):
- suite_files.register(self.suite, source=self.suite_run_dir)
# Install
- try:
- suite_files.get_suite_source_dir(self.suite)
- except WorkflowFilesError:
- # Source path is assumed to be the run directory
- suite_files.register(
- flow_name=self.suite,
- source=get_workflow_run_dir(
- self.suite))
+ source = suite_files.get_suite_source_dir(self.suite)
+ if source is None:
+ # register workflow
+ rund = get_workflow_run_dir(self.suite)
+ suite_files.register(self.suite, source=rund)
make_suite_run_tree(self.suite)
@@ -306,21 +298,12 @@ async def install(self):
extract_resources(
suite_files.get_suite_srv_dir(self.suite),
['etc/job.sh'])
-
- # Copy local python modules from source to run directory
+ # Add python dirs to sys.path
for sub_dir in ["python", os.path.join("lib", "python")]:
# TODO - eventually drop the deprecated "python" sub-dir.
- suite_py = os.path.join(self.suite_dir, sub_dir)
- if (os.path.realpath(self.suite_dir) !=
- os.path.realpath(self.suite_run_dir) and
- os.path.isdir(suite_py)):
- suite_run_py = os.path.join(self.suite_run_dir, sub_dir)
- try:
- rmtree(suite_run_py)
- except OSError:
- pass
- copytree(suite_py, suite_run_py)
- sys.path.append(os.path.join(self.suite_dir, sub_dir))
+ suite_py = os.path.join(self.suite_run_dir, sub_dir)
+ if os.path.isdir(suite_py):
+ sys.path.append(os.path.join(self.suite_run_dir, sub_dir))
async def initialise(self):
"""Initialise the components and sub-systems required to run the flow.
@@ -381,7 +364,6 @@ async def initialise(self):
proc_pool=self.proc_pool,
suite_run_dir=self.suite_run_dir,
suite_share_dir=self.suite_share_dir,
- suite_source_dir=self.suite_dir
)
self.task_events_mgr = TaskEventsManager(
@@ -424,20 +406,6 @@ async def configure(self):
pri_dao.select_suite_template_vars(self._load_template_vars)
pri_dao.execute_queued_items()
- # Copy local python modules from source to run directory
- for sub_dir in ["python", os.path.join("lib", "python")]:
- # TODO - eventually drop the deprecated "python" sub-dir.
- suite_py = os.path.join(self.suite_dir, sub_dir)
- if (os.path.realpath(self.suite_dir) !=
- os.path.realpath(self.suite_run_dir) and
- os.path.isdir(suite_py)):
- suite_run_py = os.path.join(self.suite_run_dir, sub_dir)
- try:
- rmtree(suite_run_py)
- except OSError:
- pass
- copytree(suite_py, suite_run_py)
-
self.profiler.log_memory("scheduler.py: before load_flow_file")
self.load_flow_file()
self.profiler.log_memory("scheduler.py: after load_flow_file")
diff --git a/cylc/flow/scripts/graph.py b/cylc/flow/scripts/graph.py
index 385b955400d..633172a1fb5 100755
--- a/cylc/flow/scripts/graph.py
+++ b/cylc/flow/scripts/graph.py
@@ -26,7 +26,7 @@
from cylc.flow.config import SuiteConfig
from cylc.flow.cycling.loader import get_point
-from cylc.flow.exceptions import UserInputError, WorkflowFilesError
+from cylc.flow.exceptions import UserInputError
from cylc.flow.option_parsers import CylcOptionParser as COP
from cylc.flow.suite_files import get_flow_file
from cylc.flow.templatevars import load_template_vars
@@ -168,10 +168,9 @@ def graph_inheritance(config):
def get_config(suite, opts, template_vars=None):
"""Return a SuiteConfig object for the provided reg / path."""
- try:
- flow_file = get_flow_file(suite)
- except WorkflowFilesError:
- # could not find suite, assume we have been given a path instead
+ flow_file = get_flow_file(suite)
+ # could not find suite, assume we have been given a path instead
+ if not flow_file:
flow_file = suite
suite = 'test'
return SuiteConfig(suite, flow_file, opts, template_vars=template_vars)
diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py
index ce14aea1c5d..3128c2b2004 100755
--- a/cylc/flow/scripts/validate.py
+++ b/cylc/flow/scripts/validate.py
@@ -86,11 +86,13 @@ def main(_, options, reg):
"""cylc validate CLI."""
profiler = Profiler(None, options.profile_mode)
profiler.start()
+
if not cylc.flow.flags.debug:
# for readability omit timestamps from logging unless in debug mode
for handler in LOG.handlers:
if isinstance(handler.formatter, CylcLogFormatter):
handler.formatter.configure(timestamp=False)
+
suite, flow_file = parse_suite_arg(options, reg)
cfg = SuiteConfig(
suite,
diff --git a/cylc/flow/subprocctx.py b/cylc/flow/subprocctx.py
index 4ffd63a6787..c1439c9fa09 100644
--- a/cylc/flow/subprocctx.py
+++ b/cylc/flow/subprocctx.py
@@ -142,12 +142,12 @@ def __init__(self, label, func_name, func_args, func_kwargs, intvl=None):
super(SubFuncContext, self).__init__(
'xtrigger-func', cmd=[], shell=False)
- def update_command(self, suite_source_dir):
+ def update_command(self, suite_run_dir):
"""Update the function wrap command after changes."""
self.cmd = ['cylc', 'function-run', self.func_name,
json.dumps(self.func_args),
json.dumps(self.func_kwargs),
- suite_source_dir]
+ suite_run_dir]
def get_signature(self):
"""Return the function call signature (as a string)."""
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index d78de09d173..5333803ddb1 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -18,6 +18,7 @@
import aiofiles
from enum import Enum
+import logging
import os
from pathlib import Path
from random import shuffle
@@ -44,13 +45,12 @@
get_platform, get_install_target_to_platforms_map)
from cylc.flow.hostuserutil import (
get_user,
- is_remote_host,
- is_remote_user
+ is_remote_host
)
from cylc.flow.remote import construct_ssh_cmd
from cylc.flow.suite_db_mgr import SuiteDatabaseManager
-from cylc.flow.unicode_rules import SuiteNameValidator
from cylc.flow.loggingutil import CylcLogFormatter
+from cylc.flow.unicode_rules import SuiteNameValidator
from cylc.flow.wallclock import get_current_time_string
@@ -362,35 +362,34 @@ def get_contact_file(reg):
get_suite_srv_dir(reg), SuiteFiles.Service.CONTACT)
-def get_flow_file(reg, suite_owner=None):
+def get_flow_file(reg):
"""Return the path of a suite's flow.cylc file."""
- return os.path.join(
- get_suite_source_dir(reg, suite_owner), SuiteFiles.FLOW_FILE)
+ flow_file = os.path.join(get_workflow_run_dir(reg), SuiteFiles.FLOW_FILE)
+ if os.path.exists(flow_file):
+ return flow_file
-def get_suite_source_dir(reg, suite_owner=None):
- """Return the source directory path of a suite.
-
- Will register un-registered suites located in the cylc run dir.
+def get_suite_source_dir(reg):
+ """Return the source directory path of a workflow.
"""
- srv_d = get_suite_srv_dir(reg, suite_owner)
- fname = os.path.join(get_workflow_run_dir(reg), SuiteFiles.SOURCE)
+ cwd = Path.cwd()
+ source_path = Path(
+ cwd,
+ SuiteFiles.Install.DIRNAME,
+ SuiteFiles.SOURCE)
+ alt_source_path = Path(
+ cwd.parent,
+ SuiteFiles.Install.DIRNAME,
+ SuiteFiles.SOURCE)
try:
- source = os.readlink(fname)
+ source = os.readlink(source_path)
except OSError:
- suite_d = os.path.dirname(srv_d)
- if os.path.exists(suite_d) and not is_remote_user(suite_owner):
- register(flow_name=reg, source=suite_d)
- return suite_d
- raise WorkflowFilesError(f"Suite not found: {reg}")
- else:
- if not os.path.isabs(source):
- source = os.path.normpath(os.path.join(srv_d, source))
- flow_file_path = os.path.join(source, SuiteFiles.FLOW_FILE)
- if not os.path.exists(flow_file_path):
- # suite exists but is probably using deprecated suite.rc
- register(flow_name=reg, source=source)
- return source
+ try:
+ source = os.readlink(alt_source_path)
+ except OSError:
+ source = None
+
+ return source
def get_suite_srv_dir(reg, suite_owner=None):
@@ -461,9 +460,8 @@ def parse_suite_arg(options, arg):
path = os.path.abspath(arg)
else:
name = arg
- try:
- path = get_flow_file(arg)
- except WorkflowFilesError:
+ path = get_flow_file(arg)
+ if not path:
arg = os.path.abspath(arg)
if os.path.isdir(arg):
path = os.path.join(arg, SuiteFiles.FLOW_FILE)
@@ -472,7 +470,7 @@ def parse_suite_arg(options, arg):
# Probably using deprecated suite.rc
path = os.path.join(arg, SuiteFiles.SUITE_RC)
if not os.path.exists(path):
- raise SuiteServiceFileError(
+ raise WorkflowFilesError(
f'no {SuiteFiles.FLOW_FILE} or '
f'{SuiteFiles.SUITE_RC} in {arg}')
else:
@@ -592,7 +590,7 @@ def init_clean(reg, opts):
reg (str): Workflow name.
opts (optparse.Values): CLI options object for cylc clean.
"""
- local_run_dir = Path(get_suite_run_dir(reg))
+ local_run_dir = Path(get_workflow_run_dir(reg))
try:
_clean_check(reg, local_run_dir)
except FileNotFoundError as exc:
@@ -623,7 +621,7 @@ def clean(reg):
Args:
reg (str): Workflow name.
"""
- run_dir = Path(get_suite_run_dir(reg))
+ run_dir = Path(get_workflow_run_dir(reg))
try:
_clean_check(reg, run_dir)
except FileNotFoundError as exc:
@@ -1044,7 +1042,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
raise WorkflowFilesError(
'Run name cannot be "_cylc-install".'
' Please choose another run name.')
- validate_source_dir(source)
+ validate_source_dir(source, flow_name)
run_path_base = Path(get_workflow_run_dir(flow_name)).expanduser()
relink = False
run_num = 0
@@ -1138,7 +1136,7 @@ def validate_flow_name(flow_name):
f'Workflow name cannot be an absolute path: {flow_name}')
-def validate_source_dir(source):
+def validate_source_dir(source, flow_name):
"""Ensure the source directory is valid.
Args:
@@ -1154,15 +1152,16 @@ def validate_source_dir(source):
path_to_check = Path(source, dir_)
if path_to_check.exists():
raise WorkflowFilesError(
- f'Installation failed. - {dir_} exists in source directory.')
+ f'{flow_name} installation failed. - {dir_} exists in source '
+ 'directory.')
cylc_run_dir = Path(
get_platform()['run directory'].replace('$HOME', '~')
).expanduser()
if os.path.abspath(os.path.realpath(cylc_run_dir)
) in os.path.abspath(os.path.realpath(source)):
raise WorkflowFilesError(
- f'Installation failed. Source directory should not be in'
- f' {cylc_run_dir}')
+ f'{flow_name} installation failed. Source directory should not be '
+ f'in {cylc_run_dir}')
def unlink_runN(run_n):
diff --git a/cylc/flow/xtrigger_mgr.py b/cylc/flow/xtrigger_mgr.py
index f5a84b08053..49913d113bc 100644
--- a/cylc/flow/xtrigger_mgr.py
+++ b/cylc/flow/xtrigger_mgr.py
@@ -94,7 +94,6 @@ class XtriggerManager:
proc_pool (SubProcPool): pool of Subprocesses
suite_run_dir (str): suite run directory
suite_share_dir (str): suite share directory
- suite_source_dir (str): suite source directory
"""
@@ -108,7 +107,6 @@ def __init__(
proc_pool: SubProcPool = None,
suite_run_dir: str = None,
suite_share_dir: str = None,
- suite_source_dir: str = None,
):
# Suite function and clock triggers by label.
self.functx_map = {}
@@ -121,6 +119,8 @@ def __init__(
# All trigger and clock signatures in the current task pool.
self.all_xtrig = []
+ self.suite_run_dir = suite_run_dir
+
self.pflag = False
# For function arg templating.
@@ -136,7 +136,6 @@ def __init__(
self.proc_pool = proc_pool
self.broadcast_mgr = broadcast_mgr
self.data_store_mgr = data_store_mgr
- self.suite_source_dir = suite_source_dir
@staticmethod
def validate_xtrigger(fname: str, fdir: str):
@@ -272,7 +271,7 @@ def get_xtrig_ctx(self, itask: TaskProxy, label: str) -> SubFuncContext:
kwargs[key] = val
ctx.func_args = args
ctx.func_kwargs = kwargs
- ctx.update_command(self.suite_source_dir)
+ ctx.update_command(self.suite_run_dir)
return ctx
def satisfy_xtriggers(self, itask: TaskProxy):
diff --git a/tests/flakyfunctional/restart/14-multicycle.t b/tests/flakyfunctional/restart/14-multicycle.t
index 895d762c2c2..a3ad448a4c7 100755
--- a/tests/flakyfunctional/restart/14-multicycle.t
+++ b/tests/flakyfunctional/restart/14-multicycle.t
@@ -23,7 +23,6 @@ fi
set_test_number 6
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-export TEST_DIR
#-------------------------------------------------------------------------------
run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
if ! command -v 'sqlite3' > /dev/null; then
@@ -38,7 +37,7 @@ suite_run_ok "${TEST_NAME_BASE}-restart-run" \
#-------------------------------------------------------------------------------
# The waiting tasks below have two parents and are spawned by the earlier
# intercycle dependencies.
-cmp_ok "${TEST_DIR}/pre-restart-db" <<'__DB_DUMP__'
+cmp_ok "${SUITE_RUN_DIR}/pre-restart-db" <<'__DB_DUMP__'
bar|20130923T0000Z|1|1|succeeded
bar|20130923T1200Z|1|1|succeeded
bar|20130924T0000Z|1|1|succeeded
@@ -51,7 +50,7 @@ foo|20130924T0000Z|1|1|succeeded
foo|20130924T1200Z|1|1|succeeded
foo|20130925T0000Z|0||waiting
__DB_DUMP__
-contains_ok "${TEST_DIR}/post-restart-db" <<'__DB_DUMP__'
+contains_ok "${SUITE_RUN_DIR}/post-restart-db" <<'__DB_DUMP__'
bar|20130923T0000Z|1|1|succeeded
bar|20130923T1200Z|1|1|succeeded
bar|20130924T0000Z|1|1|succeeded
diff --git a/tests/flakyfunctional/restart/14-multicycle/flow.cylc b/tests/flakyfunctional/restart/14-multicycle/flow.cylc
index 7427b1f4e54..ec5da378c23 100644
--- a/tests/flakyfunctional/restart/14-multicycle/flow.cylc
+++ b/tests/flakyfunctional/restart/14-multicycle/flow.cylc
@@ -29,7 +29,7 @@
sleep 5
ctb-select-task-states \
"${CYLC_SUITE_RUN_DIR}" "${CYLC_TASK_NAME}" \
- > {{ TEST_DIR }}/$OUTPUT_SUFFIX-db
+ > "${CYLC_SUITE_RUN_DIR}/$OUTPUT_SUFFIX-db"
"""
[[shutdown]]
inherit = OUTPUT
diff --git a/tests/flakyfunctional/restart/21-task-elapsed.t b/tests/flakyfunctional/restart/21-task-elapsed.t
index 636eec0e1d3..2c3008dee9b 100755
--- a/tests/flakyfunctional/restart/21-task-elapsed.t
+++ b/tests/flakyfunctional/restart/21-task-elapsed.t
@@ -37,7 +37,7 @@ for datum in data['tasks']:
assert isinstance(datum['meanElapsedTime'], float)
__PYTHON__
}
-
+cd "${SUITE_RUN_DIR}" || exit 1
run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
RUND="${RUN_DIR}/${SUITE_NAME}"
@@ -47,7 +47,7 @@ suite_run_ok "${TEST_NAME_BASE}-restart-1" \
cylc restart "${SUITE_NAME}" --stop-point=2028 --debug --no-detach
sed -n '/LOADING task run times/,+2{s/^.* INFO - //;s/[0-9]\(,\|$\)/%d\1/g;p}' \
"${RUND}/log/suite/log" >'restart-1.out'
-contains_ok 'restart-1.out' <<'__OUT__'
+contains_ok "restart-1.out" <<'__OUT__'
LOADING task run times
+ t2: %d,%d,%d,%d,%d
+ t1: %d,%d,%d,%d,%d
diff --git a/tests/functional/cyclers/20-multidaily_local.t b/tests/functional/cyclers/20-multidaily_local.t
index 6e87b504d38..62545da37ed 100755
--- a/tests/functional/cyclers/20-multidaily_local.t
+++ b/tests/functional/cyclers/20-multidaily_local.t
@@ -26,7 +26,7 @@ CURRENT_TZ_UTC_OFFSET="$(date +%z)"
if [[ $CURRENT_TZ_UTC_OFFSET == '+0000' ]]; then
CURRENT_TZ_UTC_OFFSET="Z"
fi
-sed -i "s/Z/$CURRENT_TZ_UTC_OFFSET/g" 'reference.log'
+sed -i "s/Z/$CURRENT_TZ_UTC_OFFSET/g" "${SUITE_RUN_DIR}/reference.log"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
diff --git a/tests/functional/cylc-clean/00-basic.t b/tests/functional/cylc-clean/00-basic.t
index dd98333ff6e..775d769f331 100644
--- a/tests/functional/cylc-clean/00-basic.t
+++ b/tests/functional/cylc-clean/00-basic.t
@@ -89,7 +89,6 @@ ${TEST_DIR}/${SYM_NAME}-run
|-- flow.cylc
|-- log -> ${TEST_DIR}/${SYM_NAME}-log/cylc-run/${SUITE_NAME}/log
|-- share -> ${TEST_DIR}/${SYM_NAME}-share/cylc-run/${SUITE_NAME}/share
- |-- source -> ${TEST_DIR}/${SUITE_NAME}
\`-- work -> ${TEST_DIR}/${SYM_NAME}-work/cylc-run/${SUITE_NAME}/work
${TEST_DIR}/${SYM_NAME}-share
\`-- cylc-run
diff --git a/tests/functional/cylc-get-site-config/05-host-bool-override.t b/tests/functional/cylc-get-site-config/05-host-bool-override.t
index 78556433dfe..80b9cb70e74 100644
--- a/tests/functional/cylc-get-site-config/05-host-bool-override.t
+++ b/tests/functional/cylc-get-site-config/05-host-bool-override.t
@@ -20,7 +20,6 @@
. "$(dirname "$0")/test_header"
set_test_number 2
-mkdir etc/
cat etc/global.cylc <<'__hi__'
[platforms]
[[desktop\d\d|laptop\d\d]]
diff --git a/tests/functional/cylc-install/02-failures.t b/tests/functional/cylc-install/02-failures.t
index 1723d8aa9d3..7f9cfa7a7d1 100644
--- a/tests/functional/cylc-install/02-failures.t
+++ b/tests/functional/cylc-install/02-failures.t
@@ -66,6 +66,7 @@ rm -rf "${PWD:?}/${SOURCE_DIR_1}" "${PWD:?}/${SOURCE_DIR_2}"
rm -rf "${RUN_DIR:?}/${TEST_NAME_BASE}"
popd || exit
+
# Test fail no suite source dir
TEST_NAME="${TEST_NAME_BASE}-nodir"
make_rnd_suite
@@ -112,7 +113,7 @@ for DIR in 'work' 'share' 'log' '_cylc-install'; do
mkdir ${DIR}
run_fail "${TEST_NAME}" cylc install
contains_ok "${TEST_NAME}.stderr" <<__ERR__
-WorkflowFilesError: Installation failed. - ${DIR} exists in source directory.
+WorkflowFilesError: ${RND_SUITE_NAME} installation failed. - ${DIR} exists in source directory.
__ERR__
purge_rnd_suite
popd || exit 1
@@ -125,7 +126,7 @@ mkdir -p "${RUN_DIR}/${BASE_NAME}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}" && cd "$
touch flow.cylc
run_fail "${TEST_NAME}" cylc install
contains_ok "${TEST_NAME}.stderr" <<__ERR__
-WorkflowFilesError: Installation failed. Source directory should not be in ${RUN_DIR}
+WorkflowFilesError: ${TEST_NAME} installation failed. Source directory should not be in ${RUN_DIR}
__ERR__
cd "${RUN_DIR}" || exit
rm -rf "${BASE_NAME}"
diff --git a/tests/functional/cylc-install/03-file-transfer.t b/tests/functional/cylc-install/03-file-transfer.t
index 524f1c3d638..7e7b72c6ef7 100644
--- a/tests/functional/cylc-install/03-file-transfer.t
+++ b/tests/functional/cylc-install/03-file-transfer.t
@@ -18,6 +18,9 @@
#------------------------------------------------------------------------------
# Test rsync of workflow installation
. "$(dirname "$0")/test_header"
+if ! command -v 'tree' >'/dev/null'; then
+ skip_all '"tree" command not available'
+fi
set_test_number 6
export RND_SUITE_NAME
diff --git a/tests/functional/cylc-ping/05-check-keys-sharedfs.t b/tests/functional/cylc-ping/05-check-keys-sharedfs.t
index 5113c13f7e3..b20cef43c33 100644
--- a/tests/functional/cylc-ping/05-check-keys-sharedfs.t
+++ b/tests/functional/cylc-ping/05-check-keys-sharedfs.t
@@ -20,7 +20,6 @@ export REQUIRE_PLATFORM='loc:remote fs:shared comms:tcp'
. "$(dirname "$0")/test_header"
set_test_number 4
-export CYLC_TEST_PLATFORM="$CYLC_TEST_PLATFORM_WSFS"
init_suite "${TEST_NAME_BASE}" <<'__FLOW_CYLC__'
#!jinja2
[scheduler]
@@ -43,7 +42,7 @@ RRUND="cylc-run/${SUITE_NAME}"
RSRVD="${RRUND}/.service"
poll_grep_suite_log 'Holding all waiting or queued tasks now'
SSH="$(cylc get-global-config -i "[platforms][$CYLC_TEST_PLATFORM]ssh command")"
-${SSH} "${CYLC_TEST_PLATFORM}" \
+${SSH} "${CYLC_TEST_HOST}" \
find "${RSRVD}" -type f -name "*key*"|awk -F/ '{print $NF}'|sort >'find.out'
cmp_ok 'find.out' <<'__OUT__'
client.key_secret
@@ -52,7 +51,7 @@ server.key
server.key_secret
__OUT__
cylc stop --max-polls=60 --interval=1 "${SUITE_NAME}"
-${SSH} "${CYLC_TEST_PLATFORM}" \
+${SSH} "${CYLC_TEST_HOST}" \
find "${RRUND}" -type f -name "*key*"|awk -F/ '{print $NF}'|sort >'find.out'
cmp_ok 'find.out' <<'__OUT__'
__OUT__
diff --git a/tests/functional/cylc-run/01-invalid-suite.t b/tests/functional/cylc-run/01-invalid-suite.t
deleted file mode 100644
index f5cf3cf7838..00000000000
--- a/tests/functional/cylc-run/01-invalid-suite.t
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
-# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#-------------------------------------------------------------------------------
-# Test that ``cylc-run`` does not create directories for invalid/not existent
-# suites. See https://github.com/cylc/cylc-flow/issues/3097 for more.
-. "$(dirname "$0")/test_header"
-#-------------------------------------------------------------------------------
-set_test_number 3
-#-------------------------------------------------------------------------------
-INVALID_SUITE_NAME="broken-parachute-8877-mp5"
-run_fail "${TEST_NAME_BASE}-run" cylc run "${INVALID_SUITE_NAME}"
-grep_ok "WorkflowFilesError: Suite not found: ${INVALID_SUITE_NAME}" "${TEST_NAME_BASE}-run.stderr"
-exists_fail "${HOME}/cylc-run/${INVALID_SUITE_NAME}"
-
-exit
diff --git a/tests/functional/cylc-run/04-profiler.t b/tests/functional/cylc-run/01-profiler.t
similarity index 100%
rename from tests/functional/cylc-run/04-profiler.t
rename to tests/functional/cylc-run/01-profiler.t
diff --git a/tests/functional/cylc-run/03-remote-run-host.t b/tests/functional/cylc-run/03-remote-run-host.t
index 4133f606bd1..8d14e099281 100644
--- a/tests/functional/cylc-run/03-remote-run-host.t
+++ b/tests/functional/cylc-run/03-remote-run-host.t
@@ -21,7 +21,7 @@ export REQUIRE_PLATFORM='loc:remote fs:shared runner:background'
set_test_number 2
# shellcheck disable=SC2016
-TEST_DIR="$HOME/cylc-run/" init_suite "${TEST_NAME_BASE}" <<< '
+init_suite "${TEST_NAME_BASE}" <<< '
# A total non-entity workflow - just something to run.
[scheduling]
initial cycle point = 2020
diff --git a/tests/functional/cylc-search/00-basic.t b/tests/functional/cylc-search/00-basic.t
index 2768c855b0b..03b5a4ec091 100644
--- a/tests/functional/cylc-search/00-basic.t
+++ b/tests/functional/cylc-search/00-basic.t
@@ -22,6 +22,7 @@ set_test_number 2
install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}"
+cd "${SUITE_RUN_DIR}" || exit 1
run_ok "${TEST_NAME}" cylc search "${SUITE_NAME}" 'initial cycle point'
cmp_ok "${TEST_NAME}.stdout" <<__OUT__
diff --git a/tests/functional/cylc-show/03-clock-triggered-non-utc-mode.t b/tests/functional/cylc-show/03-clock-triggered-non-utc-mode.t
index 2cf756f6136..e560727f3d2 100644
--- a/tests/functional/cylc-show/03-clock-triggered-non-utc-mode.t
+++ b/tests/functional/cylc-show/03-clock-triggered-non-utc-mode.t
@@ -22,6 +22,7 @@ set_test_number 3
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" clock-triggered-non-utc-mode
#-------------------------------------------------------------------------------
+cd "${SUITE_RUN_DIR}" || exit 1
TEST_SHOW_OUTPUT_PATH="$PWD/${TEST_NAME_BASE}-show.stdout"
TZ_OFFSET_EXTENDED=$(date +%:z | sed "/^%/d")
if [[ -z "${TZ_OFFSET_EXTENDED}" ]]; then
diff --git a/tests/functional/events/17-task-event-job-logs-retrieve-command.t b/tests/functional/events/17-task-event-job-logs-retrieve-command.t
index e8ae721d976..0e2a626f788 100755
--- a/tests/functional/events/17-task-event-job-logs-retrieve-command.t
+++ b/tests/functional/events/17-task-event-job-logs-retrieve-command.t
@@ -30,14 +30,14 @@ create_test_global_config "" "
OPT_SET='-s GLOBALCFG=True'
install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
-mkdir -p "${SUITE_RUN_DIR}/bin"
-cat >"${SUITE_RUN_DIR}/bin/my-rsync" <<'__BASH__'
+mkdir -p "${RUN_DIR}/${SUITE_NAME}/bin"
+cat >"${RUN_DIR}/${SUITE_NAME}/bin/my-rsync" <<'__BASH__'
#!/usr/bin/env bash
set -eu
echo "$@" >>"${CYLC_SUITE_LOG_DIR}/my-rsync.log"
exec rsync -a "$@"
__BASH__
-chmod +x "${SUITE_RUN_DIR}/bin/my-rsync"
+chmod +x "${RUN_DIR}/${SUITE_NAME}/bin/my-rsync"
# shellcheck disable=SC2086
run_ok "${TEST_NAME_BASE}-validate" \
@@ -47,7 +47,7 @@ suite_run_ok "${TEST_NAME_BASE}-run" \
cylc run --reference-test --debug --no-detach ${OPT_SET} \
-s "PLATFORM='${CYLC_TEST_PLATFORM}'" "${SUITE_NAME}"
-SUITE_LOG_D="$RUN_DIR/${SUITE_NAME}/log"
+SUITE_LOG_D="${RUN_DIR}/${SUITE_NAME}/log"
sed 's/^.* -v //' "${SUITE_LOG_D}/suite/my-rsync.log" >'my-rsync.log.edited'
OPT_HEAD='--include=/1 --include=/1/t1'
diff --git a/tests/functional/events/47-long-output.t b/tests/functional/events/47-long-output.t
index 1ebf8078cde..098f11e3f8e 100755
--- a/tests/functional/events/47-long-output.t
+++ b/tests/functional/events/47-long-output.t
@@ -41,32 +41,31 @@ init_suite "${TEST_NAME_BASE}" <<__FLOW_CONFIG__
[[[events]]]
succeeded handler = cat "${CYLC_REPO_DIR}/COPYING" "${CYLC_REPO_DIR}/COPYING" "${CYLC_REPO_DIR}/COPYING" && echo
__FLOW_CONFIG__
+cd "$SUITE_RUN_DIR" || exit 1
cat >'reference.log' <<'__REFLOG__'
Initial point: 1
Final point: 1
[t1.1] -triggered off []
__REFLOG__
-
run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
suite_run_ok "${TEST_NAME_BASE}-run" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-cylc cat-log "${SUITE_NAME}" >'log'
-sed -n 's/^.*\(GNU GENERAL PUBLIC LICENSE\)/\1/p' 'log' >'log-1'
+cylc cat-log "${SUITE_NAME}" >'catlog'
+sed -n 's/^.*\(GNU GENERAL PUBLIC LICENSE\)/\1/p' 'catlog' >'log-1'
contains_ok 'log-1' <<'__LOG__'
GNU GENERAL PUBLIC LICENSE
GNU GENERAL PUBLIC LICENSE
GNU GENERAL PUBLIC LICENSE
__LOG__
run_ok "log-event-handler-00-out" \
- grep -qF "[(('event-handler-00', 'succeeded'), 1) out]" 'log'
+ grep -qF "[(('event-handler-00', 'succeeded'), 1) out]" 'catlog'
run_ok "log-event-handler-ret-code" \
- grep -qF "[(('event-handler-00', 'succeeded'), 1) ret_code] 0" 'log'
+ grep -qF "[(('event-handler-00', 'succeeded'), 1) ret_code] 0" 'catlog'
purge
-# Forcibly remove log directory
-rm -rf "${TEST_DIR}/${SUITE_NAME}/log"
+
# REPEAT: Long STDERR output
init_suite "${TEST_NAME_BASE}" <<__FLOW_CONFIG__
[scheduling]
@@ -78,6 +77,7 @@ init_suite "${TEST_NAME_BASE}" <<__FLOW_CONFIG__
[[[events]]]
succeeded handler = cat "${CYLC_REPO_DIR}/COPYING" "${CYLC_REPO_DIR}/COPYING" "${CYLC_REPO_DIR}/COPYING" >&2 && echo
__FLOW_CONFIG__
+cd "${SUITE_RUN_DIR}" || exit 1
cat >'reference.log' <<'__REFLOG__'
Initial point: 1
Final point: 1
@@ -89,17 +89,17 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
suite_run_ok "${TEST_NAME_BASE}-run" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-cylc cat-log "${SUITE_NAME}" >'log'
-sed -n 's/^.*\(GNU GENERAL PUBLIC LICENSE\)/\1/p' 'log' >'log-1'
+cylc cat-log "${SUITE_NAME}" >'catlog'
+sed -n 's/^.*\(GNU GENERAL PUBLIC LICENSE\)/\1/p' 'catlog' >'log-1'
contains_ok 'log-1' <<'__LOG__'
GNU GENERAL PUBLIC LICENSE
GNU GENERAL PUBLIC LICENSE
GNU GENERAL PUBLIC LICENSE
__LOG__
run_ok "log-event-handler-00-err" \
- grep -qF "[(('event-handler-00', 'succeeded'), 1) err]" 'log'
+ grep -qF "[(('event-handler-00', 'succeeded'), 1) err]" 'catlog'
run_ok "log-event-handler-00-ret-code" \
- grep -qF "[(('event-handler-00', 'succeeded'), 1) ret_code] 0" 'log'
+ grep -qF "[(('event-handler-00', 'succeeded'), 1) ret_code] 0" 'catlog'
purge
diff --git a/tests/functional/job-submission/13-tidy-submits-of-prev-run-remote-host.t b/tests/functional/job-submission/13-tidy-submits-of-prev-run-remote-host.t
index 757c46f500b..8206ee162ea 100755
--- a/tests/functional/job-submission/13-tidy-submits-of-prev-run-remote-host.t
+++ b/tests/functional/job-submission/13-tidy-submits-of-prev-run-remote-host.t
@@ -29,8 +29,8 @@ suite_run_ok "${TEST_NAME_BASE}-run" \
-s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'"
RLOGD1="cylc-run/${SUITE_NAME}/log/job/1/t1/01"
RLOGD2="cylc-run/${SUITE_NAME}/log/job/1/t1/02"
-LOGD1="$RUN_DIR/${SUITE_NAME}/log/job/1/t1/01"
-LOGD2="$RUN_DIR/${SUITE_NAME}/log/job/1/t1/02"
+LOGD1="${RUN_DIR}/${SUITE_NAME}/log/job/1/t1/01"
+LOGD2="${RUN_DIR}/${SUITE_NAME}/log/job/1/t1/02"
SSH='ssh -n -oBatchMode=yes -oConnectTimeout=5'
# shellcheck disable=SC2086
@@ -39,8 +39,9 @@ run_ok "exists-rlogd1" ${SSH} "${CYLC_TEST_HOST}" test -e "${RLOGD1}"
run_ok "exists-rlogd2" ${SSH} "${CYLC_TEST_HOST}" test -e "${RLOGD2}"
exists_ok "${LOGD1}"
exists_ok "${LOGD2}"
-sed -i 's/script =.*$/script = true/' "flow.cylc"
-sed -i -n '1,/triggered off/p' "reference.log"
+sed -i 's/script =.*$/script = true/' "${RUN_DIR}/${SUITE_NAME}/flow.cylc"
+sed -i -n '1,/triggered off/p' "${RUN_DIR}/${SUITE_NAME}/reference.log"
+
suite_run_ok "${TEST_NAME_BASE}-run" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}" \
-s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'"
diff --git a/tests/functional/job-submission/14-tidy-submits-of-prev-run-remote-host-with-shared-fs.t b/tests/functional/job-submission/14-tidy-submits-of-prev-run-remote-host-with-shared-fs.t
index 997e1505e77..490fdc116f8 100755
--- a/tests/functional/job-submission/14-tidy-submits-of-prev-run-remote-host-with-shared-fs.t
+++ b/tests/functional/job-submission/14-tidy-submits-of-prev-run-remote-host-with-shared-fs.t
@@ -31,8 +31,8 @@ LOGD1="$RUN_DIR/${SUITE_NAME}/log/job/1/t1/01"
LOGD2="$RUN_DIR/${SUITE_NAME}/log/job/1/t1/02"
exists_ok "${LOGD1}"
exists_ok "${LOGD2}"
-sed -i 's/script =.*$/script = true/' "flow.cylc"
-sed -i -n '1,/triggered off/p' "reference.log"
+sed -i 's/script =.*$/script = true/' "${SUITE_RUN_DIR}/flow.cylc"
+sed -i -n '1,/triggered off/p' "${SUITE_RUN_DIR}/reference.log"
suite_run_ok "${TEST_NAME_BASE}-run" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}" \
-s "CYLC_TEST_PLATFORM='${CYLC_TEST_PLATFORM}'"
diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header
index 0db52028b93..8617a6688f9 100644
--- a/tests/functional/lib/bash/test_header
+++ b/tests/functional/lib/bash/test_header
@@ -424,8 +424,8 @@ init_suite() {
SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}"
mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
cat "${FLOW_CONFIG}" >"${TEST_DIR}/${SUITE_NAME}/flow.cylc"
- cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}"
cd "${TEST_DIR}/${SUITE_NAME}"
+ cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}"
}
install_suite() {
@@ -435,8 +435,8 @@ install_suite() {
SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}"
mkdir -p "${TEST_DIR}/${SUITE_NAME}/" # make source dir
cp -r "${TEST_SOURCE_DIR}/${TEST_SOURCE_BASE}/"* "${TEST_DIR}/${SUITE_NAME}/"
- cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}"
- cd "${TEST_DIR}/${SUITE_NAME}/"
+ cd "${TEST_DIR}/${SUITE_NAME}"
+ cylc install --no-run-name --flow-name="${SUITE_NAME}" --directory="${TEST_DIR}/${SUITE_NAME}"
}
log_scan () {
diff --git a/tests/functional/reload/22-remove-task-cycling.t b/tests/functional/reload/22-remove-task-cycling.t
index e82224e813f..b6d4a181388 100644
--- a/tests/functional/reload/22-remove-task-cycling.t
+++ b/tests/functional/reload/22-remove-task-cycling.t
@@ -50,7 +50,7 @@ $(declare -f poll_grep)
# Remove bar and tell the server to reload.
if (( CYLC_TASK_CYCLE_POINT == CYLC_SUITE_INITIAL_CYCLE_POINT )); then
- sed -i 's/^.*remove*$//g' "\${CYLC_SUITE_DEF_PATH}/flow.cylc"
+ sed -i 's/^.*remove*$//g' "\${CYLC_SUITE_RUN_DIR}/flow.cylc"
cylc reload "\${CYLC_SUITE_NAME}"
poll_grep -F 'Reload complete' "\${CYLC_SUITE_RUN_DIR}/log/suite/log"
# kill the long-running orphaned bar task.
diff --git a/tests/functional/restart/01-broadcast.t b/tests/functional/restart/01-broadcast.t
index 6d60070d9bd..30d0653c257 100755
--- a/tests/functional/restart/01-broadcast.t
+++ b/tests/functional/restart/01-broadcast.t
@@ -22,18 +22,18 @@ fi
#-------------------------------------------------------------------------------
set_test_number 8
#-------------------------------------------------------------------------------
-install_suite "${TEST_NAME_BASE}" broadcast
-cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$RUN_DIR/${SUITE_NAME}/"
+install_suite "${TEST_NAME_BASE}" 'broadcast'
+cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "${SUITE_RUN_DIR}/"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
cmp_ok "${TEST_NAME}.stderr" <'/dev/null'
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-run"
-suite_run_ok "${TEST_NAME}" cylc run --no-detach --abort-if-any-task-fails "${SUITE_NAME}"
+suite_run_ok "${TEST_NAME}" cylc run --debug --no-detach --abort-if-any-task-fails "${SUITE_NAME}"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-restart-run"
-suite_run_ok "${TEST_NAME}" cylc restart --no-detach --abort-if-any-task-fails "${SUITE_NAME}"
+suite_run_ok "${TEST_NAME}" cylc restart --no-detach --debug --abort-if-any-task-fails "${SUITE_NAME}"
#-------------------------------------------------------------------------------
grep_ok "send_a_broadcast_task|20130923T0000Z|1|1|succeeded" \
"${SUITE_RUN_DIR}/pre-restart-db"
@@ -41,9 +41,9 @@ contains_ok "${SUITE_RUN_DIR}/post-restart-db" <<'__DB_DUMP__'
send_a_broadcast_task|20130923T0000Z|1|1|succeeded
shutdown|20130923T0000Z|1|1|succeeded
__DB_DUMP__
-"${TEST_SOURCE_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
- > "${SUITE_RUN_DIR}/db"
-contains_ok "${RUN_DIR}/db" <<'__DB_DUMP__'
+"${SUITE_RUN_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
+ > "${TEST_DIR}/db"
+contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__'
broadcast_task|20130923T0000Z|1|1|succeeded
finish|20130923T0000Z|1|1|succeeded
output_states|20130923T0000Z|1|1|succeeded
diff --git a/tests/functional/restart/02-failed.t b/tests/functional/restart/02-failed.t
index 6372eb1f65f..dd63af3a4b3 100755
--- a/tests/functional/restart/02-failed.t
+++ b/tests/functional/restart/02-failed.t
@@ -23,8 +23,7 @@ fi
set_test_number 7
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" 'failed'
-cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$TEST_DIR/${SUITE_NAME}/"
-export TEST_DIR
+cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "${RUN_DIR}/${SUITE_NAME}/"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
@@ -37,14 +36,14 @@ TEST_NAME="${TEST_NAME_BASE}-restart-run"
suite_run_ok "${TEST_NAME}" cylc restart --debug --no-detach "${SUITE_NAME}"
#-------------------------------------------------------------------------------
grep_ok "failed_task|20130923T0000Z|1|1|failed" \
- "${TEST_DIR}/pre-restart-db"
-contains_ok "${TEST_DIR}/post-restart-db" <<'__DB_DUMP__'
+ "${RUN_DIR}/${SUITE_NAME}/pre-restart-db"
+contains_ok "${RUN_DIR}/${SUITE_NAME}/post-restart-db" <<'__DB_DUMP__'
failed_task|20130923T0000Z|1|1|failed
shutdown|20130923T0000Z|1|1|succeeded
__DB_DUMP__
"${TEST_SOURCE_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
- > "${TEST_DIR}/db"
-contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__'
+ > "${RUN_DIR}/${SUITE_NAME}/db"
+contains_ok "${RUN_DIR}/${SUITE_NAME}/db" <<'__DB_DUMP__'
failed_task|20130923T0000Z|1|1|failed
finish|20130923T0000Z|1|1|succeeded
output_states|20130923T0000Z|1|1|succeeded
diff --git a/tests/functional/restart/05-submit-failed.t b/tests/functional/restart/05-submit-failed.t
index 8b9ea7cd6ec..9929bfa3c2e 100755
--- a/tests/functional/restart/05-submit-failed.t
+++ b/tests/functional/restart/05-submit-failed.t
@@ -31,8 +31,7 @@ create_test_global_config "
"
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" 'submit-failed'
-cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$TEST_DIR/${SUITE_NAME}/"
-export TEST_DIR
+cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$RUN_DIR/${SUITE_NAME}/"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
@@ -45,11 +44,11 @@ TEST_NAME="${TEST_NAME_BASE}-restart"
suite_run_ok "${TEST_NAME}" cylc restart --debug --no-detach "${SUITE_NAME}"
#-------------------------------------------------------------------------------
grep_ok "submit_failed_task|20130923T0000Z|1|1|submit-failed" \
- "${TEST_DIR}/pre-restart-db"
-contains_ok "${TEST_DIR}/post-restart-db" <<'__DB_DUMP__'
+ "${SUITE_RUN_DIR}/pre-restart-db"
+contains_ok "${SUITE_RUN_DIR}/post-restart-db" <<'__DB_DUMP__'
submit_failed_task|20130923T0000Z|1|1|submit-failed
__DB_DUMP__
-"${TEST_SOURCE_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
+"${SUITE_RUN_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
> "${TEST_DIR}/db"
contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__'
submit_failed_task|20130923T0000Z|1|1|submit-failed
diff --git a/tests/functional/restart/06-succeeded.t b/tests/functional/restart/06-succeeded.t
index 9abc7c6f34d..78fcbcbfc76 100755
--- a/tests/functional/restart/06-succeeded.t
+++ b/tests/functional/restart/06-succeeded.t
@@ -23,8 +23,7 @@ fi
set_test_number 7
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" 'succeeded'
-cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$TEST_DIR/${SUITE_NAME}/"
-export TEST_DIR
+cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "${SUITE_RUN_DIR}/"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
@@ -37,12 +36,12 @@ TEST_NAME=${TEST_NAME_BASE}-restart-run
suite_run_ok "${TEST_NAME}" cylc restart --debug --no-detach "${SUITE_NAME}"
#-------------------------------------------------------------------------------
grep_ok "succeeded_task|20130923T0000Z|1|1|succeeded" \
- "${TEST_DIR}/pre-restart-db"
-contains_ok "${TEST_DIR}/post-restart-db" <<'__DB_DUMP__'
+ "${SUITE_RUN_DIR}/pre-restart-db"
+contains_ok "${SUITE_RUN_DIR}/post-restart-db" <<'__DB_DUMP__'
shutdown|20130923T0000Z|1|1|succeeded
succeeded_task|20130923T0000Z|1|1|succeeded
__DB_DUMP__
-"${TEST_SOURCE_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
+"${SUITE_RUN_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
> "${TEST_DIR}/db"
contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__'
finish|20130923T0000Z|1|1|succeeded
diff --git a/tests/functional/restart/07-waiting.t b/tests/functional/restart/07-waiting.t
index 12ca7d34605..6d9ed94744b 100755
--- a/tests/functional/restart/07-waiting.t
+++ b/tests/functional/restart/07-waiting.t
@@ -26,8 +26,7 @@ fi
set_test_number 6
#-------------------------------------------------------------------------------
install_suite "${TEST_NAME_BASE}" 'waiting'
-cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$TEST_DIR/${SUITE_NAME}/"
-export TEST_DIR
+cp "$TEST_SOURCE_DIR/lib/flow-runtime-restart.cylc" "$RUN_DIR/${SUITE_NAME}/"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
@@ -39,10 +38,10 @@ suite_run_ok "${TEST_NAME}" cylc run --debug --no-detach "${SUITE_NAME}"
TEST_NAME="${TEST_NAME_BASE}-restart-run"
suite_run_ok "${TEST_NAME}" cylc restart --debug --no-detach "${SUITE_NAME}"
#-------------------------------------------------------------------------------
-contains_ok "${TEST_DIR}/post-restart-db" <<'__DB_DUMP__'
+contains_ok "$SUITE_RUN_DIR/post-restart-db" <<'__DB_DUMP__'
shutdown|20130923T0000Z|1|1|succeeded
__DB_DUMP__
-"${TEST_SOURCE_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
+"${SUITE_RUN_DIR}/bin/ctb-select-task-states" "${SUITE_RUN_DIR}" \
> "${TEST_DIR}/db"
contains_ok "${TEST_DIR}/db" <<'__DB_DUMP__'
finish|20130923T0000Z|1|1|succeeded
diff --git a/tests/functional/restart/34-auto-restart-basic.t b/tests/functional/restart/34-auto-restart-basic.t
index c9ba92a1f89..432847ea379 100644
--- a/tests/functional/restart/34-auto-restart-basic.t
+++ b/tests/functional/restart/34-auto-restart-basic.t
@@ -38,7 +38,7 @@ BASE_GLOBAL_CONFIG="
available = localhost, ${CYLC_TEST_HOST}"
TEST_NAME="${TEST_NAME_BASE}"
-TEST_DIR="$HOME/cylc-run/" init_suite "${TEST_NAME}" - <<'__FLOW_CONFIG__'
+init_suite "${TEST_NAME}" - <<'__FLOW_CONFIG__'
[task parameters]
foo = 1..25
[scheduling]
diff --git a/tests/functional/restart/35-auto-restart-recovery.t b/tests/functional/restart/35-auto-restart-recovery.t
index ea609228b9e..7ca8b59a143 100644
--- a/tests/functional/restart/35-auto-restart-recovery.t
+++ b/tests/functional/restart/35-auto-restart-recovery.t
@@ -36,7 +36,7 @@ BASE_GLOBAL_CONFIG="
available = localhost, ${CYLC_TEST_HOST}"
TEST_NAME="${TEST_NAME_BASE}"
-TEST_DIR="$HOME/cylc-run/" init_suite "${TEST_NAME}" <<< '
+init_suite "${TEST_NAME}" <<< '
[scheduling]
[[graph]]
R1 = foo
diff --git a/tests/functional/restart/37-auto-restart-delay.t b/tests/functional/restart/37-auto-restart-delay.t
index c51bda701b1..be9a49f5a76 100644
--- a/tests/functional/restart/37-auto-restart-delay.t
+++ b/tests/functional/restart/37-auto-restart-delay.t
@@ -41,7 +41,7 @@ BASE_GLOBAL_CONFIG="
"
#-------------------------------------------------------------------------------
# Test the delayed restart feature
-TEST_DIR="$HOME/cylc-run/" init_suite "${TEST_NAME_BASE}" <<< '
+init_suite "${TEST_NAME_BASE}" <<< '
[scheduler]
UTC mode = True
[scheduling]
diff --git a/tests/functional/restart/41-auto-restart-local-jobs.t b/tests/functional/restart/41-auto-restart-local-jobs.t
index 8f4f6990d27..d8032c694b0 100644
--- a/tests/functional/restart/41-auto-restart-local-jobs.t
+++ b/tests/functional/restart/41-auto-restart-local-jobs.t
@@ -36,7 +36,7 @@ BASE_GLOBAL_CONFIG="
timeout = PT2M
"
-TEST_DIR="$HOME/cylc-run/" init_suite "${TEST_NAME_BASE}" <<< '
+init_suite "${TEST_NAME_BASE}" <<< '
[scheduling]
[[graph]]
R1 = foo => bar
@@ -46,6 +46,7 @@ TEST_DIR="$HOME/cylc-run/" init_suite "${TEST_NAME_BASE}" <<< '
[[bar]]
script = sleep 60
'
+cd "${SUITE_RUN_DIR}" || exit 1
create_test_global_config '' "
${BASE_GLOBAL_CONFIG}
diff --git a/tests/functional/restart/42-auto-restart-ping-pong.t b/tests/functional/restart/42-auto-restart-ping-pong.t
index 1f3d77d69ce..67e58c9946d 100644
--- a/tests/functional/restart/42-auto-restart-ping-pong.t
+++ b/tests/functional/restart/42-auto-restart-ping-pong.t
@@ -35,7 +35,7 @@ BASE_GLOBAL_CONFIG='
timeout = PT2M
'
-TEST_DIR="$HOME/cylc-run/" init_suite "${TEST_NAME_BASE}" <<< '
+init_suite "${TEST_NAME_BASE}" <<< '
[scheduling]
initial cycle point = 2000
final cycle point = 9999 # test cylc/cylc-flow/issues/2799
@@ -45,7 +45,7 @@ TEST_DIR="$HOME/cylc-run/" init_suite "${TEST_NAME_BASE}" <<< '
[[root]]
script = sleep 5
'
-
+cd "${SUITE_RUN_DIR}" || exit 1
stuck_in_the_middle() {
# swap the condemned host forcing the suite to jump ship
local temp="${JOKERS}"
diff --git a/tests/functional/restart/43-auto-restart-force-override-normal.t b/tests/functional/restart/43-auto-restart-force-override-normal.t
index e7968e60f1c..5bddd9b74e5 100644
--- a/tests/functional/restart/43-auto-restart-force-override-normal.t
+++ b/tests/functional/restart/43-auto-restart-force-override-normal.t
@@ -35,7 +35,7 @@ BASE_GLOBAL_CONFIG='
timeout = PT2M
'
-TEST_DIR="$HOME/cylc-run/" init_suite "${TEST_NAME_BASE}" <<< '
+init_suite "${TEST_NAME_BASE}" <<< '
[scheduler]
[[events]]
[scheduling]
diff --git a/tests/functional/restart/44-stop-point.t b/tests/functional/restart/44-stop-point.t
index 9f9096a2d06..2272ad57cd4 100644
--- a/tests/functional/restart/44-stop-point.t
+++ b/tests/functional/restart/44-stop-point.t
@@ -60,7 +60,7 @@ case "${CYLC_TASK_CYCLE_POINT}" in
cylc stop "${CYLC_SUITE_NAME}"
:;;
2016)
- sed -i 's/\(final cycle point =\) 2024/\1 2025/' "${CYLC_SUITE_DEF_PATH}/flow.cylc"
+ sed -i 's/\(final cycle point =\) 2024/\1 2025/' "${CYLC_SUITE_RUN_DIR}/flow.cylc"
cylc reload "${CYLC_SUITE_NAME}"
cylc__job__poll_grep_suite_log "Reload completed"
:;;
diff --git a/tests/functional/restart/broadcast/flow.cylc b/tests/functional/restart/broadcast/flow.cylc
index 380e1ea7041..0d509f26603 100644
--- a/tests/functional/restart/broadcast/flow.cylc
+++ b/tests/functional/restart/broadcast/flow.cylc
@@ -1,11 +1,10 @@
#!jinja2
-
+{%- set TEST_DIR = environ['TEST_DIR'] %}
[scheduler]
UTC mode = True
[[events]]
abort on timeout = True
timeout = PT1M
-
[scheduling]
initial cycle point = 20130923T00
final cycle point = 20130923T00
@@ -16,15 +15,10 @@
output_states => broadcast_task
broadcast_task => finish
"""
-
[runtime]
[[send_a_broadcast_task]]
script = """
- cylc broadcast \
- -n broadcast_task \
- -p $CYLC_TASK_CYCLE_POINT \
- -s "[environment]MY_VALUE='something'" \
- $CYLC_SUITE_NAME
+ cylc broadcast -n broadcast_task -p $CYLC_TASK_CYCLE_POINT -s "[environment]MY_VALUE='something'" $CYLC_SUITE_NAME
cylc broadcast -d $CYLC_SUITE_NAME
"""
[[[meta]]]
@@ -41,5 +35,4 @@
description = "Broadcast-recipient task (runs after restart)"
[[[environment]]]
MY_VALUE=nothing
-
{% include 'flow-runtime-restart.cylc' %}
diff --git a/tests/functional/xtriggers/02-persistence.t b/tests/functional/xtriggers/02-persistence.t
index 8d9ddf5159f..257b3c86bd4 100644
--- a/tests/functional/xtriggers/02-persistence.t
+++ b/tests/functional/xtriggers/02-persistence.t
@@ -31,8 +31,9 @@ set_test_number 6
install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
# Install the succeeding xtrigger function.
+cd "${SUITE_RUN_DIR}" || exit 1
mkdir -p 'lib/python'
-cp "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/faker_succ.py" 'lib/python/faker.py'
+cp "${SUITE_RUN_DIR}/faker_succ.py" 'lib/python/faker.py'
# Validate the test suite.
run_ok "${TEST_NAME_BASE}-val" cylc val --debug "${SUITE_NAME}"
@@ -46,7 +47,7 @@ cylc cat-log "${SUITE_NAME}" 'foo.2010' >'foo.2010.out'
grep_ok 'NAME is bob' 'foo.2010.out'
# Replace the xtrigger function with one that will fail if called again.
-cp "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/faker_fail.py" 'lib/python/faker.py'
+cp "${SUITE_RUN_DIR}/faker_fail.py" 'lib/python/faker.py'
# Validate again (with the new xtrigger function).
run_ok "${TEST_NAME_BASE}-val2" cylc val --debug "${SUITE_NAME}"
diff --git a/tests/unit/test_suite_files.py b/tests/unit/test_suite_files.py
index 79dddaecf96..3b2f9a6eebf 100644
--- a/tests/unit/test_suite_files.py
+++ b/tests/unit/test_suite_files.py
@@ -172,9 +172,10 @@ def test_rundir_children_that_contain_workflows_raise_error(
@pytest.mark.parametrize(
- 'reg, expected_err',
- [('foo/bar/', None),
- ('/foo/bar', SuiteServiceFileError)]
+ 'reg, expected_err, expected_msg',
+ [('foo/bar/', None, None),
+ ('/foo/bar', SuiteServiceFileError, "cannot be an absolute path"),
+ ('$HOME/alone', SuiteServiceFileError, "invalid suite name")]
)
def test_validate_reg(reg, expected_err, expected_msg):
if expected_err:
@@ -270,7 +271,7 @@ def test_init_clean_ok(
mocked_remote_clean = mock.Mock()
monkeypatch.setattr('cylc.flow.suite_files.remote_clean',
mocked_remote_clean)
- monkeypatch.setattr('cylc.flow.suite_files.get_suite_run_dir',
+ monkeypatch.setattr('cylc.flow.suite_files.get_workflow_run_dir',
lambda x: tmp_path.joinpath('cylc-run', x))
_get_platforms_from_db = suite_files.get_platforms_from_db
@@ -427,7 +428,7 @@ def test_clean_broken_symlink_run_dir(monkeypatch, tmp_path):
run_dir.symlink_to(target)
target.rmdir()
- monkeypatch.setattr('cylc.flow.suite_files.get_suite_run_dir',
+ monkeypatch.setattr('cylc.flow.suite_files.get_workflow_run_dir',
lambda x: tmp_path.joinpath('cylc-run', x))
suite_files.clean(reg)
From 6401d78b6a01325f0fddf88b00c5accbd46670d2 Mon Sep 17 00:00:00 2001
From: Oliver Sanders
Date: Thu, 21 Jan 2021 10:29:43 +0000
Subject: [PATCH 09/13] tests: make purge safer
* abort if no flow is to be purged
* only wait on lsof for a maximum of five tries
---
tests/functional/lib/bash/test_header | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header
index 8617a6688f9..6531ab3ef6d 100644
--- a/tests/functional/lib/bash/test_header
+++ b/tests/functional/lib/bash/test_header
@@ -503,13 +503,22 @@ purge () {
return 0
fi
+ if [[ -z $SUITE_NAME ]]; then
+ echo 'no flow to purge' >&2
+ return 1
+ fi
+
local SUITE_RUN_DIR="${RUN_DIR}/${SUITE_NAME}"
# wait for local processes to let go of their file handles
if ${HAS_LSOF}; then
# NOTE: lsof can hang, so call with "timeout".
# NOTE: lsof can raise warnings with some filesystems so ignore stderr
- while grep -q "${SUITE_RUN_DIR}" < <(timeout 5 lsof 2>/dev/null); do
+ # shellcheck disable=SC2034
+ for try in $(seq 1 5); do
+ if grep -q "${SUITE_RUN_DIR}" < <(timeout 5 lsof 2>/dev/null); then
+ break
+ fi
sleep 1
done
fi
From f79b5648cfbf1e5126295b051d5b50768850f5df Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Fri, 22 Jan 2021 16:50:10 +0000
Subject: [PATCH 10/13] Raise error with mix of cylc install options
Update yml timeout
---
cylc/flow/suite_files.py | 101 ++++++++++++++----
.../authentication/02-suite2-stop-suite1.t | 15 +--
tests/functional/cylc-install/02-failures.t | 86 ++++++++++++---
3 files changed, 159 insertions(+), 43 deletions(-)
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index 5333803ddb1..327993b7bd4 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -148,9 +148,6 @@ class SuiteFiles:
SUITE_RC = 'suite.rc'
"""Deprecated workflow configuration file."""
- SOURCE = 'source'
- """Symlink to the workflow source directory (For workflow dir)"""
-
class Service:
"""The directory containing Cylc system files."""
@@ -376,11 +373,11 @@ def get_suite_source_dir(reg):
source_path = Path(
cwd,
SuiteFiles.Install.DIRNAME,
- SuiteFiles.SOURCE)
+ SuiteFiles.Install.SOURCE)
alt_source_path = Path(
cwd.parent,
SuiteFiles.Install.DIRNAME,
- SuiteFiles.SOURCE)
+ SuiteFiles.Install.SOURCE)
try:
source = os.readlink(source_path)
except OSError:
@@ -1044,29 +1041,21 @@ def install_workflow(flow_name=None, source=None, run_name=None,
' Please choose another run name.')
validate_source_dir(source, flow_name)
run_path_base = Path(get_workflow_run_dir(flow_name)).expanduser()
- relink = False
- run_num = 0
- if no_run_name:
- rundir = run_path_base
- elif run_name:
- rundir = run_path_base.joinpath(run_name)
- else:
- run_n = Path(run_path_base, 'runN').expanduser()
- run_num = get_next_rundir_number(run_path_base)
- rundir = Path(run_path_base, f'run{run_num}')
- if run_num == 1 and rundir.exists():
- WorkflowFilesError(
- f"This path: {rundir} exists. Try using --run-name")
- unlink_runN(run_n)
- relink = True
+ relink, run_num, rundir = get_run_dir(run_path_base, run_name, no_run_name)
+ if Path(rundir).exists():
+ raise WorkflowFilesError(
+ f"\"{rundir}\" exists."
+ " Try using cylc reinstall. Alternatively, install with another"
+ " name, using the --run-name option.")
check_nested_run_dirs(rundir, flow_name)
+ symlinks_created = {}
if not no_symlinks:
sub_dir = flow_name
if run_num:
sub_dir += '/' + f'run{run_num}'
symlinks_created = make_localhost_symlinks(rundir, sub_dir)
_open_install_log(rundir)
- if not no_symlinks and bool(symlinks_created):
+ if not no_symlinks and bool(symlinks_created) is True:
for src, dst in symlinks_created.items():
INSTALL_LOG.info(f"Symlink created from {src} to {dst}")
try:
@@ -1112,7 +1101,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
INSTALL_LOG.info("Symlink from {source_link} to {source} in place.")
else:
raise WorkflowFilesError(
- "Source directory between runs are not consistent")
+ "Source directory between runs are not consistent.")
# check source link matches the source symlink from workflow dir.
INSTALL_LOG.info(f'INSTALLED {flow_name} from {source} -> {rundir}')
print(f'INSTALLED {flow_name} from {source} -> {rundir}')
@@ -1120,6 +1109,74 @@ def install_workflow(flow_name=None, source=None, run_name=None,
return source, rundir, flow_name
+def get_run_dir(run_path_base, run_name, no_run_name):
+ """ Build run directory for current install.
+
+ Args:
+ run_path_base (Path):
+ The workflow directory.
+ run_name (str):
+ Name of the run.
+ no_run_name (bool):
+ Flag as True to incidate no run name - worklow installed into
+ ~/cylc-run/$(basename $PWD).
+
+ Returns:
+ relink (bool):
+ True if runN symlink needs updating.
+ run_num (int):
+ Run number of the current install.
+ rundir (Path):
+ Run directory.
+ """
+ relink = False
+ run_num = 0
+ if no_run_name:
+ rundir = run_path_base
+ elif run_name:
+ rundir = run_path_base.joinpath(run_name)
+ if (run_path_base.exists() and
+ detect_flow_exists(run_path_base, True)):
+ raise WorkflowFilesError(
+ f"This path: \"{run_path_base}\" contains installed numbered"
+ " runs. Try again, using cylc install without --run-name.")
+ else:
+ run_n = Path(run_path_base, 'runN').expanduser()
+ run_num = get_next_rundir_number(run_path_base)
+ rundir = Path(run_path_base, f'run{run_num}')
+ if run_path_base.exists() and detect_flow_exists(run_path_base, False):
+ raise WorkflowFilesError(
+ f"This path: \"{run_path_base}\" contains an installed"
+ " workflow. Try again, using --run-name.")
+ unlink_runN(run_n)
+ relink = True
+ return relink, run_num, rundir
+
+
+def detect_flow_exists(run_path_base, numbered):
+ """Returns True if installed flow already exists.
+
+ Args:
+ run_path_base (Path):
+ Workflow run directory
+ numbered (bool):
+ If true, will detect if numbered runs exist
+ If false, will detect if non-numbered runs exist, i.e. runs
+ installed by --run-name)
+
+ Returns:
+ True if installed flows exist.
+
+ """
+ for entry in Path(run_path_base).iterdir():
+ isNumbered = bool(re.search(r'^run\d+$', entry.name))
+ if (entry.is_dir() and
+ entry.name not in [SuiteFiles.Install.DIRNAME, 'runN'] and
+ Path(entry, SuiteFiles.FLOW_FILE).exists() and
+ isNumbered == numbered):
+ return True
+
+
def create_workflow_srv_dir(rundir=None, source=None):
"""Create suite service directory"""
diff --git a/tests/functional/authentication/02-suite2-stop-suite1.t b/tests/functional/authentication/02-suite2-stop-suite1.t
index f8492fb9397..eb4870c3dab 100755
--- a/tests/functional/authentication/02-suite2-stop-suite1.t
+++ b/tests/functional/authentication/02-suite2-stop-suite1.t
@@ -24,16 +24,16 @@ RUND="$RUN_DIR"
NAME1="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}-1"
NAME2="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}-2"
SUITE1_RUND="${RUND}/${NAME1}"
-mkdir -p "${SUITE1_RUND}"
RND_SUITE_NAME=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)
RND_SUITE_SOURCE="$PWD/${RND_SUITE_NAME}"
mkdir -p "${RND_SUITE_SOURCE}"
cp -p "${TEST_SOURCE_DIR}/basic/flow.cylc" "${RND_SUITE_SOURCE}"
-cylc install --flow-name="${NAME1}" --no-run-name --directory="${RND_SUITE_SOURCE}"
-SUITE2_RUND="${RUND}/${NAME2}"
-mkdir -p "${SUITE2_RUND}"
-rm "${RND_SUITE_SOURCE}/flow.cylc"
-cat >"${RND_SUITE_SOURCE}/flow.cylc" <<__FLOW_CONFIG__
+cylc install --flow-name="${NAME1}" --directory="${RND_SUITE_SOURCE}" --no-run-name
+
+RND_SUITE_NAME2=x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)
+RND_SUITE_SOURCE2="$PWD/${RND_SUITE_NAME2}"
+mkdir -p "${RND_SUITE_SOURCE2}"
+cat >"${RND_SUITE_SOURCE2}/flow.cylc" <<__FLOW_CONFIG__
[scheduler]
[[events]]
[scheduling]
@@ -43,7 +43,7 @@ cat >"${RND_SUITE_SOURCE}/flow.cylc" <<__FLOW_CONFIG__
[[t1]]
script=cylc shutdown "${NAME1}"
__FLOW_CONFIG__
-cylc install --flow-name="${NAME2}" --directory="${RND_SUITE_SOURCE}" --no-run-name
+cylc install --flow-name="${NAME2}" --directory="${RND_SUITE_SOURCE2}" --no-run-name
cylc run --no-detach "${NAME1}" 1>'1.out' 2>&1 &
SUITE_RUN_DIR="${SUITE1_RUND}" poll_suite_running
run_ok "${TEST_NAME_BASE}" cylc run --no-detach --abort-if-any-task-fails "${NAME2}"
@@ -51,4 +51,5 @@ cylc shutdown "${NAME1}" --max-polls=20 --interval=1 1>'/dev/null' 2>&1 || true
purge "${NAME1}"
purge "${NAME2}"
rm -rf "${RND_SUITE_SOURCE}"
+rm -rf "${RND_SUITE_SOURCE2}"
exit
diff --git a/tests/functional/cylc-install/02-failures.t b/tests/functional/cylc-install/02-failures.t
index 7f9cfa7a7d1..73240b2cfa9 100644
--- a/tests/functional/cylc-install/02-failures.t
+++ b/tests/functional/cylc-install/02-failures.t
@@ -41,7 +41,7 @@ function purge_rnd_suite() {
}
. "$(dirname "$0")/test_header"
-set_test_number 23
+set_test_number 35
# Test source directory between runs that are not consistent result in error
@@ -60,7 +60,7 @@ touch flow.cylc
run_fail "${TEST_NAME}" cylc install
contains_ok "${TEST_NAME}.stderr" <<__ERR__
-WorkflowFilesError: Source directory between runs are not consistent
+WorkflowFilesError: Source directory between runs are not consistent.
__ERR__
rm -rf "${PWD:?}/${SOURCE_DIR_1}" "${PWD:?}/${SOURCE_DIR_2}"
rm -rf "${RUN_DIR:?}/${TEST_NAME_BASE}"
@@ -68,6 +68,7 @@ popd || exit
# Test fail no suite source dir
+
TEST_NAME="${TEST_NAME_BASE}-nodir"
make_rnd_suite
rm -rf "${RND_SUITE_SOURCE}"
@@ -78,6 +79,7 @@ __ERR__
purge_rnd_suite
# Test fail no flow.cylc or suite.rc file
+
TEST_NAME="${TEST_NAME_BASE}-no-flow-file"
make_rnd_suite
rm -f "${RND_SUITE_SOURCE}/flow.cylc"
@@ -88,6 +90,7 @@ __ERR__
purge_rnd_suite
# Test cylc install fails when given flow-name that is an absolute path
+
TEST_NAME="${TEST_NAME_BASE}-no-abs-path-flow-name"
make_rnd_suite
run_fail "${TEST_NAME}" cylc install --flow-name="${RND_SUITE_SOURCE}" -C "${RND_SUITE_SOURCE}"
@@ -97,6 +100,7 @@ __ERR__
purge_rnd_suite
# Test cylc install fails when given run-name _cylc-install
+
TEST_NAME="${TEST_NAME_BASE}-run-name-cylc-install-forbidden"
make_rnd_suite
run_fail "${TEST_NAME}" cylc install --run-name=_cylc-install -C "${RND_SUITE_SOURCE}"
@@ -106,6 +110,7 @@ __ERR__
purge_rnd_suite
# Test source dir can not contain '_cylc-install, log, share, work' dirs
+
for DIR in 'work' 'share' 'log' '_cylc-install'; do
TEST_NAME="${TEST_NAME_BASE}-${DIR}-forbidden-in-source"
make_rnd_suite
@@ -119,7 +124,71 @@ __ERR__
popd || exit 1
done
+# Test --run-name and --no-run-name options are mutually exclusive
+
+TEST_NAME="${TEST_NAME_BASE}--no-run-name-and--run-name-forbidden"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_fail "${TEST_NAME}" cylc install --run-name="${RND_SUITE_NAME}" --no-run-name
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+cylc: error: options --no-run-name and --run-name are mutually exclusive.
+__ERR__
+purge_rnd_suite
+popd || exit 1
+
+# Test running cylc install twice, first using --run-name, followed by standard run results in error
+
+TEST_NAME="${TEST_NAME_BASE}-install-twice-mix-options-1"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}-1st-cylc-install" cylc install . --run-name=olaf
+contains_ok "${TEST_NAME}-1st-cylc-install.stdout" <<__OUT__
+INSTALLED ${RND_SUITE_NAME} from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/olaf
+__OUT__
+run_fail "${TEST_NAME}-2nd-cylc-install" cylc install "${RND_SUITE_NAME}"
+contains_ok "${TEST_NAME}-2nd-cylc-install.stderr" <<__ERR__
+WorkflowFilesError: This path: "${RND_SUITE_RUNDIR}" contains an installed workflow. Try again, using --run-name.
+__ERR__
+
+popd || exit 1
+purge_rnd_suite
+
+# Test running cylc install twice, first using standard run, followed by --run-name results in error
+
+TEST_NAME="${TEST_NAME_BASE}-install-twice-mix-options-2"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}-1st-cylc-install" cylc install .
+contains_ok "${TEST_NAME}-1st-cylc-install.stdout" <<__OUT__
+INSTALLED ${RND_SUITE_NAME} from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/run1
+__OUT__
+run_fail "${TEST_NAME}-2nd-cylc-install" cylc install "${RND_SUITE_NAME}" --run-name=olaf
+contains_ok "${TEST_NAME}-2nd-cylc-install.stderr" <<__ERR__
+WorkflowFilesError: This path: "${RND_SUITE_RUNDIR}" contains installed numbered runs. Try again, using cylc install without --run-name.
+__ERR__
+
+popd || exit 1
+purge_rnd_suite
+
+# Test running cylc install twice, using the same --run-name results in error
+
+TEST_NAME="${TEST_NAME_BASE}-install-twice-same-run-name"
+make_rnd_suite
+pushd "${RND_SUITE_SOURCE}" || exit 1
+run_ok "${TEST_NAME}-1st-cylc-install" cylc install . --run-name=olaf
+contains_ok "${TEST_NAME}-1st-cylc-install.stdout" <<__OUT__
+INSTALLED ${RND_SUITE_NAME} from ${RND_SUITE_SOURCE} -> ${RND_SUITE_RUNDIR}/olaf
+__OUT__
+run_fail "${TEST_NAME}-2nd-cylc-install" cylc install --run-name=olaf
+contains_ok "${TEST_NAME}-2nd-cylc-install.stderr" <<__ERR__
+WorkflowFilesError: "${RND_SUITE_RUNDIR}/olaf" exists. \
+Try using cylc reinstall. Alternatively, install with another name, using the --run-name option.
+__ERR__
+
+popd || exit 1
+purge_rnd_suite
# Test cylc install can not be run from within the cylc-run directory
+
TEST_NAME="${TEST_NAME_BASE}-forbid-cylc-run-dir-install"
BASE_NAME="test-install-${CYLC_TEST_TIME_INIT}"
mkdir -p "${RUN_DIR}/${BASE_NAME}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}" && cd "$_" || exit
@@ -128,20 +197,9 @@ run_fail "${TEST_NAME}" cylc install
contains_ok "${TEST_NAME}.stderr" <<__ERR__
WorkflowFilesError: ${TEST_NAME} installation failed. Source directory should not be in ${RUN_DIR}
__ERR__
+
cd "${RUN_DIR}" || exit
rm -rf "${BASE_NAME}"
purge_rnd_suite
-# Test --run-name and --no-run-name options are mutually exclusive
-
-TEST_NAME="${TEST_NAME_BASE}--no-run-name-and--run-name-forbidden"
-make_rnd_suite
-pushd "${RND_SUITE_SOURCE}" || exit 1
-run_fail "${TEST_NAME}" cylc install --run-name="${RND_SUITE_NAME}" --no-run-name
-contains_ok "${TEST_NAME}.stderr" <<__ERR__
-cylc: error: options --no-run-name and --run-name are mutually exclusive.
-__ERR__
-purge_rnd_suite
-popd || exit 1
-
exit
From 6ce05cc372c3ced0e56c821df11be7a4b62f5c9c Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Tue, 26 Jan 2021 20:56:44 +0000
Subject: [PATCH 11/13] Refactor install, add cylc install bash test
---
cylc/flow/scheduler.py | 2 +-
cylc/flow/suite_files.py | 68 +++++++++++----------
tests/functional/cylc-install/02-failures.t | 13 +++-
3 files changed, 49 insertions(+), 34 deletions(-)
diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index 65ebcf31e93..5aa45e1fa59 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -283,7 +283,7 @@ async def install(self):
"""
# Install
- source = suite_files.get_suite_source_dir(self.suite)
+ source = suite_files.get_suite_source_dir()
if source is None:
# register workflow
rund = get_workflow_run_dir(self.suite)
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index 327993b7bd4..5dfb160aa1c 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -148,6 +148,9 @@ class SuiteFiles:
SUITE_RC = 'suite.rc'
"""Deprecated workflow configuration file."""
+ RUN_N = 'runN'
+ """Symbolic link for latest run"""
+
class Service:
"""The directory containing Cylc system files."""
@@ -366,8 +369,8 @@ def get_flow_file(reg):
return flow_file
-def get_suite_source_dir(reg):
- """Return the source directory path of a workflow.
+def get_suite_source_dir():
+ """Return the source directory path of the workflow in CWD.
"""
cwd = Path.cwd()
source_path = Path(
@@ -463,18 +466,7 @@ def parse_suite_arg(options, arg):
if os.path.isdir(arg):
path = os.path.join(arg, SuiteFiles.FLOW_FILE)
name = os.path.basename(arg)
- if not os.path.exists(path):
- # Probably using deprecated suite.rc
- path = os.path.join(arg, SuiteFiles.SUITE_RC)
- if not os.path.exists(path):
- raise WorkflowFilesError(
- f'no {SuiteFiles.FLOW_FILE} or '
- f'{SuiteFiles.SUITE_RC} in {arg}')
- else:
- LOG.warning(
- f'The filename "{SuiteFiles.SUITE_RC}" is '
- f'deprecated in favour of '
- f'"{SuiteFiles.FLOW_FILE}".')
+ check_flow_file(arg, 'LOG')
else:
path = arg
name = os.path.basename(os.path.dirname(arg))
@@ -1066,20 +1058,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
if relink:
link_runN(rundir)
create_workflow_srv_dir(rundir)
- # flow.cylc must exist so we can detect accidentally reversed args.
- flow_file_path = source.joinpath(SuiteFiles.FLOW_FILE)
- if not flow_file_path.is_file():
- # If using deprecated suite.rc, symlink it into flow.cylc:
- suite_rc_path = source.joinpath(SuiteFiles.SUITE_RC)
- if suite_rc_path.is_file():
- flow_file_path.symlink_to(suite_rc_path)
- INSTALL_LOG.warning(
- f'The filename "{SuiteFiles.SUITE_RC}" is deprecated in favour'
- f' of "{SuiteFiles.FLOW_FILE}". Symlink created.')
- else:
- raise WorkflowFilesError(
- f'no {SuiteFiles.FLOW_FILE} or {SuiteFiles.SUITE_RC}'
- f' in {source}')
+ check_flow_file(source, 'INSTALL')
rsync_cmd = get_rsync_rund_cmd(source, rundir)
proc = Popen(rsync_cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True)
stdout, stderr = proc.communicate()
@@ -1141,7 +1120,7 @@ def get_run_dir(run_path_base, run_name, no_run_name):
f"This path: \"{run_path_base}\" contains installed numbered"
" runs. Try again, using cylc install without --run-name.")
else:
- run_n = Path(run_path_base, 'runN').expanduser()
+ run_n = Path(run_path_base, SuiteFiles.RUN_N).expanduser()
run_num = get_next_rundir_number(run_path_base)
rundir = Path(run_path_base, f'run{run_num}')
if run_path_base.exists() and detect_flow_exists(run_path_base, False):
@@ -1171,12 +1150,37 @@ def detect_flow_exists(run_path_base, numbered):
for entry in Path(run_path_base).iterdir():
isNumbered = bool(re.search(r'^run\d+$', entry.name))
if (entry.is_dir() and
- entry.name not in [SuiteFiles.Install.DIRNAME, 'runN'] and
- Path(entry, SuiteFiles.FLOW_FILE).exists() and
+ entry.name not in [SuiteFiles.Install.DIRNAME, SuiteFiles.RUN_N]
+ and Path(entry, SuiteFiles.FLOW_FILE).exists() and
isNumbered == numbered):
return True
+def check_flow_file(path, log_type):
+ """Raises error if no flow file in path sent.
+
+ Creates a symlink to flow.cylc file if suite.rc file exists.
+
+ Args:
+ path: Path to check for either suite.rc or flow.cylc file
+ log_type: Which log to log error
+
+ """
+ flow_file_path = Path(path, SuiteFiles.FLOW_FILE)
+ suite_rc_path = Path(path, SuiteFiles.SUITE_RC)
+ msg = (f'The filename "{SuiteFiles.SUITE_RC}" is deprecated in favour'
+ f' of "{SuiteFiles.FLOW_FILE}". Symlink created')
+ if flow_file_path.exists():
+ return
+ if suite_rc_path.exists():
+ log_type.warning(msg)
+ flow_file_path.symlink_to(suite_rc_path)
+ else:
+ raise WorkflowFilesError(
+ f'no {SuiteFiles.FLOW_FILE} or '
+ f'{SuiteFiles.SUITE_RC} in {path}')
+
+
def create_workflow_srv_dir(rundir=None, source=None):
"""Create suite service directory"""
@@ -1232,7 +1236,7 @@ def unlink_runN(run_n):
def link_runN(latest_run):
"""Create symlink runN, pointing at the latest run"""
latest_run = Path(latest_run).expanduser()
- run_n = Path(latest_run.parent, 'runN')
+ run_n = Path(latest_run.parent, SuiteFiles.RUN_N)
try:
run_n.symlink_to(latest_run)
except OSError:
diff --git a/tests/functional/cylc-install/02-failures.t b/tests/functional/cylc-install/02-failures.t
index 73240b2cfa9..3e9aeed86dd 100644
--- a/tests/functional/cylc-install/02-failures.t
+++ b/tests/functional/cylc-install/02-failures.t
@@ -41,7 +41,7 @@ function purge_rnd_suite() {
}
. "$(dirname "$0")/test_header"
-set_test_number 35
+set_test_number 37
# Test source directory between runs that are not consistent result in error
@@ -109,6 +109,17 @@ WorkflowFilesError: Run name cannot be "_cylc-install". Please choose another ru
__ERR__
purge_rnd_suite
+# Test cylc install invalid flow-name
+
+TEST_NAME="${TEST_NAME_BASE}--invalid-flow-name-cylc-install"
+make_rnd_suite
+run_fail "${TEST_NAME}" cylc install --flow-name=\.invalid -C "${RND_SUITE_SOURCE}"
+contains_ok "${TEST_NAME}.stderr" <<__ERR__
+WorkflowFilesError: Invalid workflow name - cannot start with: \`\`.\`\`, \`\`-\`\`
+__ERR__
+purge_rnd_suite
+
+
# Test source dir can not contain '_cylc-install, log, share, work' dirs
for DIR in 'work' 'share' 'log' '_cylc-install'; do
From ba17a8a45d22a7d20ff6cdb5c63aec8f28542a41 Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Wed, 27 Jan 2021 00:16:13 +0000
Subject: [PATCH 12/13] Add suite.rc deprecation tests
---
cylc/flow/suite_files.py | 23 +++++-----------
tests/functional/deprecations/03-suiterc.t | 31 ++++++++++++++++++----
2 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py
index 5dfb160aa1c..676392cd2ff 100644
--- a/cylc/flow/suite_files.py
+++ b/cylc/flow/suite_files.py
@@ -466,7 +466,7 @@ def parse_suite_arg(options, arg):
if os.path.isdir(arg):
path = os.path.join(arg, SuiteFiles.FLOW_FILE)
name = os.path.basename(arg)
- check_flow_file(arg, 'LOG')
+ check_flow_file(arg, LOG)
else:
path = arg
name = os.path.basename(os.path.dirname(arg))
@@ -512,23 +512,12 @@ def register(flow_name=None, source=None):
source = os.getcwd()
# flow.cylc must exist so we can detect accidentally reversed args.
source = os.path.abspath(source)
- flow_file_path = os.path.join(source, SuiteFiles.FLOW_FILE)
- if not os.path.isfile(flow_file_path):
- # If using deprecated suite.rc, symlink it into flow.cylc:
- suite_rc_path = os.path.join(source, SuiteFiles.SUITE_RC)
- if os.path.isfile(suite_rc_path):
- os.symlink(suite_rc_path, flow_file_path)
- LOG.warning(
- f'The filename "{SuiteFiles.SUITE_RC}" is deprecated in favor '
- f'of "{SuiteFiles.FLOW_FILE}". Symlink created.')
- else:
- raise WorkflowFilesError(
- f'no flow.cylc or suite.rc in {source}')
+ check_flow_file(source, LOG)
symlinks_created = make_localhost_symlinks(
get_workflow_run_dir(flow_name), flow_name)
if bool(symlinks_created):
for src, dst in symlinks_created.items():
- INSTALL_LOG.info(f"Symlink created from {src} to {dst}")
+ LOG.info(f"Symlink created from {src} to {dst}")
# Create service dir if necessary.
srv_d = get_suite_srv_dir(flow_name)
os.makedirs(srv_d, exist_ok=True)
@@ -1058,7 +1047,7 @@ def install_workflow(flow_name=None, source=None, run_name=None,
if relink:
link_runN(rundir)
create_workflow_srv_dir(rundir)
- check_flow_file(source, 'INSTALL')
+ check_flow_file(source, INSTALL_LOG)
rsync_cmd = get_rsync_rund_cmd(source, rundir)
proc = Popen(rsync_cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True)
stdout, stderr = proc.communicate()
@@ -1169,11 +1158,11 @@ def check_flow_file(path, log_type):
flow_file_path = Path(path, SuiteFiles.FLOW_FILE)
suite_rc_path = Path(path, SuiteFiles.SUITE_RC)
msg = (f'The filename "{SuiteFiles.SUITE_RC}" is deprecated in favour'
- f' of "{SuiteFiles.FLOW_FILE}". Symlink created')
+ f' of "{SuiteFiles.FLOW_FILE}". Symlink created.')
if flow_file_path.exists():
return
if suite_rc_path.exists():
- log_type.warning(msg)
+ log_type.warning(f"{msg}")
flow_file_path.symlink_to(suite_rc_path)
else:
raise WorkflowFilesError(
diff --git a/tests/functional/deprecations/03-suiterc.t b/tests/functional/deprecations/03-suiterc.t
index e4a3418b8fd..4995b371d96 100644
--- a/tests/functional/deprecations/03-suiterc.t
+++ b/tests/functional/deprecations/03-suiterc.t
@@ -18,12 +18,13 @@
# Test backwards compatibility for suite.rc files
. "$(dirname "$0")/test_header"
-set_test_number 3
+set_test_number 7
init_suiterc() {
local TEST_NAME="$1"
local FLOW_CONFIG="${2:--}"
- SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}"
+ SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}"
+ SUITE_RUN_DIR="$RUN_DIR/${SUITE_NAME}"
mkdir -p "${TEST_DIR}/${SUITE_NAME}/"
cat "${FLOW_CONFIG}" >"${TEST_DIR}/${SUITE_NAME}/suite.rc"
cd "${TEST_DIR}/${SUITE_NAME}" || exit
@@ -37,10 +38,30 @@ __FLOW__
TEST_NAME="${TEST_NAME_BASE}-validate"
run_ok "${TEST_NAME}" cylc validate .
-
+grep_ok "The filename \"suite.rc\" is deprecated in favour of \"flow.cylc\". Symlink created." "${TEST_NAME_BASE}-validate.stderr"
TEST_NAME="${TEST_NAME_BASE}-install"
run_ok "${TEST_NAME}" cylc install --flow-name="${SUITE_NAME}" --no-run-name
-
+cd "${SUITE_RUN_DIR}" || exit 1
exists_ok "flow.cylc"
-
+cd "${TEST_DIR}" || exit 1
+rm -rf "${TEST_DIR:?}/${SUITE_NAME}/"
purge
+
+# Test install upgrades suite.rc and logs deprecation notification
+
+init_suiterc "${TEST_NAME_BASE}" <<'__FLOW__'
+[scheduling]
+ [[graph]]
+ R1 = foo => bar
+__FLOW__
+
+
+TEST_NAME="${TEST_NAME_BASE}-install"
+run_ok "${TEST_NAME}" cylc install --flow-name="${SUITE_NAME}" --no-run-name
+cd "${SUITE_RUN_DIR}" || exit 1
+exists_ok "flow.cylc"
+INSTALL_LOG="$(find "${SUITE_RUN_DIR}/log/install" -type f -name '*.log')"
+grep_ok "The filename \"suite.rc\" is deprecated in favour of \"flow.cylc\". Symlink created." "${INSTALL_LOG}"
+cd "${TEST_DIR}" || exit 1
+rm -rf "${TEST_DIR:?}/${SUITE_NAME}/"
+purge
\ No newline at end of file
From 0d139181640adc06a856a1b817643b5325a11863 Mon Sep 17 00:00:00 2001
From: Mel Hall <37735232+datamel@users.noreply.github.com>
Date: Wed, 27 Jan 2021 00:30:22 +0000
Subject: [PATCH 13/13] Update tests/f/authentication/01
---
.../authentication/01-remote-suite-same-name.t | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/tests/functional/authentication/01-remote-suite-same-name.t b/tests/functional/authentication/01-remote-suite-same-name.t
index 385b5cdfca2..e3911f129f8 100755
--- a/tests/functional/authentication/01-remote-suite-same-name.t
+++ b/tests/functional/authentication/01-remote-suite-same-name.t
@@ -27,20 +27,18 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
SSH_OPTS='-oBatchMode=yes -oConnectTimeout=5'
# shellcheck disable=SC2029,SC2086
-ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" mkdir -p "cylctb-cylc-source/${SUITE_NAME}"
+#ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" mkdir -p "cylctb-cylc-source/${SUITE_NAME}"
+SRC_DIR="$(ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" mktemp -d)"
# shellcheck disable=SC2086
scp ${SSH_OPTS} -pqr "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/"* \
- "${CYLC_TEST_HOST}:cylctb-cylc-source/${SUITE_NAME}"
+ "${CYLC_TEST_HOST}:${SRC_DIR}"
# shellcheck disable=SC2086
run_ok "${TEST_NAME_BASE}-install" \
ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" \
CYLC_VERSION="$(cylc version)" cylc install --flow-name="${SUITE_NAME}" \
- --no-run-name --directory="cylctb-cylc-source/${SUITE_NAME}"
+ --no-run-name --directory="${SRC_DIR}"
suite_run_ok "${TEST_NAME_BASE}" \
cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
-# shellcheck disable=SC2086
-ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" \
- rm -rf cylctb-cylc-source
purge
exit