diff --git a/cylc/flow/cfgspec/suite.py b/cylc/flow/cfgspec/suite.py index 7b697b21db8..c91df810690 100644 --- a/cylc/flow/cfgspec/suite.py +++ b/cylc/flow/cfgspec/suite.py @@ -85,7 +85,7 @@ ''') with Conf('scheduler'): - Conf('includes', VDR.V_STRING_LIST, desc=''' + Conf('install', VDR.V_STRING_LIST, desc=''' Configure the directories and files to be included in the remote file installation. @@ -117,7 +117,7 @@ .. code-block:: cylc [scheduler] - includes = dir/, dir2/, file1, file2 + install = dir/, dir2/, file1, file2 ''') with Conf('cylc'): diff --git a/cylc/flow/config.py b/cylc/flow/config.py index 01a15361c6f..25a1d41d2bd 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -227,7 +227,7 @@ def __init__( self.cfg = self.pcfg.get(sparse=True) self.mem_log("config.py: after get(sparse=True)") - if 'scheduler' in self.cfg and 'includes' in self.cfg['scheduler']: + if 'scheduler' in self.cfg and 'install' in self.cfg['scheduler']: self.get_validated_rsync_includes() # First check for the essential scheduling section. @@ -2340,18 +2340,15 @@ def get_expected_failed_tasks(self): return None def get_validated_rsync_includes(self): - """Validate and return items configured to be included in the file - installation""" - includes = self.cfg['scheduler']['includes'] - if includes: - illegal_includes = [] - for include in includes: - if include.count("/") > 1: - illegal_includes.append(f"{include}") - if len(illegal_includes) > 0: - raise SuiteConfigError( - "Error in [scheduler] includes. " - "Directories can only be from the top level, please " - "reconfigure:" + str(illegal_includes)[1:-1]) - else: - return includes + """Validate and return items to be included in the file installation""" + includes = self.cfg['scheduler']['install'] + illegal_includes = [] + for include in includes: + if include.count("/") > 1: + illegal_includes.append(f"{include}") + if len(illegal_includes) > 0: + raise SuiteConfigError( + "Error in [scheduler] install. " + "Directories can only be from the top level, please " + "reconfigure:" + str(illegal_includes)[1:-1]) + return includes diff --git a/cylc/flow/hostuserutil.py b/cylc/flow/hostuserutil.py index 4814abf0fbe..f1ff39dd55a 100644 --- a/cylc/flow/hostuserutil.py +++ b/cylc/flow/hostuserutil.py @@ -227,19 +227,6 @@ def _is_remote_platform(self, platform): return True return False - def _is_remote_install_target(self, install_target): - """Determines whether install_target is remote or not. - Return True if install target has different IP address - to the current host. - - Return False if install target is to be treated as localhost. - """ - - if is_remote_host(install_target) is True: - return True - else: - return False - def get_host_ip_by_name(target): """Shorthand for HostUtil.get_inst().get_host_ip_by_name(target).""" @@ -276,11 +263,6 @@ def is_remote_platform(platform): return HostUtil.get_inst()._is_remote_platform(platform) -def is_remote_install_target(install_target): - """Shorthand for get_inst()._is_remote_install_target(install_target).""" - return HostUtil.get_inst()._is_remote_install_target(install_target) - - def is_remote_host(name): """Shorthand for HostUtil.get_inst().is_remote_host(name).""" return HostUtil.get_inst().is_remote_host(name) diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py index dc74bd23af0..dc336611fdf 100644 --- a/cylc/flow/pathutil.py +++ b/cylc/flow/pathutil.py @@ -74,7 +74,7 @@ def get_suite_run_log_name(suite): def get_suite_file_install_log_name(suite): - """Return suite run log file path.""" + """Return suite file install log file path.""" path = get_suite_run_dir(suite, 'log', 'suite', 'file-installation-log') return expandvars(path) diff --git a/cylc/flow/platforms.py b/cylc/flow/platforms.py index 3c9ea62671f..b417f3156fa 100644 --- a/cylc/flow/platforms.py +++ b/cylc/flow/platforms.py @@ -408,6 +408,13 @@ def get_install_target_from_platform(platform): Returns install target.""" if not platform['install target']: - platform['install target'] = platform.get('name') + platform['install target'] = platform['name'] return platform.get('install target') + + +def is_platform_with_target_in_list( + install_target, distinct_platforms_list): + """Determines whether install target is in the list of platforms""" + for distinct_platform in distinct_platforms_list: + return install_target == distinct_platform['install target'] diff --git a/cylc/flow/remote.py b/cylc/flow/remote.py index ec71a657abd..7ea40298c42 100644 --- a/cylc/flow/remote.py +++ b/cylc/flow/remote.py @@ -156,13 +156,11 @@ def construct_platform_ssh_cmd(raw_cmd, platform, **kwargs): def get_includes_to_rsync(rsync_includes=None): - """Returns a list of directories/files, configured in flow.cylc, - to be included in the remote file installation. - """ + """Returns list of configured dirs/files for remote file installation.""" configured_includes = [] - if rsync_includes: + if rsync_includes is not None: for include in rsync_includes: if include.endswith("/"): # item is a directory configured_includes.append("/" + include + "***") @@ -205,8 +203,7 @@ def construct_rsync_over_ssh_cmd( # Note to future devs - be wary of changing the order of the following # rsync options, rsync is very particular about order of in/ex-cludes. - excludes = ['log', 'share', 'work'] - for exclude in excludes: + for exclude in ['log', 'share', 'work']: rsync_cmd.append(f"--exclude={exclude}") default_includes = [ '/app/***', @@ -215,8 +212,7 @@ def construct_rsync_over_ssh_cmd( '/lib/***'] for include in default_includes: rsync_cmd.append(f"--include={include}") - configured_includes = get_includes_to_rsync(rsync_includes) - for include in configured_includes: + for include in get_includes_to_rsync(rsync_includes): rsync_cmd.append(f"--include={include}") # The following excludes are required in case these are added to the rsync_cmd.append("--exclude=*") # exclude everything else diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py index 9e8ce6dbc43..eb541d72d2c 100644 --- a/cylc/flow/scheduler.py +++ b/cylc/flow/scheduler.py @@ -72,6 +72,10 @@ get_suite_test_log_name, make_suite_run_tree, ) +from cylc.flow.platforms import ( + get_install_target_from_platform, + get_platform, + is_platform_with_target_in_list) from cylc.flow.profiler import Profiler from cylc.flow.resources import extract_resources from cylc.flow.subprocpool import SubProcPool @@ -101,9 +105,6 @@ get_time_string_from_unix_time as time2str, get_utc_mode) from cylc.flow.xtrigger_mgr import XtriggerManager -from cylc.flow.platforms import ( - get_install_target_from_platform, - get_platform) class SchedulerStop(CylcError): @@ -722,14 +723,6 @@ def restart_remote_init(self): """ - def is_platform_with_target_in_list( - install_target, distinct_platforms_list): - """Determines whether install target is in the list of platforms""" - for distinct_platform in distinct_platforms_list: - if(install_target - == distinct_platform['install target']): - return True - distinct_install_target_platforms = [] for itask in self.pool.get_rh_tasks(): @@ -750,7 +743,7 @@ def is_platform_with_target_in_list( platform, self.curve_auth, self.client_pub_key_dir) is None): incomplete_init = True - + break if incomplete_init: # TODO: Review whether this sleep is needed. sleep(1.0) diff --git a/cylc/flow/task_remote_mgr.py b/cylc/flow/task_remote_mgr.py index aa0485f8b78..854620d5c83 100644 --- a/cylc/flow/task_remote_mgr.py +++ b/cylc/flow/task_remote_mgr.py @@ -33,9 +33,7 @@ from cylc.flow import LOG, RSYNC_LOG from cylc.flow.exceptions import TaskRemoteMgmtError import cylc.flow.flags -from cylc.flow.hostuserutil import ( - is_remote_host, is_remote_install_target, is_remote_platform -) +from cylc.flow.hostuserutil import (is_remote_host, is_remote_platform) from cylc.flow.pathutil import ( get_remote_suite_run_dir, get_suite_run_dir) @@ -160,7 +158,8 @@ def remote_init(self, platform, curve_auth, client_pub_key_dir (str): Client public key directory, used by the ZMQ authenticator. platform (dict): - A dictionary containing information about platform. + A dictionary containing settings relating to platform used in + this remote installation. Return: REMOTE_INIT_NOT_REQUIRED: @@ -178,7 +177,7 @@ def remote_init(self, platform, curve_auth, # If task is running locally we can skip the rest of this function if (self.single_task_mode or - not is_remote_install_target(self.install_target)): + not is_remote_host(get_host_from_platform(platform))): LOG.debug(f"REMOTE INIT NOT REQUIRED for {self.install_target}") return REMOTE_INIT_NOT_REQUIRED diff --git a/tests/functional/cylc-ping/05-check-keys-sharedfs.t b/tests/functional/cylc-ping/05-check-keys-sharedfs.t index 122073698ce..7215d140e19 100644 --- a/tests/functional/cylc-ping/05-check-keys-sharedfs.t +++ b/tests/functional/cylc-ping/05-check-keys-sharedfs.t @@ -23,7 +23,7 @@ require_remote_platform_wsfs export CYLC_TEST_PLATFORM="$CYLC_TEST_PLATFORM_WSFS" set_test_number 4 -init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' +init_suite "${TEST_NAME_BASE}" <<'__FLOW_CYLC__' #!jinja2 [cylc] [scheduling] @@ -35,7 +35,7 @@ init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' platform = {{CYLC_TEST_PLATFORM}} [[held]] script = true -__SUITE_RC__ +__FLOW_CYLC__ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" \ -s "CYLC_TEST_PLATFORM=${CYLC_TEST_PLATFORM}" @@ -44,7 +44,7 @@ suite_run_ok "${TEST_NAME_BASE}-run" cylc run "${SUITE_NAME}" \ RRUND="cylc-run/${SUITE_NAME}" RSRVD="${RRUND}/.service" poll_grep_suite_log 'Holding all waiting or queued tasks now' -SSH='ssh -n -oBatchMode=yes -oConnectTimeout=5' +SSH="$(cylc get-global-config -i "[platforms][$CYLC_TEST_PLATFORM]ssh command")" ${SSH} "${CYLC_TEST_PLATFORM}" \ find "${RSRVD}" -type f -name "*key*"|awk -F/ '{print $NF}'|sort >'find.out' cmp_ok 'find.out' <<'__OUT__' diff --git a/tests/functional/remote/01-basic-file-install.t b/tests/functional/remote/01-file-install.t similarity index 56% rename from tests/functional/remote/01-basic-file-install.t rename to tests/functional/remote/01-file-install.t index b0180936130..075ae8d23b1 100644 --- a/tests/functional/remote/01-basic-file-install.t +++ b/tests/functional/remote/01-file-install.t @@ -15,44 +15,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- -# Checks default files (app, bin, etc, lib) are correctly installed on the -# remote platform. +# Checks configured files/directories along with default files/directories +# (app, bin, etc, lib) are correctly installed on the remote platform. export CYLC_TEST_IS_GENERIC=false . "$(dirname "$0")/test_header" require_remote_platform -set_test_number 3 - -init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' -#!jinja2 -[cylc] -[scheduling] - [[graph]] - R1 = startup => holder => held -[runtime] - [[startup]] - script = """ - for DIR in "bin" "app" "etc" "lib" - do - mkdir -p "${CYLC_SUITE_RUN_DIR}/${DIR}" - touch "${CYLC_SUITE_RUN_DIR}/${DIR}/moo" - done - """ - platform = localhost - [[holder]] - script = """cylc hold "${CYLC_SUITE_NAME}" """ - platform = {{CYLC_TEST_PLATFORM}} - [[held]] - script = true -__SUITE_RC__ +set_test_number 6 +install_suite "${TEST_NAME_BASE}" run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" \ -s "CYLC_TEST_PLATFORM=${CYLC_TEST_PLATFORM}" -suite_run_ok "${TEST_NAME_BASE}-run" cylc run "${SUITE_NAME}" \ +suite_run_ok "${TEST_NAME_BASE}-run1" cylc run "${SUITE_NAME}" \ -s "CYLC_TEST_PLATFORM=${CYLC_TEST_PLATFORM}" RRUND="cylc-run/${SUITE_NAME}" poll_grep_suite_log 'Holding all waiting or queued tasks now' -SSH='ssh -n -oBatchMode=yes -oConnectTimeout=10' +SSH="$(cylc get-global-config -i "[platforms][$CYLC_TEST_PLATFORM]ssh command")" ${SSH} "${CYLC_TEST_PLATFORM}" \ find "${RRUND}/"{app,bin,etc,lib} -type f | sort > 'find.out' cmp_ok 'find.out' <<__OUT__ @@ -63,6 +41,34 @@ ${RRUND}/lib/moo __OUT__ cylc stop --max-polls=60 --interval=1 "${SUITE_NAME}" + purge_suite_platform "${CYLC_TEST_PLATFORM}" "${SUITE_NAME}" purge_suite "${SUITE_NAME}" + +install_suite "${TEST_NAME_BASE}" + +export SECOND_RUN="dir1/, dir2/, file1, file2" +run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" \ + -s "CYLC_TEST_PLATFORM=${CYLC_TEST_PLATFORM}" -s "SECOND_RUN=${SECOND_RUN}" +suite_run_ok "${TEST_NAME_BASE}-run2" cylc run "${SUITE_NAME}" \ + -s "CYLC_TEST_PLATFORM=${CYLC_TEST_PLATFORM}" -s "SECOND_RUN=${SECOND_RUN}" +poll_grep_suite_log 'Holding all waiting or queued tasks now' +${SSH} "${CYLC_TEST_PLATFORM}" \ +find "${RRUND}/"{app,bin,dir1,dir2,file1,file2,etc,lib} -type f | sort > 'find.out' +cmp_ok 'find.out' <<__OUT__ +${RRUND}/app/moo +${RRUND}/bin/moo +${RRUND}/dir1/moo +${RRUND}/dir2/moo +${RRUND}/etc/moo +${RRUND}/file1 +${RRUND}/file2 +${RRUND}/lib/moo +__OUT__ + +cylc stop --max-polls=60 --interval=1 "${SUITE_NAME}" + +purge_suite_platform "${CYLC_TEST_PLATFORM}" "${SUITE_NAME}" +purge_suite "${SUITE_NAME}" + exit diff --git a/tests/functional/remote/01-file-install/flow.cylc b/tests/functional/remote/01-file-install/flow.cylc new file mode 100644 index 00000000000..11f652980ef --- /dev/null +++ b/tests/functional/remote/01-file-install/flow.cylc @@ -0,0 +1,33 @@ +#!jinja2 +[cylc] + +{% if SECOND_RUN is defined %} + +[scheduler] + install = {{ SECOND_RUN }} + +{% endif %} + +[scheduling] + [[graph]] + R1 = startup => holder => held +[runtime] + [[startup]] + script = """ + for DIR in "bin" "app" "etc" "lib" "dir1" "dir2" + do + mkdir -p "${CYLC_SUITE_RUN_DIR}/${DIR}" + touch "${CYLC_SUITE_RUN_DIR}/${DIR}/moo" + done + + for FILE in "file1" "file2" + do + touch "${CYLC_SUITE_RUN_DIR}/${FILE}" + done + """ + platform = localhost + [[holder]] + script = """cylc hold "${CYLC_SUITE_NAME}" """ + platform = {{CYLC_TEST_PLATFORM}} + [[held]] + script = true diff --git a/tests/functional/remote/02-file-install-configured.t b/tests/functional/remote/02-file-install-configured.t deleted file mode 100644 index fc0fcb012dc..00000000000 --- a/tests/functional/remote/02-file-install-configured.t +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -#------------------------------------------------------------------------------- -# Checks default files (app, bin, etc, lib) are correctly installed on the -# remote platform. - -export CYLC_TEST_IS_GENERIC=false -. "$(dirname "$0")/test_header" -require_remote_platform -set_test_number 3 - -init_suite "${TEST_NAME_BASE}" <<'__SUITE_RC__' -#!jinja2 -[cylc] -[scheduler] - includes = dir1/, dir2/, file1, file2 -[scheduling] - [[graph]] - R1 = startup => holder => held -[runtime] - [[startup]] - script = """ - for DIR in "dir1" "dir2" - do - mkdir -p "${CYLC_SUITE_RUN_DIR}/${DIR}" - touch "${CYLC_SUITE_RUN_DIR}/${DIR}/moo" - done - - for FILE in "file1" "file2" - do - touch "${CYLC_SUITE_RUN_DIR}/${FILE}" - done - """ - platform = localhost - [[holder]] - script = """cylc hold "${CYLC_SUITE_NAME}" """ - platform = {{CYLC_TEST_PLATFORM}} - [[held]] - script = true -__SUITE_RC__ - -run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}" \ - -s "CYLC_TEST_PLATFORM=${CYLC_TEST_PLATFORM}" -suite_run_ok "${TEST_NAME_BASE}-run" cylc run "${SUITE_NAME}" \ - -s "CYLC_TEST_PLATFORM=${CYLC_TEST_PLATFORM}" -RRUND="cylc-run/${SUITE_NAME}" -poll_grep_suite_log 'Holding all waiting or queued tasks now' -SSH='ssh -n -oBatchMode=yes -oConnectTimeout=10' -${SSH} "${CYLC_TEST_PLATFORM}" \ -find "${RRUND}/"{dir1,dir2,file1,file2} -type f | sort > 'find.out' -cmp_ok 'find.out' <<__OUT__ -${RRUND}/dir1/moo -${RRUND}/dir2/moo -${RRUND}/file1 -${RRUND}/file2 -__OUT__ - -cylc stop --max-polls=60 --interval=1 "${SUITE_NAME}" -purge_suite_platform "${CYLC_TEST_PLATFORM}" "${SUITE_NAME}" -purge_suite "${SUITE_NAME}" -exit diff --git a/tests/functional/remote/03-install-target.t b/tests/functional/remote/02-install-target.t similarity index 100% rename from tests/functional/remote/03-install-target.t rename to tests/functional/remote/02-install-target.t diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 3c77dfe4fed..439071efa38 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -465,7 +465,7 @@ def test_rsync_includes_will_not_accept_sub_directories(): [[dependencies]] graph = "blah => deeblah" [scheduler] - includes = dir/, dir2/subdir2/, file1, file2 + install = dir/, dir2/subdir2/, file1, file2 """ with TemporaryDirectory() as temp_dir: flow_cylc = Path(temp_dir, "flow.cylc") @@ -487,7 +487,7 @@ def test_valid_rsync_includes_returns_correct_list(): [[dependencies]] graph = "blah => deeblah" [scheduler] - includes = dir/, dir2/, file1, file2 + install = dir/, dir2/, file1, file2 """ with TemporaryDirectory() as temp_dir: flow_cylc = Path(temp_dir, "flow.cylc")