From fea0711c9ac7bb636520f1b9c8744044386df43c Mon Sep 17 00:00:00 2001 From: Mel Hall <37735232+datamel@users.noreply.github.com> Date: Thu, 30 Apr 2020 17:12:57 +0100 Subject: [PATCH 1/8] Fixed cylc scan, fixed test and added test --- cylc/flow/network/scan.py | 15 +++++- cylc/flow/tests/network/test_scan.py | 77 +++++++++++++++++++++++----- 2 files changed, 78 insertions(+), 14 deletions(-) diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py index 2046538dba0..f4377043ac1 100644 --- a/cylc/flow/network/scan.py +++ b/cylc/flow/network/scan.py @@ -228,6 +228,7 @@ def get_scan_items_from_fs( Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for active, or all (active plus registered but dormant), suites. + Note when active_only, suites run in cylc 7 or less will not be returned. Yields: tuple - (reg, host, port, pub_port, api) @@ -271,9 +272,19 @@ def get_scan_items_from_fs( # Choose only suites with .service and matching filter if active_only: + # Skip suites running with cylc version < 8 try: - contact_data = load_contact_file( - reg, owner) + contact_data = load_contact_file(reg, owner) + cylc_version = contact_data[ContactFileFields.VERSION] + stripped_cylc_version = int(cylc_version[:1]) + if (stripped_cylc_version < 8): + LOG.debug( + f"Suite \"{reg}\" is running in Cylc version " + f"{cylc_version}, not Cylc 8 and will not be " + f"displayed." + ) + continue + except (SuiteServiceFileError, IOError, TypeError, ValueError): continue yield ( diff --git a/cylc/flow/tests/network/test_scan.py b/cylc/flow/tests/network/test_scan.py index 7a5dab0f23c..4da875f3f59 100644 --- a/cylc/flow/tests/network/test_scan.py +++ b/cylc/flow/tests/network/test_scan.py @@ -15,6 +15,7 @@ # along with this program. If not, see . import re +import os.path from collections import namedtuple from pathlib import Path from tempfile import TemporaryDirectory @@ -116,9 +117,9 @@ def test_get_scan_items_from_fs_with_owner_do_not_descend( Args: mocked_getpwall (object): Mocked pwd.getpwall - mocked_get_suite_src_dir (oject): + mocked_get_suite_src_dir (object): Mocked suite_files.get_suite_src_dir - mocked_get_suite_title (oject): + mocked_get_suite_title (object): Mocked suite_files.get_suite_title """ with TemporaryDirectory() as homedir: @@ -151,7 +152,6 @@ def test_get_scan_items_from_fs_with_owner_active_only( mocked_contact_file_fields (object): mocked ContactFileFields mocked_load_contact_file (function): mocked load_contact_file """ - # mock sr with TemporaryDirectory() as homedir: # mock pwd.getpwall mocked_getpwall.return_value = [ @@ -161,8 +161,9 @@ def test_get_scan_items_from_fs_with_owner_active_only( mocked_contact_file_fields.HOST = 'host' mocked_contact_file_fields.PORT = 'port' mocked_contact_file_fields.PUBLISH_PORT = 'pub_port' + mocked_contact_file_fields.VERSION = 'version' - # mock srv_files_mgr.load_contact_file + # mock suite_files.load_contact_file def my_load_contact_file(reg, _): if reg == 'good': return { @@ -170,6 +171,7 @@ def my_load_contact_file(reg, _): 'port': 9999, 'pub_port': 1234, 'api': str(API), + 'version': '8' } else: raise SuiteServiceFileError(reg) @@ -178,16 +180,67 @@ def my_load_contact_file(reg, _): for suite_name in ["good", "bad", "ugly"]: suite_directory = Path(homedir, 'cylc-run', suite_name) suite_directory.mkdir(parents=True) - # mock srv_files_mgr.load_contact_file - owner_pattern = re.compile(pattern="^.oo.$") - suites = list(get_scan_items_from_fs( - owner_pattern=owner_pattern, active_only=True)) - # will match blog/five but will stop once it finds log - self.assertEqual( - [('good', 'localhost', 9999, 1234, str(API))], suites) + owner_pattern = re.compile(pattern="^.oo.$") + suites = list(get_scan_items_from_fs( + owner_pattern=owner_pattern, active_only=True)) + self.assertEqual( + [('good', 'localhost', 9999, 1234, str(API))], suites) - # --- tests for re_compile_filters() + @patch("cylc.flow.network.scan.load_contact_file") + @patch("cylc.flow.network.scan.ContactFileFields") + @patch("cylc.flow.network.scan.getpwall") + def test_get_scan_items_from_fs_with_old_authentication( + self, mocked_getpwall, + mocked_contact_file_fields, + mocked_load_contact_file): + """Test that only active suites are returned if so requested. + Args: + mocked_getpwall (object): mocked pwd.getpwall + mocked_contact_file_fields (object): mocked ContactFileFields + mocked_load_contact_file (function): mocked load_contact_file + """ + # mock sr + with TemporaryDirectory() as homedir: + # mock pwd.getpwall + mocked_getpwall.return_value = [ + self.pwentry('/bin/bash', 'root', homedir), ] + + mocked_contact_file_fields.API = 'api' + mocked_contact_file_fields.HOST = 'host' + mocked_contact_file_fields.PORT = 'port' + mocked_contact_file_fields.PUBLISH_PORT = 'pub_port' + mocked_contact_file_fields.VERSION = 'version' + # mock suite_files.load_contact_file + def my_load_contact_file(reg, _): + if reg == 'suite_cylc_version_7': + return { + 'host': 'localhost', + 'port': 9999, + 'version': '7.8.4', + 'api': str(API), + } + if reg == 'suite_cylc_version_8': + return{ + 'host': 'localhost', + 'port': 9999, + 'pub_port': 1234, + 'api': str(API), + 'version': '8.0a1' + } + else: + raise SuiteServiceFileError(reg) + mocked_load_contact_file.side_effect = my_load_contact_file + for suite_name in ['suite_cylc_version_7', 'suite_cylc_version_8']: + suite_directory = Path(homedir, 'cylc-run', suite_name) + suite_directory.mkdir(parents=True) + owner_pattern = re.compile(pattern="^.oo.$") + suites = list(get_scan_items_from_fs(owner_pattern=owner_pattern)) + self.assertEqual( + [('suite_cylc_version_8', 'localhost', 9999, 1234, str(API))], + suites) + + # --- tests for re_compile_filters() def test_re_compile_filters_nones(self): """Test with no arguments provided.""" self.assertEqual((None, None), re_compile_filters()) From b6e2528c91f9049f2def14fc005bb1b4b5d5e7b6 Mon Sep 17 00:00:00 2001 From: Mel Hall <37735232+datamel@users.noreply.github.com> Date: Thu, 5 Mar 2020 16:51:31 +0000 Subject: [PATCH 2/8] Removed redundant authentication tests --- tests/authentication/00-identity.t | 138 ------------ .../{08-shared-fs.t => 00-shared-fs.t} | 0 .../reference.log | 0 .../{08-shared-fs => 00-shared-fs}/suite.rc | 0 tests/authentication/01-description.t | 149 ------------- ...ame-name.t => 01-remote-suite-same-name.t} | 0 .../reference.log | 0 .../suite.rc | 0 tests/authentication/02-state-totals.t | 190 ----------------- ...-stop-suite1.t => 02-suite2-stop-suite1.t} | 0 tests/authentication/03-full-read.t | 193 ----------------- tests/authentication/04-shutdown.t | 190 ----------------- tests/authentication/05-full-control.t | 199 ------------------ tests/authentication/06-suite-override.t | 179 ---------------- .../10-remote-suite-passphrase-cache.t | 67 ------ .../reference.log | 4 - .../10-remote-suite-passphrase-cache/suite.rc | 12 -- tests/authentication/override/suite.rc | 24 --- tests/cylc-message/00-ssh/suite.rc | 3 +- 19 files changed, 1 insertion(+), 1347 deletions(-) delete mode 100644 tests/authentication/00-identity.t rename tests/authentication/{08-shared-fs.t => 00-shared-fs.t} (100%) rename tests/authentication/{08-shared-fs => 00-shared-fs}/reference.log (100%) rename tests/authentication/{08-shared-fs => 00-shared-fs}/suite.rc (100%) delete mode 100644 tests/authentication/01-description.t rename tests/authentication/{09-remote-suite-same-name.t => 01-remote-suite-same-name.t} (100%) rename tests/authentication/{09-remote-suite-same-name => 01-remote-suite-same-name}/reference.log (100%) rename tests/authentication/{09-remote-suite-same-name => 01-remote-suite-same-name}/suite.rc (100%) delete mode 100644 tests/authentication/02-state-totals.t rename tests/authentication/{11-suite2-stop-suite1.t => 02-suite2-stop-suite1.t} (100%) delete mode 100644 tests/authentication/03-full-read.t delete mode 100644 tests/authentication/04-shutdown.t delete mode 100644 tests/authentication/05-full-control.t delete mode 100644 tests/authentication/06-suite-override.t delete mode 100755 tests/authentication/10-remote-suite-passphrase-cache.t delete mode 100644 tests/authentication/10-remote-suite-passphrase-cache/reference.log delete mode 100644 tests/authentication/10-remote-suite-passphrase-cache/suite.rc delete mode 100644 tests/authentication/override/suite.rc diff --git a/tests/authentication/00-identity.t b/tests/authentication/00-identity.t deleted file mode 100644 index 8e1f7b05dfc..00000000000 --- a/tests/authentication/00-identity.t +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Test authentication - privilege 'identity'. - -. "$(dirname "$0")/test_header" -set_test_number 17 - -install_suite "${TEST_NAME_BASE}" basic - -TEST_NAME="${TEST_NAME_BASE}-validate" -#TODOrun_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}" - -# Run the suite. -create_test_globalrc '' ' -[authentication] - public = identity' -cylc run "${SUITE_NAME}" -unset CYLC_CONF_PATH - -SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" -HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" -PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" - -skip 2 'anon auth not supported' # TODO -#run_ok "${TEST_NAME_BASE}-client-anon" \ -# cylc client -n --host="${HOST}" --port="${PORT}" 'identify' -#run_ok "${TEST_NAME_BASE}-client-anon.stdout" \ -# grep -qF "\"name\": \"${SUITE_NAME}\"" "${TEST_NAME_BASE}-client-anon.stdout" - -TEST_NAME="${TEST_NAME_BASE}-client-cylc" -run_ok "${TEST_NAME}" \ - cylc client -n --host="${HOST}" --port="${PORT}" "${SUITE_NAME}" 'identify' -cmp_json "${TEST_NAME}.stdout" "${TEST_NAME}.stdout" -c2 <<__JSON__ - {"name": "${SUITE_NAME}"} -__JSON__ - -TEST_NAME="${TEST_NAME_BASE}-client-cylc-bad-ping-task" -run_ok "${TEST_NAME}" cylc client "${SUITE_NAME}" 'ping_task' <<'__JSON__' - {"task_id": "elephant.1", "exists_only": true} -__JSON__ -cmp_json "${TEST_NAME}.stdout" "${TEST_NAME}.stdout" <<'__JSON__' - [false, "task not found"] -__JSON__ - -TEST_NAME="${TEST_NAME_BASE}-client-cylc-ping-task" -run_ok "${TEST_NAME}" cylc client "${SUITE_NAME}" 'ping_task' <<'__JSON__' - {"task_id": "foo.1", "exists_only": true} -__JSON__ -cmp_json "${TEST_NAME}.stdout" "${TEST_NAME}.stdout" <<'__JSON__' - [true, "task found"] -__JSON__ - -# Wait for first task 'foo' to fail. -cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ - --interval=1 --max-polls=10 || exit 1 - -skip 9 'anon auth not supported' # TODO -cylc stop --max-polls=20 --interval=1 "${SUITE_NAME}" -purge_suite "${SUITE_NAME}" -exit - -# Disable the suite passphrase (to leave us with public access privilege). -mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" - -# Check scan --full output. -cylc scan --comms-timeout=5 -f --color=never -n "${SUITE_NAME}" \ - >'scan-f.out' 2>'/dev/null' -cmp_ok scan-f.out << __END__ -${SUITE_NAME} ${USER}@localhost:${PORT} - (description and state totals withheld) -__END__ - -# Check scan --describe output. -cylc scan --comms-timeout=5 -d --color=never -n "${SUITE_NAME}" \ - >'scan-d.out' 2>'/dev/null' -cmp_ok scan-d.out << __END__ -${SUITE_NAME} ${USER}@localhost:${PORT} - (description and state totals withheld) -__END__ - -# Check scan --raw output. -cylc scan --comms-timeout=5 -t raw --color=never -n "${SUITE_NAME}" \ - >'scan-r.out' 2>'/dev/null' -cmp_ok scan-r.out << __END__ -${SUITE_NAME}|${USER}|localhost|port|${PORT} -__END__ - -# Check scan --json output. -cylc scan --comms-timeout=5 -t json --color=never -n "${SUITE_NAME}" \ - >'scan-j.out' 2>'/dev/null' -cmp_json 'scan-j.out' 'scan-j.out' << __END__ -[ - [ - "localhost", - ${PORT}, - { - "owner":"${USER}", - "version": "$(cylc version)", - "name":"${SUITE_NAME}" - } - ] -] -__END__ - -# "cylc show" should be denied. -TEST_NAME="${TEST_NAME_BASE}-show" -run_fail "${TEST_NAME}" cylc show "${SUITE_NAME}" -cylc log "${SUITE_NAME}" > suite.log1 -grep_ok "\[client-connect\] DENIED (privilege 'identity' < 'description') ${USER}@.*:cylc-show" suite.log1 - -# Commands should be denied. -TEST_NAME="${TEST_NAME_BASE}-stop" -run_fail "${TEST_NAME}" cylc stop "${SUITE_NAME}" -cylc log "${SUITE_NAME}" > suite.log2 -grep_ok "\[client-connect\] DENIED (privilege 'identity' < 'shutdown') ${USER}@.*:cylc-stop" suite.log2 - -# Restore the passphrase. -mv "${SRV_D}/passphrase.DIS" "${SRV_D}/passphrase" - -# Stop and purge the suite. -cylc stop --max-polls=20 --interval=1 "${SUITE_NAME}" -purge_suite "${SUITE_NAME}" -exit diff --git a/tests/authentication/08-shared-fs.t b/tests/authentication/00-shared-fs.t similarity index 100% rename from tests/authentication/08-shared-fs.t rename to tests/authentication/00-shared-fs.t diff --git a/tests/authentication/08-shared-fs/reference.log b/tests/authentication/00-shared-fs/reference.log similarity index 100% rename from tests/authentication/08-shared-fs/reference.log rename to tests/authentication/00-shared-fs/reference.log diff --git a/tests/authentication/08-shared-fs/suite.rc b/tests/authentication/00-shared-fs/suite.rc similarity index 100% rename from tests/authentication/08-shared-fs/suite.rc rename to tests/authentication/00-shared-fs/suite.rc diff --git a/tests/authentication/01-description.t b/tests/authentication/01-description.t deleted file mode 100644 index d49c1312a31..00000000000 --- a/tests/authentication/01-description.t +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Test authentication - privilege 'description'. - -. "$(dirname "$0")/test_header" -skip_all 'anon auth not supported' # TODO -set_test_number 10 - -install_suite "${TEST_NAME_BASE}" basic - -TEST_NAME="${TEST_NAME_BASE}-validate" -run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}" - -# Run the suite. -create_test_globalrc '' ' -[authentication] - public = description' -cylc run "${SUITE_NAME}" -unset CYLC_CONF_PATH - -# Wait for first task 'foo' to fail. -cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ - --interval=1 --max-polls=10 || exit 1 - -# Disable the suite passphrase (to leave us with public access privilege). -SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" -mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" - -# Check scan --full output. -HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" -PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -f --color=never -n "${SUITE_NAME}" \ - >'scan-f.out' #2>'/dev/null' -cmp_ok 'scan-f.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - Group: - (no group) - Description: - "Stalls when the first task fails. - Here we test out a multi-line description!" - URL: - (no URL) - another_metadata: - "1" - custom_metadata: - "something_custom" - (state totals withheld) -__END__ - -# Check scan --describe output. -cylc scan --comms-timeout=5 -d --color=never -n "${SUITE_NAME}" \ - >'scan-d.out' #2>'/dev/null' -cmp_ok 'scan-d.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - Group: - (no group) - Description: - "Stalls when the first task fails. - Here we test out a multi-line description!" - URL: - (no URL) - another_metadata: - "1" - custom_metadata: - "something_custom" -__END__ - -# Check scan --raw output. -cylc scan --comms-timeout=5 -t raw --color=never -n "${SUITE_NAME}" \ - >'scan-r.out' #2>'/dev/null' -cmp_ok 'scan-r.out' <<__END__ -${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 -${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. \ -Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. -__END__ - -# Check scan --json output. -cylc scan --comms-timeout=5 -t json --color=never -n "${SUITE_NAME}" \ - >'scan-j.out' #2>'/dev/null' -cmp_json 'scan-j.out' 'scan-j.out' <<__END__ -[ - [ - "${HOST}", - "${PORT}", - { - "group":"", - "version":"$(cylc version)", - "title":"Authentication test suite.", - "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", - "meta":{ - "group":"", - "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", - "title":"Authentication test suite.", - "URL":"", - "another_metadata":"1", - "custom_metadata":"something_custom" - }, - "owner":"${USER}", - "name":"${SUITE_NAME}" - } - ] -] -__END__ - -# "cylc show" (suite info) OK. -TEST_NAME="${TEST_NAME_BASE}-show1" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" - -# "cylc show" (task info) should be denied. -TEST_NAME="${TEST_NAME_BASE}-show2" -run_fail "${TEST_NAME}" cylc show "${SUITE_NAME}" foo.1 -cylc log "${SUITE_NAME}" > suite.log1 -grep_ok "\[client-connect] DENIED (privilege 'description' < 'READ') ${USER}@.*:cylc-show" suite.log1 - -# Commands should be denied. -TEST_NAME="${TEST_NAME_BASE}-stop" -run_fail "${TEST_NAME}" cylc stop "${SUITE_NAME}" -cylc log "${SUITE_NAME}" > suite.log2 -grep_ok "\[client-connect] DENIED (privilege 'description' < 'SHUTDOWN') ${USER}@.*:cylc-stop" suite.log2 - -# Restore the passphrase. -mv "${SRV_D}/passphrase.DIS" "${SRV_D}/passphrase" - -# Stop and purge the suite. -cylc stop --max-polls=20 --interval=1 "${SUITE_NAME}" -purge_suite "${SUITE_NAME}" -exit diff --git a/tests/authentication/09-remote-suite-same-name.t b/tests/authentication/01-remote-suite-same-name.t similarity index 100% rename from tests/authentication/09-remote-suite-same-name.t rename to tests/authentication/01-remote-suite-same-name.t diff --git a/tests/authentication/09-remote-suite-same-name/reference.log b/tests/authentication/01-remote-suite-same-name/reference.log similarity index 100% rename from tests/authentication/09-remote-suite-same-name/reference.log rename to tests/authentication/01-remote-suite-same-name/reference.log diff --git a/tests/authentication/09-remote-suite-same-name/suite.rc b/tests/authentication/01-remote-suite-same-name/suite.rc similarity index 100% rename from tests/authentication/09-remote-suite-same-name/suite.rc rename to tests/authentication/01-remote-suite-same-name/suite.rc diff --git a/tests/authentication/02-state-totals.t b/tests/authentication/02-state-totals.t deleted file mode 100644 index a00768f33c0..00000000000 --- a/tests/authentication/02-state-totals.t +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Test authentication - privilege 'state-totals'. - -. "$(dirname "$0")/test_header" -skip_all 'anon auth not supported' # TODO -set_test_number 10 - -install_suite "${TEST_NAME_BASE}" basic - -TEST_NAME="${TEST_NAME_BASE}-validate" -run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}" - -# Run the suite. -create_test_globalrc '' ' -[authentication] - public = state-totals' -cylc run "${SUITE_NAME}" -unset CYLC_CONF_PATH - -# Wait for first task 'foo' to fail. -cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ - --interval=1 --max-polls=10 || exit 1 - -# Disable the suite passphrase (to leave us with public access privilege). -SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" -mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" - -# Check scan --full output. -HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" -PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -f --color=never -n "${SUITE_NAME}" >'scan-f.out' -cmp_ok 'scan-f.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - Group: - (no group) - Description: - "Stalls when the first task fails. - Here we test out a multi-line description!" - URL: - (no URL) - another_metadata: - "1" - custom_metadata: - "something_custom" - Task state totals: - failed:1 waiting:2 - 1 failed:1 waiting:1 - 2 waiting:1 -__END__ - -# Check scan --describe output. -cylc scan --comms-timeout=5 -d --color=never -n "${SUITE_NAME}" \ - >'scan-d.out' 2>'/dev/null' -cmp_ok 'scan-d.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - Group: - (no group) - Description: - "Stalls when the first task fails. - Here we test out a multi-line description!" - URL: - (no URL) - another_metadata: - "1" - custom_metadata: - "something_custom" -__END__ - -# Check scan --raw output. -cylc scan --comms-timeout=5 -t raw --color=never -n "${SUITE_NAME}" \ - >'scan-r.out' 2>'/dev/null' -cmp_ok 'scan-r.out' <<__END__ -${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 -${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 -${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 -${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 -__END__ - -# Check scan --json output. -cylc scan --comms-timeout=5 -t json --color=never -n "${SUITE_NAME}" \ - >'scan-j.out' 2>'/dev/null' -cmp_json 'scan-j.out' 'scan-j.out' <<__END__ -[ - [ - "${HOST}", - "${PORT}", - { - "group":"", - "version":"$(cylc version)", - "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", - "title":"Authentication test suite.", - "states":[ - { - "failed":1, - "waiting":2 - }, - { - "1":{ - "failed":1, - "waiting":1 - }, - "2":{ - "waiting":1 - } - } - ], - "tasks-by-state":{ - "failed":[ - [ - "", - "foo", - "1" - ] - ], - "waiting":[ - [ - 0, - "pub", - "2" - ], - [ - 0, - "bar", - "1" - ] - ] - }, - "meta":{ - "group":"", - "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", - "title":"Authentication test suite.", - "URL":"", - "another_metadata":"1", - "custom_metadata":"something_custom" - }, - "owner":"${USER}", - "update-time":"", - "name":"${SUITE_NAME}" - } - ] -] -__END__ - -# "cylc show" (suite info) OK. -TEST_NAME="${TEST_NAME_BASE}-show1" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" - -# "cylc show" (task info) should be denied. -TEST_NAME="${TEST_NAME_BASE}-show2" -run_fail "${TEST_NAME}" cylc show "${SUITE_NAME}" foo.1 -cylc log "${SUITE_NAME}" > suite.log1 -grep_ok "\[client-connect] DENIED (privilege 'state-totals' < 'READ') ${USER}@.*:cylc-show" suite.log1 - -# Commands should be denied. -TEST_NAME="${TEST_NAME_BASE}-stop" -run_fail "${TEST_NAME}" cylc stop "${SUITE_NAME}" -cylc log "${SUITE_NAME}" > suite.log2 -grep_ok "\[client-connect] DENIED (privilege 'state-totals' < 'SHUTDOWN') ${USER}@.*:cylc-stop" suite.log2 - -# Restore the passphrase. -mv "${SRV_D}/passphrase.DIS" "${SRV_D}/passphrase" - -# Stop and purge the suite. -cylc stop --max-polls=20 --interval=1 "${SUITE_NAME}" -purge_suite "${SUITE_NAME}" -exit diff --git a/tests/authentication/11-suite2-stop-suite1.t b/tests/authentication/02-suite2-stop-suite1.t similarity index 100% rename from tests/authentication/11-suite2-stop-suite1.t rename to tests/authentication/02-suite2-stop-suite1.t diff --git a/tests/authentication/03-full-read.t b/tests/authentication/03-full-read.t deleted file mode 100644 index 0847ac600ba..00000000000 --- a/tests/authentication/03-full-read.t +++ /dev/null @@ -1,193 +0,0 @@ -#!/bin/bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Test authentication - privilege 'read'. - -. "$(dirname "$0")/test_header" -skip_all 'anon auth not supported' # TODO -set_test_number 11 - -install_suite "${TEST_NAME_BASE}" basic - -TEST_NAME="${TEST_NAME_BASE}-validate" -run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}" - -# Run the suite. -create_test_globalrc '' ' -[authentication] - public = read' -cylc run "${SUITE_NAME}" -unset CYLC_CONF_PATH - -# Wait for first task 'foo' to fail. -cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ - --interval=1 --max-polls=10 || exit 1 - -# Disable the suite passphrase (to leave us with public access privilege). -SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" -mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" - -# Check scan --full output. -HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" -PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=5 -f --color=never -n "${SUITE_NAME}" \ - >'scan-f.out' 2>'/dev/null' -cmp_ok 'scan-f.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - Group: - (no group) - Description: - "Stalls when the first task fails. - Here we test out a multi-line description!" - URL: - (no URL) - another_metadata: - "1" - custom_metadata: - "something_custom" - Task state totals: - failed:1 waiting:2 - 1 failed:1 waiting:1 - 2 waiting:1 -__END__ - -# Check scan --describe output. -cylc scan --comms-timeout=5 -d --color=never -n "${SUITE_NAME}" \ - >'scan-d.out' 2>'/dev/null' -cmp_ok 'scan-d.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - Group: - (no group) - Description: - "Stalls when the first task fails. - Here we test out a multi-line description!" - URL: - (no URL) - another_metadata: - "1" - custom_metadata: - "something_custom" -__END__ - -# Check scan --raw output. -cylc scan --comms-timeout=5 -t raw --color=never -n "${SUITE_NAME}" \ - >'scan-r.out' 2>'/dev/null' -cmp_ok 'scan-r.out' <<__END__ -${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 -${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 -${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 -${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 -__END__ - -# Check scan --json output. -cylc scan --comms-timeout=5 -t json --color=never -n "${SUITE_NAME}" \ - >'scan-j.out' 2>'/dev/null' -cmp_json 'scan-j.out' 'scan-j.out' <<__END__ -[ - [ - "${HOST}", - "${PORT}", - { - "group":"", - "version":"$(cylc version)", - "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", - "title":"Authentication test suite.", - "states":[ - { - "failed":1, - "waiting":2 - }, - { - "1":{ - "failed":1, - "waiting":1 - }, - "2":{ - "waiting":1 - } - } - ], - "tasks-by-state":{ - "failed":[ - [ - "", - "foo", - "1" - ] - ], - "waiting":[ - [ - 0, - "pub", - "2" - ], - [ - 0, - "bar", - "1" - ] - ] - }, - "meta":{ - "group":"", - "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", - "title":"Authentication test suite.", - "URL":"", - "another_metadata":"1", - "custom_metadata":"something_custom" - }, - "owner":"${USER}", - "update-time":"", - "name":"${SUITE_NAME}" - } - ] -] -__END__ - -# "cylc show" (suite info) OK. -TEST_NAME="${TEST_NAME_BASE}-show1" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" -cylc log "${SUITE_NAME}" > suite.log1 -grep_ok "\[client-command] get_suite_info ${USER}@.*:cylc-show" suite.log1 - -# "cylc show" (task info) OK. -TEST_NAME="${TEST_NAME_BASE}-show2" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" foo -cylc log "${SUITE_NAME}" > suite.log2 -grep_ok "\[client-command] get_task_info ${USER}@.*:cylc-show" suite.log2 - -# Commands should be denied. -TEST_NAME="${TEST_NAME_BASE}-stop" -run_fail "${TEST_NAME}" cylc stop "${SUITE_NAME}" -cylc log "${SUITE_NAME}" > suite.log3 -grep_ok "\[client-connect] DENIED (privilege 'READ' < 'shutdown') ${USER}@.*:cylc-stop" suite.log3 - -# Restore the passphrase. -mv "${SRV_D}/passphrase.DIS" "${SRV_D}/passphrase" - -# Stop and purge the suite. -cylc stop --max-polls=20 --interval=1 "${SUITE_NAME}" -purge_suite "${SUITE_NAME}" -exit diff --git a/tests/authentication/04-shutdown.t b/tests/authentication/04-shutdown.t deleted file mode 100644 index 672602d0813..00000000000 --- a/tests/authentication/04-shutdown.t +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Test authentication - privilege 'shutdown'. - -. "$(dirname "$0")/test_header" -skip_all 'anon auth not supported' # TODO -set_test_number 12 - -install_suite "${TEST_NAME_BASE}" basic - -TEST_NAME="${TEST_NAME_BASE}-validate" -run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}" - -# Run the suite. -create_test_globalrc '' ' -[authentication] - public = shutdown' -cylc run "${SUITE_NAME}" -unset CYLC_CONF_PATH - -# Wait for first task 'foo' to fail. -cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ - --interval=1 --max-polls=10 || exit 1 - -# Disable the suite passphrase (to leave us with public access privilege). -SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" -mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" - -HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" -PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" - -cylc scan --comms-timeout=5 -f --color=never -n "${SUITE_NAME}" \ - >'scan-f.out' 2>'/dev/null' -cmp_ok 'scan-f.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - Group: - (no group) - Description: - "Stalls when the first task fails. - Here we test out a multi-line description!" - URL: - (no URL) - another_metadata: - "1" - custom_metadata: - "something_custom" - Task state totals: - failed:1 waiting:2 - 1 failed:1 waiting:1 - 2 waiting:1 -__END__ - -cylc scan --comms-timeout=5 -d --color=never -n "${SUITE_NAME}" \ - >'scan-d.out' 2>'/dev/null' -cmp_ok 'scan-d.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - Group: - (no group) - Description: - "Stalls when the first task fails. - Here we test out a multi-line description!" - URL: - (no URL) - another_metadata: - "1" - custom_metadata: - "something_custom" -__END__ - -# Check scan --raw output. -cylc scan --comms-timeout=5 -t raw --color=never -n "${SUITE_NAME}" \ - >'scan-r.out' 2>'/dev/null' -cmp_ok 'scan-r.out' <<__END__ -${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 -${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 -${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 -${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 -__END__ - -# Check scan --json output. -cylc scan --comms-timeout=5 -t json --color=never -n "${SUITE_NAME}" \ - >'scan-j.out' 2>'/dev/null' -cmp_json 'scan-j.out' 'scan-j.out' <<__END__ -[ - [ - "${HOST}", - "${PORT}", - { - "group":"", - "version":"$(cylc version)", - "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", - "title":"Authentication test suite.", - "states":[ - { - "failed":1, - "waiting":2 - }, - { - "1":{ - "failed":1, - "waiting":1 - }, - "2":{ - "waiting":1 - } - } - ], - "tasks-by-state":{ - "failed":[ - [ - "", - "foo", - "1" - ] - ], - "waiting":[ - [ - 0, - "pub", - "2" - ], - [ - 0, - "bar", - "1" - ] - ] - }, - "meta":{ - "group":"", - "description":"Stalls when the first task fails.\n Here we test out a multi-line description!", - "title":"Authentication test suite.", - "URL":"", - "another_metadata":"1", - "custom_metadata":"something_custom" - }, - "owner":"${USER}", - "update-time":"", - "name":"${SUITE_NAME}" - } - ] -] -__END__ - -# "cylc show" (suite info) OK. -TEST_NAME="${TEST_NAME_BASE}-show1" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" -cylc log "${SUITE_NAME}" > suite.log1 -grep_ok "\[client-command] get_suite_info ${USER}@.*:cylc-show" suite.log1 - -# "cylc show" (task info) OK. -TEST_NAME="${TEST_NAME_BASE}-show2" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" foo -cylc log "${SUITE_NAME}" > suite.log2 -grep_ok "\[client-command] get_task_info ${USER}@.*:cylc-show" suite.log2 - -# Commands (other than shutdown) should be denied. -TEST_NAME="${TEST_NAME_BASE}-trigger" -run_fail "${TEST_NAME}" cylc trigger "${SUITE_NAME}" foo 1 -cylc log "${SUITE_NAME}" > suite.log3 -grep_ok "\[client-connect] DENIED (privilege 'shutdown' < 'full-control') ${USER}@.*:cylc-trigger" suite.log3 - -# Stop OK (without the passphrase!). -TEST_NAME="${TEST_NAME_BASE}-stop" -run_ok "${TEST_NAME}" cylc stop --debug --max-polls=20 --interval=1 "${SUITE_NAME}" -purge_suite "${SUITE_NAME}" -exit diff --git a/tests/authentication/05-full-control.t b/tests/authentication/05-full-control.t deleted file mode 100644 index 60d49e32b42..00000000000 --- a/tests/authentication/05-full-control.t +++ /dev/null @@ -1,199 +0,0 @@ -#!/bin/bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Test authentication - privilege 'full-control' (with passphrase). - -. "$(dirname "$0")/test_header" -set_test_number 12 - -API_VERSION="$(python -c 'from cylc.flow.network import API; print(API)')" - -install_suite "${TEST_NAME_BASE}" 'basic' - -TEST_NAME="${TEST_NAME_BASE}-validate" -run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}" - -# Run the suite. -# Set public auth low to test that passphrase gives full control -create_test_globalrc '' ' -[authentication] - public = identity' -cylc run "${SUITE_NAME}" -unset CYLC_CONF_PATH - -# Wait for first task 'foo' to fail. -poll_grep 'CYLC_JOB_EXIT' "${SUITE_RUN_DIR}/log/job/1/foo/01/job.status" -cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ - --interval=1 --max-polls=10 || exit 1 - -# Check scan --full output. -SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" -HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" -PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" -PUBLISH_PORT="$(sed -n 's/^CYLC_SUITE_PUBLISH_PORT=//p' "${SRV_D}/contact")" -cylc scan --comms-timeout=10 -f --color=never -n "${SUITE_NAME}" >'scan-f.out' -cmp_ok 'scan-f.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} ${USER}@${HOST}:${PUBLISH_PORT} - Title: - Authentication test suite. - Description: - Stalls when the first task fails. - Here we test out a multi-line description! - Group: - (no Group) - API: - ${API_VERSION} - URL: - (no URL) - another_metadata: - 1 - custom_metadata: - something_custom - Task state totals: - failed:1 waiting:2 - 1 failed:1 waiting:1 - 2 waiting:1 -__END__ - -# Check scan --describe output. -cylc scan --comms-timeout=10 -d --color=never -n "${SUITE_NAME}" >'scan-d.out' -cmp_ok 'scan-d.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - Authentication test suite. - Description: - Stalls when the first task fails. - Here we test out a multi-line description! - Group: - (no Group) - API: - ${API_VERSION} - URL: - (no URL) - another_metadata: - 1 - custom_metadata: - something_custom -__END__ - -# Check scan --raw output. -cylc scan --comms-timeout=10 -f -t raw --color=never -n "${SUITE_NAME}" \ - >'scan-r.out' -cmp_ok 'scan-r.out' <<__END__ -${SUITE_NAME}|${USER}|${HOST}|port|${PORT}|publish-port|${PUBLISH_PORT} -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Here we test out a multi-line description! -${SUITE_NAME}|${USER}|${HOST}|group| -${SUITE_NAME}|${USER}|${HOST}|API|${API_VERSION} -${SUITE_NAME}|${USER}|${HOST}|URL| -${SUITE_NAME}|${USER}|${HOST}|another_metadata|1 -${SUITE_NAME}|${USER}|${HOST}|custom_metadata|something_custom -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 -${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 -${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 -__END__ - -# Check scan --json output. -cylc scan --comms-timeout=10 -f -t json --color=never -n "${SUITE_NAME}" \ - >'scan-j.out' -sed -i -r 's/[0-9\.]{10,}/""/' 'scan-j.out' -cmp_json 'scan-j.out' 'scan-j.out' <<__END__ -[ - [ - "${SUITE_NAME}", - "${HOST}", - "${PORT}", - "${PUBLISH_PORT}", - "${API_VERSION}", - { - "version":"$(cylc version)", - "states":[ - { - "failed":1, - "waiting":2 - }, - { - "1":{ - "failed":1, - "waiting":1 - }, - "2":{ - "waiting":1 - } - } - ], - "tasks-by-state":{ - "failed":[ - [ - "", - "foo", - "1" - ] - ], - "waiting":[ - [ - 0, - "pub", - "2" - ], - [ - 0, - "bar", - "1" - ] - ] - }, - "meta":{ - "group":"", - "description":"Stalls when the first task fails.\nHere we test out a multi-line description!", - "title":"Authentication test suite.", - "URL":"", - "another_metadata":"1", - "custom_metadata":"something_custom" - }, - "owner":"${USER}", - "update-time":"", - "name":"${SUITE_NAME}" - } - ] -] -__END__ - -# "cylc show" (suite info) OK. -TEST_NAME="${TEST_NAME_BASE}-show1" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" -cylc log "${SUITE_NAME}" > suite.log1 -grep_ok "\\[client-command\\] get_suite_info ${USER}@.*cylc-show" 'suite.log1' - -# "cylc show" (task info) OK. -TEST_NAME="${TEST_NAME_BASE}-show2" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" foo -cylc log "${SUITE_NAME}" > suite.log2 -grep_ok "\\[client-command\\] get_task_info ${USER}@.*cylc-show" 'suite.log2' - -# Commands OK. -# (Reset to same state). -TEST_NAME="${TEST_NAME_BASE}-trigger" -run_ok "${TEST_NAME}" cylc reset "${SUITE_NAME}" -s failed foo 1 -cylc log "${SUITE_NAME}" > suite.log3 -grep_ok "\\[client-command\\] reset_task_states ${USER}@.*cylc-reset" 'suite.log3' - -# Shutdown and purge. -TEST_NAME="${TEST_NAME_BASE}-stop" -run_ok "${TEST_NAME}" cylc stop --max-polls=20 --interval=1 "${SUITE_NAME}" -purge_suite "${SUITE_NAME}" -exit diff --git a/tests/authentication/06-suite-override.t b/tests/authentication/06-suite-override.t deleted file mode 100644 index 2eede588c69..00000000000 --- a/tests/authentication/06-suite-override.t +++ /dev/null @@ -1,179 +0,0 @@ -#!/bin/bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Test authentication - privilege 'shutdown'. -# (Suite overrides global privilege 'identity'.) - -. "$(dirname "$0")/test_header" -skip_all 'anon auth not supported' # TODO -set_test_number 12 - -install_suite "${TEST_NAME_BASE}" override - -TEST_NAME="${TEST_NAME_BASE}-validate" -run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}" - -# Run the suite. -create_test_globalrc '' ' -[authentication] - public = identity' -cylc run "${SUITE_NAME}" -unset CYLC_CONF_PATH - -# Wait for first task 'foo' to fail. -cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ - --interval=1 --max-polls=10 || exit 1 - -# Disable the suite passphrase (to leave us with public access privilege). -SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" -mv "${SRV_D}/passphrase" "${SRV_D}/passphrase.DIS" - -HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" -PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" - -cylc scan --comms-timeout=5 -f --color=never -n "${SUITE_NAME}" \ - >'scan-f.out' 2>'/dev/null' -cmp_ok 'scan-f.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - URL: - (no URL) - Group: - (no group) - Description: - "Stalls when the first task fails. - Suite overrides global authentication settings." - Task state totals: - failed:1 waiting:2 - 1 failed:1 waiting:1 - 2 waiting:1 -__END__ - -cylc scan --comms-timeout=5 -d --color=never -n "${SUITE_NAME}" \ - >'scan-d.out' 2>'/dev/null' -cmp_ok 'scan-d.out' <<__END__ -${SUITE_NAME} ${USER}@${HOST}:${PORT} - Title: - "Authentication test suite." - URL: - (no URL) - Group: - (no group) - Description: - "Stalls when the first task fails. - Suite overrides global authentication settings." -__END__ - -# Check scan --raw output. -cylc scan --comms-timeout=5 -t raw --color=never -n "${SUITE_NAME}" \ - >'scan-r.out' 2>'/dev/null' -cmp_ok 'scan-r.out' <<__END__ -${SUITE_NAME}|${USER}|${HOST}|port|${PORT} -${SUITE_NAME}|${USER}|${HOST}|description|Stalls when the first task fails. Suite overrides global authentication settings. -${SUITE_NAME}|${USER}|${HOST}|title|Authentication test suite. -${SUITE_NAME}|${USER}|${HOST}|states|failed:1 waiting:2 -${SUITE_NAME}|${USER}|${HOST}|states:1|failed:1 waiting:1 -${SUITE_NAME}|${USER}|${HOST}|states:2|waiting:1 -__END__ - -# Check scan --json output. -cylc scan --comms-timeout=5 -t json --color=never -n "${SUITE_NAME}" \ - >'scan-j.out' 2>'/dev/null' -cmp_json 'scan-j.out' 'scan-j.out' <<__END__ -[ - [ - "${HOST}", - "${PORT}", - { - "group":"", - "version":"$(cylc version)", - "description":"Stalls when the first task fails.\nSuite overrides global authentication settings.", - "title":"Authentication test suite.", - "states":[ - { - "failed":1, - "waiting":2 - }, - { - "1":{ - "failed":1, - "waiting":1 - }, - "2":{ - "waiting":1 - } - } - ], - "tasks-by-state":{ - "failed":[ - [ - "", - "foo", - "1" - ] - ], - "waiting":[ - [ - 0, - "pub", - "2" - ], - [ - 0, - "bar", - "1" - ] - ] - }, - "meta": { - "URL": "", - "group": "", - "description": "Stalls when the first task fails.\nSuite overrides global authentication settings.", - "title": "Authentication test suite." - }, - "owner":"${USER}", - "update-time":"", - "name":"${SUITE_NAME}" - } - ] -] -__END__ - -# "cylc show" (suite info) OK. -TEST_NAME="${TEST_NAME_BASE}-show1" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" -cylc log "${SUITE_NAME}" > suite.log1 -grep_ok "\[client-command] get_suite_info ${USER}@.*:cylc-show" suite.log1 - -# "cylc show" (task info) OK. -TEST_NAME="${TEST_NAME_BASE}-show2" -run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" foo -cylc log "${SUITE_NAME}" > suite.log2 -grep_ok "\[client-command] get_task_info ${USER}@.*:cylc-show" suite.log2 - -# Commands (other than shutdown) should be denied. -TEST_NAME="${TEST_NAME_BASE}-trigger" -run_fail "${TEST_NAME}" cylc trigger "${SUITE_NAME}" foo 1 -cylc log "${SUITE_NAME}" > suite.log3 -grep_ok "\[client-connect] DENIED (privilege 'shutdown' < 'full-control') ${USER}@.*:cylc-trigger" suite.log3 - -# Stop OK (without the passphrase!). -TEST_NAME="${TEST_NAME_BASE}-stop" -run_ok "${TEST_NAME}" cylc stop --max-polls=20 --interval=1 "${SUITE_NAME}" -purge_suite "${SUITE_NAME}" -exit diff --git a/tests/authentication/10-remote-suite-passphrase-cache.t b/tests/authentication/10-remote-suite-passphrase-cache.t deleted file mode 100755 index 4c8d6e8df33..00000000000 --- a/tests/authentication/10-remote-suite-passphrase-cache.t +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -#------------------------------------------------------------------------------- -# Test cache of passphrase for a suite running on a remote host with no shared -# HOME file system, but with non-interactive SSH access. -# This test assumes compatible version of cylc is available on the configured -# remote host. -export CYLC_TEST_IS_GENERIC=false -. "$(dirname "$0")/test_header" -set_test_remote_host -set_test_number 5 - -SSH_OPTS='-oBatchMode=yes -oConnectTimeout=5' -SUITE_NAME="cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}" - -# shellcheck disable=SC2029,SC2086 -ssh ${SSH_OPTS} "${CYLC_TEST_HOST}" mkdir -p "cylc-run/${SUITE_NAME}" -# shellcheck disable=SC2086 -scp ${SSH_OPTS} -pqr "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}/"* \ - "${CYLC_TEST_HOST}:cylc-run/${SUITE_NAME}" -cylc register --host="${CYLC_TEST_HOST}" "${SUITE_NAME}" "cylc-run/${SUITE_NAME}" -run_ok "${TEST_NAME_BASE}-validate" \ - cylc validate --host="${CYLC_TEST_HOST}" "${SUITE_NAME}" - -cylc run --debug --no-detach --host="${CYLC_TEST_HOST}" --reference-test "${SUITE_NAME}" \ - 1>'out' 2>'err' & -SUITE_PID=$! - -# Wait until the task failed -poll_grep_suite_log 't1.19700101T0000Z.*failed' - -run_ok "${TEST_NAME_BASE}-broadcast" \ - cylc broadcast --host="${CYLC_TEST_HOST}" "${SUITE_NAME}" \ - -n 't1' -p '1970' -s '[environment]CYLC_TEST_VAR_FOO=foo' - -run_ok "${TEST_NAME_BASE}-trigger" \ - cylc trigger --host="${CYLC_TEST_HOST}" "${SUITE_NAME}" '*:failed' - -# Check that we have cached the passphrase -CACHED="${HOME}/.cylc/auth/${USER}@${CYLC_TEST_HOST}/${SUITE_NAME}" -exists_ok "${CACHED}/passphrase" - -run_ok "${TEST_NAME_BASE}" wait "${SUITE_PID}" - -purge_suite_remote "${CYLC_TEST_HOST}" "${SUITE_NAME}" -rm -fr "${CACHED}" -( - cd "${HOME}/.cylc/auth/" \ - && rmdir -p "${USER}@${CYLC_TEST_HOST}/$(dirname "${SUITE_NAME}")" 2>'/dev/null' -) || true -rmdir "${HOME}/.cylc/auth/" 2>'/dev/null' || true - -exit diff --git a/tests/authentication/10-remote-suite-passphrase-cache/reference.log b/tests/authentication/10-remote-suite-passphrase-cache/reference.log deleted file mode 100644 index cedbfa12037..00000000000 --- a/tests/authentication/10-remote-suite-passphrase-cache/reference.log +++ /dev/null @@ -1,4 +0,0 @@ -2015-12-18T10:58:39Z INFO - Initial point: 19700101T0000Z -2015-12-18T10:58:39Z INFO - Final point: 19700101T0000Z -2015-12-18T10:58:39Z INFO - [t1.19700101T0000Z] -triggered off [] -2015-12-18T10:59:14Z INFO - [t1.19700101T0000Z] -triggered off [] diff --git a/tests/authentication/10-remote-suite-passphrase-cache/suite.rc b/tests/authentication/10-remote-suite-passphrase-cache/suite.rc deleted file mode 100644 index a47feccd5c2..00000000000 --- a/tests/authentication/10-remote-suite-passphrase-cache/suite.rc +++ /dev/null @@ -1,12 +0,0 @@ -[cylc] - UTC mode=True - [[reference test]] - expected task failures = t1.19700101T0000Z -[scheduling] - initial cycle point=1970 - final cycle point=1970 - [[graph]] - P1Y=t1 -[runtime] - [[t1]] - script=printenv CYLC_TEST_VAR_FOO diff --git a/tests/authentication/override/suite.rc b/tests/authentication/override/suite.rc deleted file mode 100644 index 4bda6a8d1b1..00000000000 --- a/tests/authentication/override/suite.rc +++ /dev/null @@ -1,24 +0,0 @@ -[meta] - title = Authentication test suite. - description = """Stalls when the first task fails. -Suite overrides global authentication settings.""" -[cylc] - [[events]] - timeout = PT30S - abort on timeout = True - [[authentication]] - public = shutdown -[scheduling] - cycling mode = integer - initial cycle point = 1 - final cycle point = 2 - [[graph]] - R/1 = foo => bar - R/2 = bar[-P1] => pub -[runtime] - [[foo]] - script = false - [[bar]] - script = true - [[pub]] - script = true diff --git a/tests/cylc-message/00-ssh/suite.rc b/tests/cylc-message/00-ssh/suite.rc index 505182807d3..136ef8c1ca4 100644 --- a/tests/cylc-message/00-ssh/suite.rc +++ b/tests/cylc-message/00-ssh/suite.rc @@ -9,9 +9,8 @@ [runtime] [[t0]] script = """ -# Ensure that contact 2 is installed, but passphrase isn't. +# Ensure that contact 2 is installed. test -f "${CYLC_SUITE_RUN_DIR}/.service/contact2" -! test -f "${CYLC_SUITE_RUN_DIR}/.service/passphrase" # A client command should not require --use-ssh option. cylc broadcast "${CYLC_SUITE_NAME}" '--name=t1' '--set=script=true' """ From 0b0488fd8d70f967da5cc8455fc1c1b17552f200 Mon Sep 17 00:00:00 2001 From: Mel Hall <37735232+datamel@users.noreply.github.com> Date: Fri, 6 Mar 2020 09:03:07 +0000 Subject: [PATCH 3/8] Remove references to passphrase --- cylc/flow/scripts/cylc_ext_trigger.py | 2 -- cylc/flow/scripts/cylc_remote_init.py | 1 - cylc/flow/scripts/cylc_scan.py | 4 +--- cylc/flow/suite_files.py | 6 ------ cylc/flow/task_remote_mgr.py | 1 - cylc/flow/tests/network/test_client.py | 5 ----- 6 files changed, 1 insertion(+), 18 deletions(-) diff --git a/cylc/flow/scripts/cylc_ext_trigger.py b/cylc/flow/scripts/cylc_ext_trigger.py index c36a620e2d8..01b10e829e2 100755 --- a/cylc/flow/scripts/cylc_ext_trigger.py +++ b/cylc/flow/scripts/cylc_ext_trigger.py @@ -31,8 +31,6 @@ Use the retry options in case the target suite is down or out of contact. -The suite passphrase must be installed in $HOME/.cylc//. - Note: to manually trigger a task use 'cylc trigger', not this command.""" from time import sleep diff --git a/cylc/flow/scripts/cylc_remote_init.py b/cylc/flow/scripts/cylc_remote_init.py index 0b394b05aa4..fe3b1409d55 100755 --- a/cylc/flow/scripts/cylc_remote_init.py +++ b/cylc/flow/scripts/cylc_remote_init.py @@ -19,7 +19,6 @@ (This command is for internal use.) Install suite service files on a task remote (i.e. a [owner@]host): .service/contact: All task -> suite communication methods. - .service/passphrase: Direct task -> suite communication only. Content of items to install from a tar file read from STDIN. diff --git a/cylc/flow/scripts/cylc_scan.py b/cylc/flow/scripts/cylc_scan.py index 561cb2f824a..5e7cc457cab 100755 --- a/cylc/flow/scripts/cylc_scan.py +++ b/cylc/flow/scripts/cylc_scan.py @@ -23,10 +23,8 @@ Use the -o/--suite-owner option to get information of running suites for other users. -Suite passphrases are not needed to get identity information (name and owner). Titles, descriptions, state totals, and cycle point state totals may also be -revealed publicly, depending on global and suite authentication settings. Suite -passphrases still grant full access regardless of what is revealed publicly. +revealed publicly, depending on global and suite authentication settings. WARNING: a suite suspended with Ctrl-Z will cause port scans to hang until the connection times out (see --comms-timeout).""" diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py index 1093e2fedda..e00c78d5ff6 100644 --- a/cylc/flow/suite_files.py +++ b/cylc/flow/suite_files.py @@ -138,9 +138,6 @@ class Service: CONTACT2 = 'contact2' """Same as ``CONTACT``, installed on remote platforms.""" - PASSPHRASE = 'passphrase' - """The suite authentication token.""" - SOURCE = 'source' """Symlink to the suite definition (suite dir).""" @@ -215,9 +212,6 @@ class ContactFileFields: NO_TITLE = "No title provided" REC_TITLE = re.compile(r"^\s*title\s*=\s*(.*)\s*$") -PASSPHRASE_CHARSET = ascii_letters + digits -PASSPHRASE_LEN = 20 - PS_OPTS = '-wopid,args' CONTACT_FILE_EXISTS_MSG = r"""suite contact file exists: %(fname)s diff --git a/cylc/flow/task_remote_mgr.py b/cylc/flow/task_remote_mgr.py index 0e35d049824..7895aa679ea 100644 --- a/cylc/flow/task_remote_mgr.py +++ b/cylc/flow/task_remote_mgr.py @@ -134,7 +134,6 @@ def remote_init(self, host, owner): Call "cylc remote-init" to install suite items to remote: ".service/contact": For TCP task communication - ".service/passphrase": For TCP task communication "python/": if source exists Return: diff --git a/cylc/flow/tests/network/test_client.py b/cylc/flow/tests/network/test_client.py index 4a3ddb482bf..e9ac55df159 100644 --- a/cylc/flow/tests/network/test_client.py +++ b/cylc/flow/tests/network/test_client.py @@ -32,11 +32,6 @@ SERVER_CONTEXT = zmq.Context() -def get_secret(): - """Return string in place of passphrase.""" - return SECRET - - class TestSuiteRuntimeClient(CylcWorkflowTestCase): """Test the workflow runtime client.""" From 7257d4a961489787bd749288b38fb1635170817d Mon Sep 17 00:00:00 2001 From: Mel Hall <37735232+datamel@users.noreply.github.com> Date: Wed, 11 Mar 2020 10:46:46 +0000 Subject: [PATCH 4/8] Removed get_auth_items --- cylc/flow/network/scan.py | 4 +- cylc/flow/suite_files.py | 251 +++----------------------------------- 2 files changed, 20 insertions(+), 235 deletions(-) diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py index f4377043ac1..4b53e9812a5 100644 --- a/cylc/flow/network/scan.py +++ b/cylc/flow/network/scan.py @@ -276,8 +276,8 @@ def get_scan_items_from_fs( try: contact_data = load_contact_file(reg, owner) cylc_version = contact_data[ContactFileFields.VERSION] - stripped_cylc_version = int(cylc_version[:1]) - if (stripped_cylc_version < 8): + major_version = int(cylc_version.split(".", 1)[0]) + if (major_version < 8): LOG.debug( f"Suite \"{reg}\" is running in Cylc version " f"{cylc_version}, not Cylc 8 and will not be " diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py index e00c78d5ff6..23bacb43c52 100644 --- a/cylc/flow/suite_files.py +++ b/cylc/flow/suite_files.py @@ -333,107 +333,6 @@ def get_contact_file(reg): get_suite_srv_dir(reg), SuiteFiles.Service.CONTACT) -def get_auth_item(item, reg, owner=None, host=None, content=False): - """Locate/load Curve private-key/ ...etc. - - Return file name, or content of file if content=True is set. - Files are searched from these locations in order: - - 1/ Server Curve ZMQ keys located in suite service directory - Client Curve ZMQ keys located in - suite service directory (private keys) - suite service directory/client_public_keys (public keys) - - 2/ For running task jobs, service directory under: - a/ $CYLC_SUITE_RUN_DIR for remote jobs. - b/ $CYLC_SUITE_RUN_DIR_ON_SUITE_HOST for local jobs or remote jobs - with SSH messaging. - - 3/ For suite on local user@host. The suite service directory. - - 4/ Location under $HOME/.cylc/ for remote suite control from accounts - that do not actually need the suite definition directory to be - installed: - $HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/ - - 5/ For remote suites, try locating the file from the suite service - directory on remote owner@host via SSH. If content=False, the value - of the located file will be dumped under: - $HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/ - - """ - if item not in [ - SuiteFiles.Service.CONTACT, - SuiteFiles.Service.CONTACT2] and not isinstance(item, KeyInfo): - raise ValueError(f"{item}: item not recognised") - - # 1 (a) - if isinstance(item, KeyInfo): - - item_location = _locate_item(item.file_name, item.key_path) - - # TODO: separate key file 'get' into own function - # Additional searches below need a file name, not a complex object - item = item.file_name - - if item_location: - return item_location - - if reg == os.getenv('CYLC_SUITE_NAME'): - env_keys = [] - if 'CYLC_SUITE_RUN_DIR' in os.environ: - # 2(a)/ Task messaging call. - env_keys.append('CYLC_SUITE_RUN_DIR') - elif ContactFileFields.SUITE_RUN_DIR_ON_SUITE_HOST in os.environ: - # 2(b)/ Task messaging call via ssh messaging. - env_keys.append(ContactFileFields.SUITE_RUN_DIR_ON_SUITE_HOST) - for key in env_keys: - path = os.path.join(os.environ[key], SuiteFiles.Service.DIRNAME) - if content: - value = _load_local_item(item, path) - else: - value = _locate_item(item, path) - if value: - return value - # 3/ Local suite service directory - if _is_local_auth_ok(reg, owner, host): - path = get_suite_srv_dir(reg) - if content: - value = _load_local_item(item, path) - else: - value = _locate_item(item, path) - if value: - return value - # 4/ Disk cache for remote suites - if owner is not None and host is not None: - paths = [_get_cache_dir(reg, owner, host)] - short_host = host.split('.', 1)[0] - if short_host != host: - paths.append(_get_cache_dir(reg, owner, short_host)) - for path in paths: - if content: - value = _load_local_item(item, path) - else: - value = _locate_item(item, path) - if value: - return value - - # 5/ Use SSH to load content from remote owner@host - # Note: It is not possible to find ".service/contact2" on the suite - # host, because it is installed on task host by "cylc remote-init" on - # demand. - if item != SuiteFiles.Service.CONTACT2: - value = _load_remote_item(item, reg, owner, host) - if value: - if not content: - path = _get_cache_dir(reg, owner, host) - _dump_item(path, item, value) - value = os.path.join(path, item) - return value - - raise SuiteServiceFileError("Couldn't get %s" % item) - - def get_suite_rc(reg, suite_owner=None): """Return the suite.rc path of a suite.""" return os.path.join( @@ -476,17 +375,19 @@ def get_suite_srv_dir(reg, suite_owner=None): return os.path.join(run_d, SuiteFiles.Service.DIRNAME) -def load_contact_file(reg, owner=None, host=None, file_base=None): +def load_contact_file(reg, owner=None, host=None): """Load contact file. Return data as key=value dict.""" - if not file_base: - file_base = SuiteFiles.Service.CONTACT - file_content = get_auth_item( - file_base, reg, owner, host, content=True) - data = {} - for line in file_content.splitlines(): - key, value = [item.strip() for item in line.split("=", 1)] - data[key] = value - return data + file_base = SuiteFiles.Service.CONTACT + path = get_suite_srv_dir(reg) + file_content = _load_local_item(file_base, path) + if file_content: + data = {} + for line in file_content.splitlines(): + key, value = [item.strip() for item in line.split("=", 1)] + data[key] = value + return data + else: + raise SuiteServiceFileError("Couldn't load contact file") def parse_suite_arg(options, arg): @@ -623,8 +524,12 @@ def register(reg=None, source=None, redirect=False, rundir=None): def create_auth_files(reg): - """Create or renew authentication keys for suite 'reg' in the .service - directory.""" + """Create or renew authentication keys for suite 'reg'. + Server Curve ZMQ keys located in suite service directory + Client Curve ZMQ keys located in: + suite service directory (private keys) + suite service directory/client_public_keys (public keys) + """ suite_srv_dir = get_suite_srv_dir(reg) @@ -697,14 +602,6 @@ def _dump_item(path, item, value): LOG.debug('Generated %s', fname) -def _get_cache_dir(reg, owner, host): - """Return the cache directory for remote suite service files.""" - return os.path.join( - os.path.expanduser("~"), ".cylc", "auth" - "%s@%s" % (owner, host), reg - ) - - def get_suite_title(reg): """Return the the suite title without a full file parse @@ -727,43 +624,6 @@ def get_suite_title(reg): return title -@lru_cache() -def _is_local_auth_ok(reg, owner, host): - """Return True if it is OK to use local passphrase file. - - Use values in ~/cylc-run/REG/.service/contact to make a judgement. - """ - if is_remote(host, owner): - fname = os.path.join( - get_suite_srv_dir(reg), SuiteFiles.Service.CONTACT) - data = {} - try: - for line in open(fname): - key, value = ( - [item.strip() for item in line.split("=", 1)]) - data[key] = value - except (IOError, ValueError): - # No contact file - return False - else: - # Contact file exists, check values match - if owner is None: - owner = get_user() - if host is None: - host = get_host() - host_value = data.get(ContactFileFields.HOST, "") - return ( - reg == data.get(ContactFileFields.NAME) and - owner == data.get(ContactFileFields.OWNER) and - ( - host == host_value or - host == host_value.split(".", 1)[0] # no domain - ) - ) - else: - return True - - def _load_local_item(item, path): """Load and return content of a file (item) in path.""" try: @@ -771,78 +631,3 @@ def _load_local_item(item, path): return file_.read() except IOError: return None - - -def _load_remote_item(item, reg, owner, host): - """Load content of service item from remote [owner@]host via SSH.""" - if not is_remote(host, owner): - return - if host is None: - host = 'localhost' - if owner is None: - owner = get_user() - if item == SuiteFiles.Service.CONTACT and not is_remote_host(host): - # Attempt to read suite contact file via the local filesystem. - path = r'%(run_d)s/%(srv_base)s' % { - 'run_d': get_remote_suite_run_dir('localhost', owner, reg), - 'srv_base': SuiteFiles.Service.DIRNAME, - } - content = _load_local_item(item, path) - if content is not None: - return content - # Else drop through and attempt via ssh to the suite account. - # Prefix STDOUT to ensure returned content is relevant - prefix = r'[CYLC-AUTH] %(suite)s' % {'suite': reg} - # Attempt to cat passphrase file under suite service directory - script = ( - r"""echo '%(prefix)s'; """ - r'''cat "%(run_d)s/%(srv_base)s/%(item)s"''' - ) % { - 'prefix': prefix, - 'run_d': get_remote_suite_run_dir(host, owner, reg), - 'srv_base': SuiteFiles.Service.DIRNAME, - 'item': item - } - import shlex - command = shlex.split( - glbl_cfg().get_host_item('ssh command', host, owner)) - command += ['-n', owner + '@' + host, script] - from subprocess import Popen, PIPE, DEVNULL # nosec - try: - proc = Popen( - command, stdin=DEVNULL, stdout=PIPE, stderr=PIPE) # nosec - except OSError: - if cylc.flow.flags.debug: - import traceback - traceback.print_exc() - return - out, err = (f.decode() for f in proc.communicate()) - ret_code = proc.wait() - # Extract passphrase from STDOUT - # It should live in the line with the correct prefix - content = "" - can_read = False - for line in out.splitlines(True): - if can_read: - content += line - elif line.strip() == prefix: - can_read = True - if not content or ret_code: - LOG.debug( - '$ %(command)s # code=%(ret_code)s\n%(err)s', - { - 'command': command, - # STDOUT may contain passphrase, so not safe to print - # 'out': out, - 'err': err, - 'ret_code': ret_code, - }) - return - return content - - -def _locate_item(item, path): - """Locate a service item in "path".""" - fname = os.path.join(path, item) - if os.path.exists(fname): - return fname From cef647932357900e5521015d452f21d1fb3242b0 Mon Sep 17 00:00:00 2001 From: Mel Hall <37735232+datamel@users.noreply.github.com> Date: Fri, 20 Mar 2020 15:01:43 +0000 Subject: [PATCH 5/8] No contact2 file,CYLC_COMMS_PROTOCOL_2 appended to contact instead --- cylc/flow/task_remote_cmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cylc/flow/task_remote_cmd.py b/cylc/flow/task_remote_cmd.py index 3847554cc85..1117896352e 100644 --- a/cylc/flow/task_remote_cmd.py +++ b/cylc/flow/task_remote_cmd.py @@ -62,8 +62,8 @@ def remote_init(uuid_str, rund, indirect_comm=None): finally: os.chdir(oldcwd) if indirect_comm: - fname = os.path.join(srvd, SuiteFiles.Service.CONTACT2) - with open(fname, 'w') as handle: + fname = os.path.join(srvd, SuiteFiles.Service.CONTACT) + with open(fname, 'a') as handle: handle.write('%s=%s\n' % ( ContactFileFields.COMMS_PROTOCOL_2, indirect_comm)) print(REMOTE_INIT_DONE) From 2a33139220ed5e602520e77192393cc501249ca5 Mon Sep 17 00:00:00 2001 From: Mel Hall <37735232+datamel@users.noreply.github.com> Date: Mon, 6 Apr 2020 17:31:17 +0100 Subject: [PATCH 6/8] Fix cylc scan return, using LOG.info. Add cylc scan output functional tests. --- cylc/flow/network/scan.py | 23 +++--- tests/cylc-scan/04-outputs.t | 113 ++++++++++++++++++++++++++++ tests/cylc-scan/04-outputs/suite.rc | 26 +++++++ 3 files changed, 153 insertions(+), 9 deletions(-) create mode 100644 tests/cylc-scan/04-outputs.t create mode 100644 tests/cylc-scan/04-outputs/suite.rc diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py index 4b53e9812a5..13b18544c17 100644 --- a/cylc/flow/network/scan.py +++ b/cylc/flow/network/scan.py @@ -228,7 +228,8 @@ def get_scan_items_from_fs( Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for active, or all (active plus registered but dormant), suites. - Note when active_only, suites run in cylc 7 or less will not be returned. + Note suites run in cylc 7 or less will only be returned as LOG.info + statement. Yields: tuple - (reg, host, port, pub_port, api) @@ -272,20 +273,24 @@ def get_scan_items_from_fs( # Choose only suites with .service and matching filter if active_only: - # Skip suites running with cylc version < 8 + # Skip suites running with cylc version < 8 (these suites + # do not have PUBLISH_PORT field) try: contact_data = load_contact_file(reg, owner) + except (SuiteServiceFileError, IOError, TypeError) as exc: + LOG.debug(f"Error loading contact file: {exc}") + continue + try: cylc_version = contact_data[ContactFileFields.VERSION] major_version = int(cylc_version.split(".", 1)[0]) if (major_version < 8): - LOG.debug( - f"Suite \"{reg}\" is running in Cylc version " - f"{cylc_version}, not Cylc 8 and will not be " - f"displayed." - ) + LOG.info(f"Suite \"{reg}\" is running in Cylc version" + f" {cylc_version}, not Cylc 8 and" + f" will not be displayed.") continue - - except (SuiteServiceFileError, IOError, TypeError, ValueError): + except Exception as exc: + LOG.debug( + f"Error getting version from contact file: {exc}") continue yield ( reg, diff --git a/tests/cylc-scan/04-outputs.t b/tests/cylc-scan/04-outputs.t new file mode 100644 index 00000000000..8d71f30b357 --- /dev/null +++ b/tests/cylc-scan/04-outputs.t @@ -0,0 +1,113 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2020 NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Test Cylc Scan output + +. "$(dirname "$0")/test_header" +set_test_number 5 +install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" + +TEST_NAME="${TEST_NAME_BASE}-validate" +run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}" + +# Run the suite. +cylc run "${SUITE_NAME}" + +# Wait for first task 'foo' to fail. +cylc suite-state "${SUITE_NAME}" --task=foo --status=failed --point=1 \ + --interval=1 --max-polls=20 || exit 1 +# Check scan --full output. +SRV_D="$(cylc get-global-config --print-run-dir)/${SUITE_NAME}/.service" +HOST="$(sed -n 's/^CYLC_SUITE_HOST=//p' "${SRV_D}/contact")" +PORT="$(sed -n 's/^CYLC_SUITE_PORT=//p' "${SRV_D}/contact")" +PUBLISH_PORT="$(sed -n 's/^CYLC_SUITE_PUBLISH_PORT=//p' "${SRV_D}/contact")" +CYLC_VERSION="$(sed -n 's/^CYLC_VERSION=//p' "${SRV_D}/contact")" + +cylc scan --comms-timeout=5 -f --color=never -n "${SUITE_NAME}" \ + >'scan-f.out' # 2>'/dev/null' +cmp_ok 'scan-f.out' <<__END__ +${SUITE_NAME} ${USER}@${HOST}:${PORT} ${USER}@${HOST}:${PUBLISH_PORT} + Title: + Cylc Scan test suite. + Description: + Stalls when the first task fails. + Here we test out a multi-line description! + Group: + (no Group) + API: + 5 + URL: + (no URL) + another_metadata: + 1 + custom_metadata: + something_custom + Task state totals: + failed:1 waiting:2 + 1 failed:1 waiting:1 + 2 waiting:1 +__END__ + +# Check scan --describe output. +cylc scan --comms-timeout=5 -d --color=never -n "${SUITE_NAME}" \ + >'scan-d.out' # 2>'/dev/null' +cmp_ok 'scan-d.out' <<__END__ +${SUITE_NAME} ${USER}@${HOST}:${PORT} + Title: + Cylc Scan test suite. + Description: + Stalls when the first task fails. + Here we test out a multi-line description! + Group: + (no Group) + API: + 5 + URL: + (no URL) + another_metadata: + 1 + custom_metadata: + something_custom +__END__ + +# Check scan --raw output. +cylc scan --comms-timeout=5 -t raw --color=never -n "${SUITE_NAME}" \ + >'scan-r.out' # 2>'/dev/null' +cmp_ok 'scan-r.out' <<__END__ +${SUITE_NAME}|${USER}|${HOST}|port|${PORT} +__END__ + +# Check scan --json output. +cylc scan --comms-timeout=5 -t json --color=never -n "${SUITE_NAME}" \ + >'scan-j.out' # 2>'/dev/null' +cmp_json 'scan-j.out' 'scan-j.out' <<__END__ +[ + [ "${SUITE_NAME}", + "${HOST}", + "${PORT}", + "$PUBLISH_PORT", + "5", + {"name":"${SUITE_NAME}", "owner": "${USER}", "version": "${CYLC_VERSION}"} + + ] +] +__END__ + +# Stop and purge the suite. +cylc stop --max-polls=20 --interval=1 "${SUITE_NAME}" +purge_suite "${SUITE_NAME}" +exit diff --git a/tests/cylc-scan/04-outputs/suite.rc b/tests/cylc-scan/04-outputs/suite.rc new file mode 100644 index 00000000000..480cc6a6c10 --- /dev/null +++ b/tests/cylc-scan/04-outputs/suite.rc @@ -0,0 +1,26 @@ +[meta] + title = Cylc Scan test suite. + description = """ + Stalls when the first task fails. + Here we test out a multi-line description! + """ + custom_metadata = something_custom + another_metadata = 1 +[cylc] + [[events]] + timeout = PT1M + abort on timeout = True +[scheduling] + cycling mode = integer + initial cycle point = 1 + final cycle point = 2 + [[graph]] + R/1 = foo => bar + R/2 = bar[-P1] => pub +[runtime] + [[foo]] + script = /bin/false + [[bar]] + script = /bin/true + [[pub]] + script = /bin/true From 200161b4ed7818d460217fe2496b6ebe61cec5f5 Mon Sep 17 00:00:00 2001 From: Mel Hall <37735232+datamel@users.noreply.github.com> Date: Thu, 16 Apr 2020 16:57:53 +0100 Subject: [PATCH 7/8] Remove Contact2 --- cylc/flow/network/scan.py | 6 ++---- cylc/flow/suite_files.py | 4 ---- cylc/flow/task_remote_cmd.py | 22 +++++++++------------- tests/cylc-message/00-ssh/suite.rc | 2 -- tests/cylc-scan/04-outputs/suite.rc | 6 +++--- 5 files changed, 14 insertions(+), 26 deletions(-) diff --git a/cylc/flow/network/scan.py b/cylc/flow/network/scan.py index 13b18544c17..351ca505848 100644 --- a/cylc/flow/network/scan.py +++ b/cylc/flow/network/scan.py @@ -278,15 +278,13 @@ def get_scan_items_from_fs( try: contact_data = load_contact_file(reg, owner) except (SuiteServiceFileError, IOError, TypeError) as exc: - LOG.debug(f"Error loading contact file: {exc}") + LOG.debug(f"Error loading contact file for: {reg}") continue try: cylc_version = contact_data[ContactFileFields.VERSION] major_version = int(cylc_version.split(".", 1)[0]) if (major_version < 8): - LOG.info(f"Suite \"{reg}\" is running in Cylc version" - f" {cylc_version}, not Cylc 8 and" - f" will not be displayed.") + LOG.info(f"Omitting \"{reg}\" (cylc-{cylc_version})") continue except Exception as exc: LOG.debug( diff --git a/cylc/flow/suite_files.py b/cylc/flow/suite_files.py index 23bacb43c52..2a0c9d9eb75 100644 --- a/cylc/flow/suite_files.py +++ b/cylc/flow/suite_files.py @@ -22,7 +22,6 @@ import re import shutil import stat -from string import ascii_letters, digits import zmq.auth from cylc.flow import LOG @@ -135,9 +134,6 @@ class Service: For details of the fields see ``ContactFileFields``. """ - CONTACT2 = 'contact2' - """Same as ``CONTACT``, installed on remote platforms.""" - SOURCE = 'source' """Symlink to the suite definition (suite dir).""" diff --git a/cylc/flow/task_remote_cmd.py b/cylc/flow/task_remote_cmd.py index 1117896352e..6e0411fddff 100644 --- a/cylc/flow/task_remote_cmd.py +++ b/cylc/flow/task_remote_cmd.py @@ -78,19 +78,15 @@ def remote_tidy(rund): """ rund = os.path.expandvars(rund) srvd = os.path.join(rund, SuiteFiles.Service.DIRNAME) - for name in [ - SuiteFiles.Service.CONTACT, - SuiteFiles.Service.CONTACT2 - ]: - fname = os.path.join(srvd, name) - try: - os.unlink(fname) - except OSError: - if os.path.exists(fname): - raise - else: - if cylc.flow.flags.debug: - print('Deleted: %s' % fname) + fname = os.path.join(srvd, SuiteFiles.Service.CONTACT) + try: + os.unlink(fname) + except OSError: + if os.path.exists(fname): + raise + else: + if cylc.flow.flags.debug: + print('Deleted: %s' % fname) try: os.rmdir(srvd) # remove directory if empty except OSError: diff --git a/tests/cylc-message/00-ssh/suite.rc b/tests/cylc-message/00-ssh/suite.rc index 136ef8c1ca4..b44b64f3dac 100644 --- a/tests/cylc-message/00-ssh/suite.rc +++ b/tests/cylc-message/00-ssh/suite.rc @@ -9,8 +9,6 @@ [runtime] [[t0]] script = """ -# Ensure that contact 2 is installed. -test -f "${CYLC_SUITE_RUN_DIR}/.service/contact2" # A client command should not require --use-ssh option. cylc broadcast "${CYLC_SUITE_NAME}" '--name=t1' '--set=script=true' """ diff --git a/tests/cylc-scan/04-outputs/suite.rc b/tests/cylc-scan/04-outputs/suite.rc index 480cc6a6c10..5834e6113cc 100644 --- a/tests/cylc-scan/04-outputs/suite.rc +++ b/tests/cylc-scan/04-outputs/suite.rc @@ -19,8 +19,8 @@ R/2 = bar[-P1] => pub [runtime] [[foo]] - script = /bin/false + script = false [[bar]] - script = /bin/true + script = true [[pub]] - script = /bin/true + script = true From 1c065e70ae8681557ad3b95498554c8ac9e19ad5 Mon Sep 17 00:00:00 2001 From: Mel Hall <37735232+datamel@users.noreply.github.com> Date: Mon, 4 May 2020 13:56:05 +0100 Subject: [PATCH 8/8] skip task comms test --- tests/cylc-message/00-ssh.t | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/cylc-message/00-ssh.t b/tests/cylc-message/00-ssh.t index 9facbb88714..94722ec4f6f 100755 --- a/tests/cylc-message/00-ssh.t +++ b/tests/cylc-message/00-ssh.t @@ -19,6 +19,9 @@ # installed on the remote host. export CYLC_TEST_IS_GENERIC=false . "$(dirname "$0")/test_header" +# TODO: Fix test once ssh task comms reinstated +skip_all 'ssh task comm not currently functional' + #------------------------------------------------------------------------------- set_test_remote_host set_test_number 3