Merge pull request #3509 from datamel/authentication_washup_cylc_scan

Authentication Washup & Fixed Cylc Scan

cylc/flow/network/scan.py

@@ -228,6 +228,8 @@ def get_scan_items_from_fs(
 
     Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for
     active, or all (active plus registered but dormant), suites.
+    Note suites run in cylc 7 or less will only be returned as LOG.info
+    statement.
 
     Yields:
         tuple - (reg, host, port, pub_port, api)
@@ -271,10 +273,22 @@ def get_scan_items_from_fs(
 
             # Choose only suites with .service and matching filter
             if active_only:
+                # Skip suites running with cylc version < 8 (these suites
+                # do not have PUBLISH_PORT field)
                 try:
-                    contact_data = load_contact_file(
-                        reg, owner)
-                except (SuiteServiceFileError, IOError, TypeError, ValueError):
+                    contact_data = load_contact_file(reg, owner)
+                except (SuiteServiceFileError, IOError, TypeError) as exc:
+                    LOG.debug(f"Error loading contact file for: {reg}")
+                    continue
+                try:
+                    cylc_version = contact_data[ContactFileFields.VERSION]
+                    major_version = int(cylc_version.split(".", 1)[0])
+                    if (major_version < 8):
+                        LOG.info(f"Omitting \"{reg}\" (cylc-{cylc_version})")
+                        continue
+                except Exception as exc:
+                    LOG.debug(
+                        f"Error getting version from contact file: {exc}")
                     continue
                 yield (
                     reg,

cylc/flow/scripts/cylc_ext_trigger.py

@@ -31,8 +31,6 @@
 
 Use the retry options in case the target suite is down or out of contact.
 
-The suite passphrase must be installed in $HOME/.cylc/<SUITE>/.
-
 Note: to manually trigger a task use 'cylc trigger', not this command."""
 
 from time import sleep

cylc/flow/scripts/cylc_remote_init.py

@@ -19,7 +19,6 @@
 (This command is for internal use.)
 Install suite service files on a task remote (i.e. a [owner@]host):
     .service/contact: All task -> suite communication methods.
-    .service/passphrase: Direct task -> suite communication only.
 
 Content of items to install from a tar file read from STDIN.
 

cylc/flow/scripts/cylc_scan.py

@@ -23,10 +23,8 @@
 Use the -o/--suite-owner option to get information of running suites for other
 users.
 
-Suite passphrases are not needed to get identity information (name and owner).
 Titles, descriptions, state totals, and cycle point state totals may also be
-revealed publicly, depending on global and suite authentication settings. Suite
-passphrases still grant full access regardless of what is revealed publicly.
+revealed publicly, depending on global and suite authentication settings.
 
 WARNING: a suite suspended with Ctrl-Z will cause port scans to hang until the
 connection times out (see --comms-timeout)."""

cylc/flow/suite_files.py

@@ -22,7 +22,6 @@
 import re
 import shutil
 import stat
-from string import ascii_letters, digits
 import zmq.auth
 
 from cylc.flow import LOG
@@ -135,12 +134,6 @@ class Service:
         For details of the fields see ``ContactFileFields``.
         """
 
-        CONTACT2 = 'contact2'
-        """Same as ``CONTACT``, installed on remote platforms."""
-
-        PASSPHRASE = 'passphrase'
-        """The suite authentication token."""
-
         SOURCE = 'source'
         """Symlink to the suite definition (suite dir)."""
 
@@ -215,9 +208,6 @@ class ContactFileFields:
 NO_TITLE = "No title provided"
 REC_TITLE = re.compile(r"^\s*title\s*=\s*(.*)\s*$")
 
-PASSPHRASE_CHARSET = ascii_letters + digits
-PASSPHRASE_LEN = 20
-
 PS_OPTS = '-wopid,args'
 
 CONTACT_FILE_EXISTS_MSG = r"""suite contact file exists: %(fname)s
@@ -339,107 +329,6 @@ def get_contact_file(reg):
         get_suite_srv_dir(reg), SuiteFiles.Service.CONTACT)
 
 
-def get_auth_item(item, reg, owner=None, host=None, content=False):
-    """Locate/load Curve private-key/ ...etc.
-
-    Return file name, or content of file if content=True is set.
-    Files are searched from these locations in order:
-
-    1/  Server Curve ZMQ keys located in suite service directory
-        Client Curve ZMQ keys located in
-            suite service directory (private keys)
-            suite service directory/client_public_keys (public keys)
-
-    2/ For running task jobs, service directory under:
-       a/ $CYLC_SUITE_RUN_DIR for remote jobs.
-       b/ $CYLC_SUITE_RUN_DIR_ON_SUITE_HOST for local jobs or remote jobs
-          with SSH messaging.
-
-    3/ For suite on local user@host. The suite service directory.
-
-    4/ Location under $HOME/.cylc/ for remote suite control from accounts
-       that do not actually need the suite definition directory to be
-       installed:
-       $HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/
-
-    5/ For remote suites, try locating the file from the suite service
-       directory on remote owner@host via SSH. If content=False, the value
-       of the located file will be dumped under:
-       $HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/
-
-    """
-    if item not in [
-            SuiteFiles.Service.CONTACT,
-            SuiteFiles.Service.CONTACT2] and not isinstance(item, KeyInfo):
-        raise ValueError(f"{item}: item not recognised")
-
-        # 1 (a)
-    if isinstance(item, KeyInfo):
-
-        item_location = _locate_item(item.file_name, item.key_path)
-
-        # TODO: separate key file 'get' into own function
-        # Additional searches below need a file name, not a complex object
-        item = item.file_name
-
-        if item_location:
-            return item_location
-
-    if reg == os.getenv('CYLC_SUITE_NAME'):
-        env_keys = []
-        if 'CYLC_SUITE_RUN_DIR' in os.environ:
-            # 2(a)/ Task messaging call.
-            env_keys.append('CYLC_SUITE_RUN_DIR')
-        elif ContactFileFields.SUITE_RUN_DIR_ON_SUITE_HOST in os.environ:
-            # 2(b)/ Task messaging call via ssh messaging.
-            env_keys.append(ContactFileFields.SUITE_RUN_DIR_ON_SUITE_HOST)
-        for key in env_keys:
-            path = os.path.join(os.environ[key], SuiteFiles.Service.DIRNAME)
-            if content:
-                value = _load_local_item(item, path)
-            else:
-                value = _locate_item(item, path)
-            if value:
-                return value
-    # 3/ Local suite service directory
-    if _is_local_auth_ok(reg, owner, host):
-        path = get_suite_srv_dir(reg)
-        if content:
-            value = _load_local_item(item, path)
-        else:
-            value = _locate_item(item, path)
-        if value:
-            return value
-    # 4/ Disk cache for remote suites
-    if owner is not None and host is not None:
-        paths = [_get_cache_dir(reg, owner, host)]
-        short_host = host.split('.', 1)[0]
-        if short_host != host:
-            paths.append(_get_cache_dir(reg, owner, short_host))
-        for path in paths:
-            if content:
-                value = _load_local_item(item, path)
-            else:
-                value = _locate_item(item, path)
-            if value:
-                return value
-
-    # 5/ Use SSH to load content from remote owner@host
-    # Note: It is not possible to find ".service/contact2" on the suite
-    # host, because it is installed on task host by "cylc remote-init" on
-    # demand.
-    if item != SuiteFiles.Service.CONTACT2:
-        value = _load_remote_item(item, reg, owner, host)
-        if value:
-            if not content:
-                path = _get_cache_dir(reg, owner, host)
-                _dump_item(path, item, value)
-                value = os.path.join(path, item)
-            return value
-
-    raise SuiteServiceFileError("Couldn't get %s" % item)
-
-
 def get_suite_rc(reg, suite_owner=None):
     """Return the suite.rc path of a suite."""
     return os.path.join(
@@ -482,17 +371,19 @@ def get_suite_srv_dir(reg, suite_owner=None):
     return os.path.join(run_d, SuiteFiles.Service.DIRNAME)
 
 
-def load_contact_file(reg, owner=None, host=None, file_base=None):
+def load_contact_file(reg, owner=None, host=None):
     """Load contact file. Return data as key=value dict."""
-    if not file_base:
-        file_base = SuiteFiles.Service.CONTACT
-    file_content = get_auth_item(
-        file_base, reg, owner, host, content=True)
-    data = {}
-    for line in file_content.splitlines():
-        key, value = [item.strip() for item in line.split("=", 1)]
-        data[key] = value
-    return data
+    file_base = SuiteFiles.Service.CONTACT
+    path = get_suite_srv_dir(reg)
+    file_content = _load_local_item(file_base, path)
+    if file_content:
+        data = {}
+        for line in file_content.splitlines():
+            key, value = [item.strip() for item in line.split("=", 1)]
+            data[key] = value
+        return data
+    else:
+        raise SuiteServiceFileError("Couldn't load contact file")
 
 
 def parse_suite_arg(options, arg):
@@ -629,8 +520,12 @@ def register(reg=None, source=None, redirect=False, rundir=None):
 
 
 def create_auth_files(reg):
-    """Create or renew authentication keys for suite 'reg' in the .service
-     directory."""
+    """Create or renew authentication keys for suite 'reg'.
+     Server Curve ZMQ keys located in suite service directory
+     Client Curve ZMQ keys located in:
+        suite service directory (private keys)
+        suite service directory/client_public_keys (public keys)
+    """
 
     suite_srv_dir = get_suite_srv_dir(reg)
 
@@ -703,14 +598,6 @@ def _dump_item(path, item, value):
     LOG.debug('Generated %s', fname)
 
 
-def _get_cache_dir(reg, owner, host):
-    """Return the cache directory for remote suite service files."""
-    return os.path.join(
-        os.path.expanduser("~"), ".cylc", "auth"
-        "%s@%s" % (owner, host), reg
-    )
-
-
 def get_suite_title(reg):
     """Return the the suite title without a full file parse
 
@@ -733,122 +620,10 @@ def get_suite_title(reg):
     return title
 
 
-@lru_cache()
-def _is_local_auth_ok(reg, owner, host):
-    """Return True if it is OK to use local passphrase file.
-
-    Use values in ~/cylc-run/REG/.service/contact to make a judgement.
-    """
-    if is_remote(host, owner):
-        fname = os.path.join(
-            get_suite_srv_dir(reg), SuiteFiles.Service.CONTACT)
-        data = {}
-        try:
-            for line in open(fname):
-                key, value = (
-                    [item.strip() for item in line.split("=", 1)])
-                data[key] = value
-        except (IOError, ValueError):
-            # No contact file
-            return False
-        else:
-            # Contact file exists, check values match
-            if owner is None:
-                owner = get_user()
-            if host is None:
-                host = get_host()
-            host_value = data.get(ContactFileFields.HOST, "")
-            return (
-                reg == data.get(ContactFileFields.NAME) and
-                owner == data.get(ContactFileFields.OWNER) and
-                (
-                    host == host_value or
-                    host == host_value.split(".", 1)[0]  # no domain
-                )
-            )
-    else:
-        return True
-
-
 def _load_local_item(item, path):
     """Load and return content of a file (item) in path."""
     try:
         with open(os.path.join(path, item)) as file_:
             return file_.read()
     except IOError:
         return None
-
-
-def _load_remote_item(item, reg, owner, host):
-    """Load content of service item from remote [owner@]host via SSH."""
-    if not is_remote(host, owner):
-        return
-    if host is None:
-        host = 'localhost'
-    if owner is None:
-        owner = get_user()
-    if item == SuiteFiles.Service.CONTACT and not is_remote_host(host):
-        # Attempt to read suite contact file via the local filesystem.
-        path = r'%(run_d)s/%(srv_base)s' % {
-            'run_d': get_remote_suite_run_dir('localhost', owner, reg),
-            'srv_base': SuiteFiles.Service.DIRNAME,
-        }
-        content = _load_local_item(item, path)
-        if content is not None:
-            return content
-        # Else drop through and attempt via ssh to the suite account.
-    # Prefix STDOUT to ensure returned content is relevant
-    prefix = r'[CYLC-AUTH] %(suite)s' % {'suite': reg}
-    # Attempt to cat passphrase file under suite service directory
-    script = (
-        r"""echo '%(prefix)s'; """
-        r'''cat "%(run_d)s/%(srv_base)s/%(item)s"'''
-    ) % {
-        'prefix': prefix,
-        'run_d': get_remote_suite_run_dir(host, owner, reg),
-        'srv_base': SuiteFiles.Service.DIRNAME,
-        'item': item
-    }
-    import shlex
-    command = shlex.split(
-        glbl_cfg().get_host_item('ssh command', host, owner))
-    command += ['-n', owner + '@' + host, script]
-    from subprocess import Popen, PIPE, DEVNULL  # nosec
-    try:
-        proc = Popen(
-            command, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)  # nosec
-    except OSError:
-        if cylc.flow.flags.debug:
-            import traceback
-            traceback.print_exc()
-        return
-    out, err = (f.decode() for f in proc.communicate())
-    ret_code = proc.wait()
-    # Extract passphrase from STDOUT
-    # It should live in the line with the correct prefix
-    content = ""
-    can_read = False
-    for line in out.splitlines(True):
-        if can_read:
-            content += line
-        elif line.strip() == prefix:
-            can_read = True
-    if not content or ret_code:
-        LOG.debug(
-            '$ %(command)s  # code=%(ret_code)s\n%(err)s',
-            {
-                'command': command,
-                # STDOUT may contain passphrase, so not safe to print
-                # 'out': out,
-                'err': err,
-                'ret_code': ret_code,
-            })
-        return
-    return content
-
-
-def _locate_item(item, path):
-    """Locate a service item in "path"."""
-    fname = os.path.join(path, item)
-    if os.path.exists(fname):
-        return fname