Merge branch 'trs/singularity/compat'

CHANGES.md

@@ -13,6 +13,61 @@ development source code and as such may not be routinely kept up to date.
 
 # __NEXT__
 
+This release is mostly a bug fix release for our Conda and Singularity
+runtimes.  However, it contains a **potentially-breaking change** for existing
+usages of the Singularity runtime: **the minimum required Singularity version
+has changed from 2.6.0 to 3.0.0**.  This change was required for a critical bug
+fix.  If you do not use the Singularity runtime, there are no
+potentially-breaking changes in this release.
+
+## Improvements
+
+* `nextstrain shell` now notes which runtime is being entered in its initial
+  messaging to establish more context for the user (and for developers when
+  troubleshooting).
+  ([#283][])
+
+* The Singularity runtime now checks for the minimum required Singularity
+  version (3.0.0 with this release) during `nextstrain check-setup`.
+  ([#283][])
+
+## Bug fixes
+
+* Setup and upgrade of the Conda runtime now only uses stable "main" channel
+  releases when determining the latest release version, as intended.
+  Previously, testing and development releases could be selected if they were
+  newer than the last stable release.  Additionally, if there are multiple
+  builds for a release version, the highest numbered build (i.e. newest) is now
+  used instead of the lowest.
+  ([#280](https://github.com/nextstrain/cli/pull/280))
+
+* The Singularity runtime now works with our container runtime images from
+  `build-20230411T103027Z` onwards.  The Snakemake upgrade in that image
+  version resulted in "read-only file system" errors which referenced the
+  user's home directory.  Those errors are now fixed.
+  ([#283][])
+
+* The prompt for `nextstrain shell`—a stylized variant of the Nextstrain
+  wordmark—now works when using the Singularity runtime regardless of
+  Singularity version.  Previously Singularity's default prompt of
+  `Singularity> ` overrode ours when using Singularity versions ≥3.5.3.
+  ([#283][])
+
+* More robust command-line processing is used for the Singularity runtime on
+  Singularity versions ≥3.10.0.  Singularity's early (and unexpected)
+  evaluation of arguments that look like (but aren't) shell variable
+  substitutions is disabled.
+  ([#283][])
+
+## Development
+
+* The command lines and environment overrides of many (but not all) process
+  invocations are now logged when `NEXTSTRAIN_DEBUG` is enabled.
+  ([#283][])
+
+
+[#283]: https://github.com/nextstrain/cli/pull/283
+
 
 # 6.2.1 (24 March 2023)
 

doc/installation.rst

@@ -193,6 +193,8 @@ dependencies as validated versions are already bundled into a container image
 by the Nextstrain team.
 
 Run ``nextstrain setup singularity`` to get started.
+Singularity version 3.0.0 or newer is required, but we recommend at least
+version 3.10.0 or newer when possible.
 
 Note that the Singularity project forked into two separate projects in late
 2021: `SingularityCE`_ under `Sylabs`_ and `Apptainer`_ under the `Linux

nextstrain/cli/command/shell.py

@@ -61,7 +61,7 @@ def run(opts):
             Use the Docker or Singularity runtimes (via --docker or --singularity) if overlays are necessary.
             """)
 
-    print(colored("bold", "Entering the Nextstrain runtime"))
+    print(colored("bold", f"Entering the Nextstrain runtime ({runner_name(opts.__runner__)})"))
     print()
 
     if opts.volumes and opts.__runner__ in {docker, singularity}:

nextstrain/cli/debug.py

@@ -8,5 +8,13 @@
     parent exceptions in an exception chain are omitted from handled errors.
 """
 from os import environ
+from sys import stderr
 
 DEBUGGING = bool(environ.get("NEXTSTRAIN_DEBUG"))
+
+if DEBUGGING:
+    def debug(*args):
+        print("DEBUG:", *args, file = stderr)
+else:
+    def debug(*args):
+        pass

nextstrain/cli/remote/s3.py

@@ -248,7 +248,7 @@ def exists(object: S3Object) -> bool:
         object.load()
         return True
     except ClientError as error:
-        if 404 == int(error.response['Error']['Code']):
+        if 404 == int(error.response.get("Error", {}).get("Code", 0)):
             return False
         else:
             raise

nextstrain/cli/runner/singularity.py

@@ -8,10 +8,13 @@
 
 import itertools
 import os
+import re
 import shutil
 import subprocess
+from functools import lru_cache
+from packaging.version import Version, InvalidVersion
 from pathlib import Path
-from typing import Iterable, List
+from typing import Iterable, List, Optional
 from urllib.parse import urlsplit
 from .. import config, hostenv
 from ..errors import UserError
@@ -40,24 +43,99 @@
              or "docker://nextstrain/base"
 
 
+SINGULARITY_MINIMUM_VERSION = "3.0.0"
+
 SINGULARITY_CONFIG_ENV = {
     # Store image caches in our runtime root instead of ~/.singularity/…
     "SINGULARITY_CACHEDIR": str(CACHE),
+
+    # PROMPT_COMMAND is used by Singularity 3.5.3 onwards to forcibly set PS1
+    # to "Singularity> " on the first evaluation.¹  This happens *after* our
+    # bashrc is evaluated, so our Nextstrain prompt is overwritten.
+    # Singularity appends to any existing PROMPT_COMMAND value, so use a
+    # well-placed comment char (#) to avoid evaluating what it appends.
+    # Additionally unset PROMPT_COMMAND the first time it's evaluated so this
+    # silly workaround doesn't happen on every prompt.
+    #
+    # We set this via the special-cased environment passthru instead of setting
+    # it via an --env arg because --env is only first available in 3.6.0.
+    #
+    # ¹ <https://github.com/sylabs/singularity/commit/30823afc>
+    #   <https://github.com/apptainer/singularity/pull/4616>
+    #   <https://github.com/apptainer/singularity/issues/2721>
+    "SINGULARITYENV_PROMPT_COMMAND": "unset PROMPT_COMMAND; #",
 }
 
-SINGULARITY_EXEC_ARGS = [
+# Not "… = lambda: [" due to mypy.  See commit history.
+def SINGULARITY_EXEC_ARGS(): return [
     # Increase isolation.
     #
     # In the future, we may find we want to use additional related flags to
     # further increase isolation.¹  Note, however, that we'll want to think
     # about the minimum Singularity version we want to support, as many flags
     # in this area are not available on older versions.
     #
-    # ¹ e.g. <https://docs.sylabs.io/guides/latest/user-guide/singularity_and_docker.html#docker-like-compat-flag>
+    #   --compat                (available since 3.9.0; a bundle option)
+    #     --containall            (available since 2.2; a bundle option)
+    #       --contain
+    #       --cleanenv
+    #       --ipc
+    #       --pid
+    #     --writable-tmpfs        (3.0.0)
+    #     --no-init               (3.0.0)
+    #     --no-umask              (3.7.0)
+    #     --no-eval               (3.10.0)
+    #
+    # We opt not to use the --compat bundle option itself mainly for broader
+    # version compatibility but also because what it includes will likely
+    # change over time with newer Singularity releases.  We'd rather a stable,
+    # predictable set of behaviour of our choosing that maximizes
+    # compatibility.
+    #
+    # The options we use here are compatible with Singularity 2.6.0 and newer.
+    #
+    # XXX TODO: Once Singularity 4.0 is released and widely available, we *may*
+    # consider switching from --compat to --oci² for a) stronger Docker-like
+    # isolation and b) no longer having to convert our Docker (OCI) images to
+    # Singularity (SIF) images.  Alternatively, we may want to keep this
+    # runtime as a "middle ground" between the relatively strict isolation of
+    # our Docker runtime and the much looser isolation of the Conda runtime.
+    # Not sure!
+    #   -trs, 23 May 2023
+    #
+    # ¹ <https://docs.sylabs.io/guides/latest/user-guide/singularity_and_docker.html#docker-like-compat-flag>
+    # ² <https://docs.sylabs.io/guides/latest/user-guide/oci_runtime.html#oci-mode>
     "--contain",
+
+    # Don't mount anything at all at the container's value of HOME.  This is
+    # necesary because --compat includes --containall which includes --contain
+    # which makes HOME in the container an empty temporary directory.
+    # --no-home is available since 2.6.0.
     "--no-home",
+
+    # Singularity really wants to default HOME inside the container to the
+    # value from outside the container, thus ignoring the value set by the
+    # upstream Docker image which is only used as a default by the Singularity
+    # image.  Singularity forbids using --env to directly override HOME, so
+    # instead we use --home <src>:<dst> with two empty values.  <src> doesn't
+    # apply because we use --no-home, and setting <dst> to an empty value
+    # allows the container's default to apply (thus avoiding hardcoding it
+    # here).
+    "--home", ":",
+
+    # Allow writes to the image filesystem, discarded at container exit, à la
+    # Docker.  Snakemake, for example, needs to be able to write to HOME
+    # (/nextstrain).
+    "--writable-tmpfs",
+
+    # Don't copy entire host environment.  We forward our own hostenv.
     "--cleanenv",
 
+    # Don't evaluate the entrypoint command line (e.g. arguments passed via
+    # `nextstrain build`) before exec-ing the entrypoint.  It leads to unwanted
+    # substitutions that happen too early.
+    *(["--no-eval"] if singularity_version_at_least("3.10.0") else []),
+
     # Since we use --no-home above, avoid warnings about not being able to cd
     # to $HOME (the default behaviour).  run() will override this by specifying
     # --pwd again.
@@ -111,7 +189,7 @@ def run(opts, argv, working_volume = None, extra_env = {}, cpus: int = None, mem
     }
 
     return exec_or_return([
-        "singularity", "run", *SINGULARITY_EXEC_ARGS,
+        "singularity", "run", *SINGULARITY_EXEC_ARGS(),
 
         # Map directories to bind mount into the container.
         *flatten(("--bind", "%s:%s:%s" % (v.src.resolve(strict = True), docker.mount_point(v), "rw" if v.writable else "ro"))
@@ -166,7 +244,7 @@ def test_setup() -> RunnerTestResults:
     def test_run():
         try:
             capture_output([
-                "singularity", "exec", *SINGULARITY_EXEC_ARGS,
+                "singularity", "exec", *SINGULARITY_EXEC_ARGS(),
 
                 # XXX TODO: We should test --bind, as that's maybe most likely
                 # to be adminstratively disabled, but it's a bit more ceremony
@@ -189,6 +267,8 @@ def test_run():
     return [
         ("singularity is installed",
             shutil.which("singularity") is not None),
+        (f"singularity version {singularity_version()} ≥ {SINGULARITY_MINIMUM_VERSION}",
+            singularity_version_at_least(SINGULARITY_MINIMUM_VERSION)),
         ("singularity works",
             test_run()),
     ]
@@ -374,6 +454,34 @@ def run_bash(script: str, image: str = DEFAULT_IMAGE) -> List[str]:
     Returns the output of the script as a list of strings.
     """
     return capture_output([
-        "singularity", "run", *SINGULARITY_EXEC_ARGS, image_path(image),
+        "singularity", "run", *SINGULARITY_EXEC_ARGS(), image_path(image),
             "bash", "-c", script
     ])
+
+
+@lru_cache(maxsize = None)
+def singularity_version_at_least(min_version: str) -> bool:
+    version = singularity_version()
+
+    if not version:
+        return False
+
+    return version >= Version(min_version)
+
+
+@lru_cache(maxsize = None)
+def singularity_version() -> Optional[Version]:
+    try:
+        raw_version = capture_output(["singularity", "version"])[0]
+    except (OSError, subprocess.CalledProcessError):
+        return None
+
+    try:
+        return Version(raw_version)
+    except InvalidVersion:
+        # Singularity sometimes reports a version like 3.11.1-bionic with a
+        # (for Python) non-standard suffix ("-bionic"), so try stripping it.
+        try:
+            return Version(re.sub(r'-.+$', '', raw_version))
+        except InvalidVersion:
+            return None

nextstrain/cli/util.py

@@ -20,6 +20,7 @@
 from textwrap import dedent, indent
 from wcmatch.glob import globmatch, GLOBSTAR, EXTGLOB, BRACE, MATCHBASE, NEGATE
 from .__version__ import __version__
+from .debug import debug
 from .types import RunnerModule, RunnerTestResults
 
 
@@ -276,6 +277,8 @@ def capture_output(argv, extra_env: Mapping = {}):
     If an *extra_env* mapping is passed, the provided keys and values are
     overlayed onto the current environment.
     """
+    debug(f"capture_output({argv!r}, {extra_env!r})")
+
     env = os.environ.copy()
 
     if extra_env:
@@ -308,6 +311,8 @@ def exec_or_return(argv: List[str], extra_env: Mapping = {}) -> int:
 
     ¹ https://bugs.python.org/issue9148
     """
+    debug(f"exec_or_return({argv!r}, {extra_env!r})")
+
     env = os.environ.copy()
 
     if extra_env:

setup.py

@@ -138,6 +138,8 @@ def find_namespaced_packages(namespace):
             "sphinx-autobuild",
             "sphinx-markdown-tables !=0.0.16",
             "sphinx_rtd_theme",
+            "types-boto3",
+            "types-botocore",
             "types-docutils",
             "types-setuptools",
             "types-requests; python_version != '3.6'",