diff --git a/CHANGES.md b/CHANGES.md index 1ed8e825..3c33aa2a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,61 @@ development source code and as such may not be routinely kept up to date. # __NEXT__ +This release is mostly a bug fix release for our Conda and Singularity +runtimes. However, it contains a **potentially-breaking change** for existing +usages of the Singularity runtime: **the minimum required Singularity version +has changed from 2.6.0 to 3.0.0**. This change was required for a critical bug +fix. If you do not use the Singularity runtime, there are no +potentially-breaking changes in this release. + +## Improvements + +* `nextstrain shell` now notes which runtime is being entered in its initial + messaging to establish more context for the user (and for developers when + troubleshooting). + ([#283][]) + +* The Singularity runtime now checks for the minimum required Singularity + version (3.0.0 with this release) during `nextstrain check-setup`. + ([#283][]) + +## Bug fixes + +* Setup and upgrade of the Conda runtime now only uses stable "main" channel + releases when determining the latest release version, as intended. + Previously, testing and development releases could be selected if they were + newer than the last stable release. Additionally, if there are multiple + builds for a release version, the highest numbered build (i.e. newest) is now + used instead of the lowest. + ([#280](https://github.com/nextstrain/cli/pull/280)) + +* The Singularity runtime now works with our container runtime images from + `build-20230411T103027Z` onwards. The Snakemake upgrade in that image + version resulted in "read-only file system" errors which referenced the + user's home directory. Those errors are now fixed. + ([#283][]) + +* The prompt for `nextstrain shell`—a stylized variant of the Nextstrain + wordmark—now works when using the Singularity runtime regardless of + Singularity version. Previously Singularity's default prompt of + `Singularity> ` overrode ours when using Singularity versions ≥3.5.3. + ([#283][]) + +* More robust command-line processing is used for the Singularity runtime on + Singularity versions ≥3.10.0. Singularity's early (and unexpected) + evaluation of arguments that look like (but aren't) shell variable + substitutions is disabled. + ([#283][]) + +## Development + +* The command lines and environment overrides of many (but not all) process + invocations are now logged when `NEXTSTRAIN_DEBUG` is enabled. + ([#283][]) + + +[#283]: https://github.com/nextstrain/cli/pull/283 + # 6.2.1 (24 March 2023) diff --git a/doc/installation.rst b/doc/installation.rst index 468de179..9d9d67d3 100644 --- a/doc/installation.rst +++ b/doc/installation.rst @@ -193,6 +193,8 @@ dependencies as validated versions are already bundled into a container image by the Nextstrain team. Run ``nextstrain setup singularity`` to get started. +Singularity version 3.0.0 or newer is required, but we recommend at least +version 3.10.0 or newer when possible. Note that the Singularity project forked into two separate projects in late 2021: `SingularityCE`_ under `Sylabs`_ and `Apptainer`_ under the `Linux diff --git a/nextstrain/cli/command/shell.py b/nextstrain/cli/command/shell.py index e63a5583..406dd980 100644 --- a/nextstrain/cli/command/shell.py +++ b/nextstrain/cli/command/shell.py @@ -61,7 +61,7 @@ def run(opts): Use the Docker or Singularity runtimes (via --docker or --singularity) if overlays are necessary. """) - print(colored("bold", "Entering the Nextstrain runtime")) + print(colored("bold", f"Entering the Nextstrain runtime ({runner_name(opts.__runner__)})")) print() if opts.volumes and opts.__runner__ in {docker, singularity}: diff --git a/nextstrain/cli/debug.py b/nextstrain/cli/debug.py index 56f13abb..cac16bba 100644 --- a/nextstrain/cli/debug.py +++ b/nextstrain/cli/debug.py @@ -8,5 +8,13 @@ parent exceptions in an exception chain are omitted from handled errors. """ from os import environ +from sys import stderr DEBUGGING = bool(environ.get("NEXTSTRAIN_DEBUG")) + +if DEBUGGING: + def debug(*args): + print("DEBUG:", *args, file = stderr) +else: + def debug(*args): + pass diff --git a/nextstrain/cli/remote/s3.py b/nextstrain/cli/remote/s3.py index dddee5c9..a90d84a3 100644 --- a/nextstrain/cli/remote/s3.py +++ b/nextstrain/cli/remote/s3.py @@ -248,7 +248,7 @@ def exists(object: S3Object) -> bool: object.load() return True except ClientError as error: - if 404 == int(error.response['Error']['Code']): + if 404 == int(error.response.get("Error", {}).get("Code", 0)): return False else: raise diff --git a/nextstrain/cli/runner/singularity.py b/nextstrain/cli/runner/singularity.py index 9e7a6713..28a22e5e 100644 --- a/nextstrain/cli/runner/singularity.py +++ b/nextstrain/cli/runner/singularity.py @@ -8,10 +8,13 @@ import itertools import os +import re import shutil import subprocess +from functools import lru_cache +from packaging.version import Version, InvalidVersion from pathlib import Path -from typing import Iterable, List +from typing import Iterable, List, Optional from urllib.parse import urlsplit from .. import config, hostenv from ..errors import UserError @@ -40,12 +43,31 @@ or "docker://nextstrain/base" +SINGULARITY_MINIMUM_VERSION = "3.0.0" + SINGULARITY_CONFIG_ENV = { # Store image caches in our runtime root instead of ~/.singularity/… "SINGULARITY_CACHEDIR": str(CACHE), + + # PROMPT_COMMAND is used by Singularity 3.5.3 onwards to forcibly set PS1 + # to "Singularity> " on the first evaluation.¹ This happens *after* our + # bashrc is evaluated, so our Nextstrain prompt is overwritten. + # Singularity appends to any existing PROMPT_COMMAND value, so use a + # well-placed comment char (#) to avoid evaluating what it appends. + # Additionally unset PROMPT_COMMAND the first time it's evaluated so this + # silly workaround doesn't happen on every prompt. + # + # We set this via the special-cased environment passthru instead of setting + # it via an --env arg because --env is only first available in 3.6.0. + # + # ¹ + # + # + "SINGULARITYENV_PROMPT_COMMAND": "unset PROMPT_COMMAND; #", } -SINGULARITY_EXEC_ARGS = [ +# Not "… = lambda: [" due to mypy. See commit history. +def SINGULARITY_EXEC_ARGS(): return [ # Increase isolation. # # In the future, we may find we want to use additional related flags to @@ -53,11 +75,67 @@ # about the minimum Singularity version we want to support, as many flags # in this area are not available on older versions. # - # ¹ e.g. + # --compat (available since 3.9.0; a bundle option) + # --containall (available since 2.2; a bundle option) + # --contain + # --cleanenv + # --ipc + # --pid + # --writable-tmpfs (3.0.0) + # --no-init (3.0.0) + # --no-umask (3.7.0) + # --no-eval (3.10.0) + # + # We opt not to use the --compat bundle option itself mainly for broader + # version compatibility but also because what it includes will likely + # change over time with newer Singularity releases. We'd rather a stable, + # predictable set of behaviour of our choosing that maximizes + # compatibility. + # + # The options we use here are compatible with Singularity 2.6.0 and newer. + # + # XXX TODO: Once Singularity 4.0 is released and widely available, we *may* + # consider switching from --compat to --oci² for a) stronger Docker-like + # isolation and b) no longer having to convert our Docker (OCI) images to + # Singularity (SIF) images. Alternatively, we may want to keep this + # runtime as a "middle ground" between the relatively strict isolation of + # our Docker runtime and the much looser isolation of the Conda runtime. + # Not sure! + # -trs, 23 May 2023 + # + # ¹ + # ² "--contain", + + # Don't mount anything at all at the container's value of HOME. This is + # necesary because --compat includes --containall which includes --contain + # which makes HOME in the container an empty temporary directory. + # --no-home is available since 2.6.0. "--no-home", + + # Singularity really wants to default HOME inside the container to the + # value from outside the container, thus ignoring the value set by the + # upstream Docker image which is only used as a default by the Singularity + # image. Singularity forbids using --env to directly override HOME, so + # instead we use --home : with two empty values. doesn't + # apply because we use --no-home, and setting to an empty value + # allows the container's default to apply (thus avoiding hardcoding it + # here). + "--home", ":", + + # Allow writes to the image filesystem, discarded at container exit, à la + # Docker. Snakemake, for example, needs to be able to write to HOME + # (/nextstrain). + "--writable-tmpfs", + + # Don't copy entire host environment. We forward our own hostenv. "--cleanenv", + # Don't evaluate the entrypoint command line (e.g. arguments passed via + # `nextstrain build`) before exec-ing the entrypoint. It leads to unwanted + # substitutions that happen too early. + *(["--no-eval"] if singularity_version_at_least("3.10.0") else []), + # Since we use --no-home above, avoid warnings about not being able to cd # to $HOME (the default behaviour). run() will override this by specifying # --pwd again. @@ -111,7 +189,7 @@ def run(opts, argv, working_volume = None, extra_env = {}, cpus: int = None, mem } return exec_or_return([ - "singularity", "run", *SINGULARITY_EXEC_ARGS, + "singularity", "run", *SINGULARITY_EXEC_ARGS(), # Map directories to bind mount into the container. *flatten(("--bind", "%s:%s:%s" % (v.src.resolve(strict = True), docker.mount_point(v), "rw" if v.writable else "ro")) @@ -166,7 +244,7 @@ def test_setup() -> RunnerTestResults: def test_run(): try: capture_output([ - "singularity", "exec", *SINGULARITY_EXEC_ARGS, + "singularity", "exec", *SINGULARITY_EXEC_ARGS(), # XXX TODO: We should test --bind, as that's maybe most likely # to be adminstratively disabled, but it's a bit more ceremony @@ -189,6 +267,8 @@ def test_run(): return [ ("singularity is installed", shutil.which("singularity") is not None), + (f"singularity version {singularity_version()} ≥ {SINGULARITY_MINIMUM_VERSION}", + singularity_version_at_least(SINGULARITY_MINIMUM_VERSION)), ("singularity works", test_run()), ] @@ -374,6 +454,34 @@ def run_bash(script: str, image: str = DEFAULT_IMAGE) -> List[str]: Returns the output of the script as a list of strings. """ return capture_output([ - "singularity", "run", *SINGULARITY_EXEC_ARGS, image_path(image), + "singularity", "run", *SINGULARITY_EXEC_ARGS(), image_path(image), "bash", "-c", script ]) + + +@lru_cache(maxsize = None) +def singularity_version_at_least(min_version: str) -> bool: + version = singularity_version() + + if not version: + return False + + return version >= Version(min_version) + + +@lru_cache(maxsize = None) +def singularity_version() -> Optional[Version]: + try: + raw_version = capture_output(["singularity", "version"])[0] + except (OSError, subprocess.CalledProcessError): + return None + + try: + return Version(raw_version) + except InvalidVersion: + # Singularity sometimes reports a version like 3.11.1-bionic with a + # (for Python) non-standard suffix ("-bionic"), so try stripping it. + try: + return Version(re.sub(r'-.+$', '', raw_version)) + except InvalidVersion: + return None diff --git a/nextstrain/cli/util.py b/nextstrain/cli/util.py index 8e09c54a..98be7239 100644 --- a/nextstrain/cli/util.py +++ b/nextstrain/cli/util.py @@ -20,6 +20,7 @@ from textwrap import dedent, indent from wcmatch.glob import globmatch, GLOBSTAR, EXTGLOB, BRACE, MATCHBASE, NEGATE from .__version__ import __version__ +from .debug import debug from .types import RunnerModule, RunnerTestResults @@ -276,6 +277,8 @@ def capture_output(argv, extra_env: Mapping = {}): If an *extra_env* mapping is passed, the provided keys and values are overlayed onto the current environment. """ + debug(f"capture_output({argv!r}, {extra_env!r})") + env = os.environ.copy() if extra_env: @@ -308,6 +311,8 @@ def exec_or_return(argv: List[str], extra_env: Mapping = {}) -> int: ¹ https://bugs.python.org/issue9148 """ + debug(f"exec_or_return({argv!r}, {extra_env!r})") + env = os.environ.copy() if extra_env: diff --git a/setup.py b/setup.py index 041b11ca..06a7f707 100644 --- a/setup.py +++ b/setup.py @@ -138,6 +138,8 @@ def find_namespaced_packages(namespace): "sphinx-autobuild", "sphinx-markdown-tables !=0.0.16", "sphinx_rtd_theme", + "types-boto3", + "types-botocore", "types-docutils", "types-setuptools", "types-requests; python_version != '3.6'",