Merge branch 'dev' into schema

CHANGELOG.md

@@ -17,11 +17,14 @@
 - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors.
 - Add actions workflow to respond to `@nf-core-bot fix linting` comments on nf-core/tools PRs
 - Linting: Don't allow a `.nf-core.yaml` file, should be `.yml` ([#1515](https://github.com/nf-core/tools/pull/1515)).
-- Not all definitions in JSON schema have a "properties", leading to an error ([#1419](https://github.com/nf-core/tools/issues/1419))
+- Remove empty JSON schema definition groups to avoid usage errors ([#1419](https://github.com/nf-core/tools/issues/1419))
+- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520))
 
 ### Modules
 
 - Escaped test run output before logging it, to avoid a rich ` MarkupError`
+- Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image.
+- Add a new command `nf-core modules test` which runs pytests locally.
 
 ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24]
 

nf_core/__main__.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 """ nf-core: Helper tools for use with nf-core Nextflow pipelines. """
-
 from rich import print
 import logging
 import os
@@ -49,7 +48,7 @@
  },
  {
  "name": "Developing new modules",
- "commands": ["create", "create-test-yml", "lint", "bump-versions"],
+ "commands": ["create", "create-test-yml", "lint", "bump-versions", "mulled", "test"],
  },
  ],
 }
@@ -674,6 +673,62 @@ def bump_versions(ctx, tool, dir, all, show_all):
  sys.exit(1)
 
 
+# nf-core modules mulled
+@modules.command()
+@click.argument("specifications", required=True, nargs=-1, metavar="<tool==version> <...>")
+@click.option(
+ "--build-number",
+ type=int,
+ default=0,
+ show_default=True,
+ metavar="<number>",
+ help="The build number for this image. This is an incremental value that starts at zero.",
+)
+def mulled(specifications, build_number):
+ """
+ Generate the name of a BioContainers mulled image version 2.
+
+ When you know the specific dependencies and their versions of a multi-tool container image and you need the name of
+ that image, this command can generate it for you.
+
+ """
+ from nf_core.modules.mulled import MulledImageNameGenerator
+
+ image_name = MulledImageNameGenerator.generate_image_name(
+ MulledImageNameGenerator.parse_targets(specifications), build_number=build_number
+ )
+ print(image_name)
+ if not MulledImageNameGenerator.image_exists(image_name):
+ log.error(
+ "The generated multi-tool container image name does not seem to exist yet. Please double check that your "
+ "provided combination of tools and versions exists in the file:\n"
+ "https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv\n"
+ "If it does not, please add your desired combination as detailed at:\n"
+ "https://github.com/BioContainers/multi-package-containers\n"
+ )
+ sys.exit(1)
+
+
+# nf-core modules test
+@modules.command("test")
+@click.pass_context
+@click.argument("tool", type=str, required=False, metavar="<tool> or <tool/subtool>")
+@click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting")
+@click.option("-a", "--pytest_args", type=str, required=False, multiple=True, help="Additional pytest arguments")
+def test_module(ctx, tool, no_prompts, pytest_args):
+ """
+ Run module tests locally.
+
+ Given the name of a module, runs the Nextflow test command.
+ """
+ try:
+ meta_builder = nf_core.modules.ModulesTest(tool, no_prompts, pytest_args)
+ meta_builder.run()
+ except UserWarning as e:
+ log.critical(e)
+ sys.exit(1)
+
+
 # nf-core schema subcommands
 @nf_core_cli.group()
 def schema():

nf_core/launch.py

@@ -420,7 +420,7 @@ def prompt_param(self, param_id, param_obj, is_required, answers):
 
  # If required and got an empty reponse, ask again
  while type(answer[param_id]) is str and answer[param_id].strip() == "" and is_required:
- log.error("'–-{}' is required".format(param_id))
+ log.error("'--{}' is required".format(param_id))
  answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style)
 
  # Ignore if empty

nf_core/lint/actions_ci.py

@@ -137,7 +137,7 @@ def actions_ci(self):
  # Check that we are testing the minimum nextflow version
  try:
  matrix = ciwf["jobs"]["test"]["strategy"]["matrix"]["include"]
- assert any([i["NXF_VER"] == self.minNextflowVersion for i in matrix])
+ assert any([i.get("NXF_VER") == self.minNextflowVersion for i in matrix])
  except (KeyError, TypeError):
  failed.append("'.github/workflows/ci.yml' does not check minimum NF version")
  except AssertionError:

nf_core/modules/__init__.py

@@ -9,3 +9,5 @@
 from .update import ModuleUpdate
 from .remove import ModuleRemove
 from .info import ModuleInfo
+from .mulled import MulledImageNameGenerator
+from .module_test import ModulesTest

nf_core/modules/install.py

@@ -130,6 +130,10 @@ def install(self, module):
  if not self.download_module_file(module, version, self.modules_repo, install_folder):
  return False
 
+ # Print include statement
+ module_name = "_".join(module.upper().split("/"))
+ log.info(f"Include statement: include {{ {module_name} }} from '.{os.path.join(*install_folder, module)}/main’")
+
  # Update module.json with newly installed module
  self.update_modules_json(modules_json, self.modules_repo.name, module, version)
  return True

nf_core/modules/module_test.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+"""
+The ModulesTest class runs the tests locally
+"""
+
+import logging
+import questionary
+import os
+import pytest
+import sys
+import rich
+from pathlib import Path
+from shutil import which
+
+import nf_core.utils
+from .modules_repo import ModulesRepo
+
+log = logging.getLogger(__name__)
+
+
+class ModulesTest(object):
+ """
+ Class to run module pytests.
+
+ ...
+
+ Attributes
+ ----------
+ module_name : str
+ name of the tool to run tests for
+ no_prompts : bool
+ flat indicating if prompts are used
+ pytest_args : tuple
+ additional arguments passed to pytest command
+
+ Methods
+ -------
+ run():
+ Run test steps
+ _check_inputs():
+ Check inputs. Ask for module_name if not provided and check that the directory exists
+ _set_profile():
+ Set software profile
+ _run_pytests(self):
+ Run pytest
+ """
+
+ def __init__(
+ self,
+ module_name=None,
+ no_prompts=False,
+ pytest_args="",
+ ):
+ self.module_name = module_name
+ self.no_prompts = no_prompts
+ self.pytest_args = pytest_args
+
+ def run(self):
+ """Run test steps"""
+ if not self.no_prompts:
+ log.info(
+ "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses"
+ )
+ self._check_inputs()
+ self._set_profile()
+ self._check_profile()
+ self._run_pytests()
+
+ def _check_inputs(self):
+ """Do more complex checks about supplied flags."""
+
+ # Get the tool name if not specified
+ if self.module_name is None:
+ if self.no_prompts:
+ raise UserWarning(
+ "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL."
+ )
+ modules_repo = ModulesRepo()
+ modules_repo.get_modules_file_tree()
+ self.module_name = questionary.autocomplete(
+ "Tool name:",
+ choices=modules_repo.modules_avail_module_names,
+ style=nf_core.utils.nfcore_question_style,
+ ).ask()
+ module_dir = Path("modules") / self.module_name
+
+ # First, sanity check that the module directory exists
+ if not module_dir.is_dir():
+ raise UserWarning(
+ f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?"
+ )
+
+ def _set_profile(self):
+ """Set $PROFILE env variable.
+ The config expects $PROFILE and Nextflow fails if it's not set.
+ """
+ if os.environ.get("PROFILE") is None:
+ os.environ["PROFILE"] = ""
+ if self.no_prompts:
+ log.info(
+ "Setting environment variable '$PROFILE' to an empty string as not set.\n"
+ "Tests will run with Docker by default. "
+ "To use Singularity set 'export PROFILE=singularity' in your shell before running this command."
+ )
+ else:
+ question = {
+ "type": "list",
+ "name": "profile",
+ "message": "Choose software profile",
+ "choices": ["Docker", "Singularity", "Conda"],
+ }
+ answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style)
+ profile = answer["profile"].lower()
+ os.environ["PROFILE"] = profile
+ log.info(f"Setting environment variable '$PROFILE' to '{profile}'")
+
+ def _check_profile(self):
+ """Check if profile is available"""
+ profile = os.environ.get("PROFILE")
+ # Make sure the profile read from the environment is a valid Nextflow profile.
+ valid_nextflow_profiles = ["docker", "singularity", "conda"]
+ if profile in valid_nextflow_profiles:
+ if not which(profile):
+ raise UserWarning(f"Command '{profile}' not found - is it installed?")
+ else:
+ raise UserWarning(
+ f"The PROFILE '{profile}' set in the shell environment is not valid.\n"
+ f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'."
+ )
+
+ def _run_pytests(self):
+ """Given a module name, run tests."""
+ # Print nice divider line
+ console = rich.console.Console()
+ console.rule(self.module_name, style="black")
+
+ # Set pytest arguments
+ command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"]
+ command_args += self.pytest_args
+
+ # Run pytest
+ log.info(f"Running pytest for module '{self.module_name}'")
+ sys.exit(pytest.main(command_args))

nf_core/modules/mulled.py

@@ -0,0 +1,67 @@
+"""Generate the name of a BioContainers mulled image version 2."""
+
+
+import logging
+import re
+from packaging.version import Version, InvalidVersion
+from typing import Iterable, Tuple, List
+
+import requests
+from galaxy.tool_util.deps.mulled.util import build_target, v2_image_name
+
+
+log = logging.getLogger(__name__)
+
+
+class MulledImageNameGenerator:
+ """
+ Define a service class for generating BioContainers version 2 mulled image names.
+
+ Adapted from https://gist.github.com/natefoo/19cefeedd1942c30f9d88027a61b3f83.
+
+ """
+
+ _split_pattern = re.compile(r"==?")
+
+ @classmethod
+ def parse_targets(cls, specifications: Iterable[str]) -> List[Tuple[str, str]]:
+ """
+ Parse tool, version pairs from specification strings.
+
+ Args:
+ specifications: An iterable of strings that contain tools and their versions.
+
+ """
+ result = []
+ for spec in specifications:
+ try:
+ tool, version = cls._split_pattern.split(spec, maxsplit=1)
+ except ValueError:
+ raise ValueError(
+ f"The specification {spec} does not have the expected format <tool==version> or <tool=version>."
+ ) from None
+ try:
+ Version(version)
+ except InvalidVersion:
+ raise ValueError(f"{version} in {spec} is not a PEP440 compliant version specification.") from None
+ result.append((tool.strip(), version.strip()))
+ return result
+
+ @classmethod
+ def generate_image_name(cls, targets: Iterable[Tuple[str, str]], build_number: int = 0) -> str:
+ """
+ Generate the name of a BioContainers mulled image version 2.
+
+ Args:
+ targets: One or more tool, version pairs of the multi-tool container image.
+ build_number: The build number for this image. This is an incremental value that starts at zero.
+
+ """
+ return v2_image_name([build_target(name, version) for name, version in targets], image_build=str(build_number))
+
+ @classmethod
+ def image_exists(cls, image_name: str) -> bool:
+ """Check whether a given BioContainers image name exists via a call to the quay.io API."""
+ response = requests.get(f"https://quay.io/biocontainers/{image_name}/", allow_redirects=True)
+ log.debug(response.text)
+ return response.status_code == 200

pytest.ini

@@ -3,3 +3,4 @@ filterwarnings =
  ignore::pytest.PytestRemovedIn8Warning:_pytest.nodes:140
 testpaths =
  tests
+python_files = test_*.py

requirements.txt

@@ -1,4 +1,5 @@
 click
+galaxy-tool-util
 GitPython
 jinja2
 jsonschema>=3.0

tests/modules/module_test.py

@@ -0,0 +1,27 @@
+"""Test the 'modules test' command which runs module pytests."""
+import os
+import pytest
+
+import nf_core.modules
+
+
+def test_modules_test_check_inputs(self):
+ """Test the check_inputs() function - raise UserWarning because module doesn't exist"""
+ cwd = os.getcwd()
+ os.chdir(self.nfcore_modules)
+ meta_builder = nf_core.modules.ModulesTest("none", True, "")
+ with pytest.raises(UserWarning) as excinfo:
+ meta_builder._check_inputs()
+ os.chdir(cwd)
+ assert "Cannot find directory" in str(excinfo.value)
+
+
+def test_modules_test_no_name_no_prompts(self):
+ """Test the check_inputs() function - raise UserWarning prompts are deactivated and module name is not provided."""
+ cwd = os.getcwd()
+ os.chdir(self.nfcore_modules)
+ meta_builder = nf_core.modules.ModulesTest(None, True, "")
+ with pytest.raises(UserWarning) as excinfo:
+ meta_builder._check_inputs()
+ os.chdir(cwd)
+ assert "Tool name not provided and prompts deactivated." in str(excinfo.value)

tests/test_modules.py

@@ -113,3 +113,8 @@ def test_modulesrepo_class(self):
  test_modules_bump_versions_fail,
  test_modules_bump_versions_fail_unknown_version,
  )
+
+ from .modules.module_test import (
+ test_modules_test_check_inputs,
+ test_modules_test_no_name_no_prompts,
+ )