From 82bbba87e983c848a29f1a828d46e4dde901ab1d Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Wed, 20 Apr 2022 22:03:41 +0200 Subject: [PATCH 01/37] feat: add a modules mulled command --- nf_core/__main__.py | 20 ++++++++++++- nf_core/modules/__init__.py | 1 + nf_core/modules/mulled.py | 58 +++++++++++++++++++++++++++++++++++ requirements.txt | 1 + tests/test_mullled.py | 60 +++++++++++++++++++++++++++++++++++++ 5 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 nf_core/modules/mulled.py create mode 100644 tests/test_mullled.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 8affd23bfe..569a45d657 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -49,7 +49,7 @@ }, { "name": "Developing new modules", - "commands": ["create", "create-test-yml", "lint", "bump-versions"], + "commands": ["create", "create-test-yml", "lint", "bump-versions", "mulled"], }, ], } @@ -674,6 +674,24 @@ def bump_versions(ctx, tool, dir, all, show_all): sys.exit(1) +# nf-core modules mulled +@modules.command() +@click.argument("tools", required=True, nargs=-1, metavar=" <...>") +def mulled(tools): + """ + Generate the name of a BioContainers mulled image version 2. + + When you know the specific dependencies and their versions of a multi-tool container image and you need the name of + that image, this command can generate it for you. + + """ + print( + nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( + nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(tools) + ) + ) + + # nf-core schema subcommands @nf_core_cli.group() def schema(): diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index f833d52564..6b999aea50 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -9,3 +9,4 @@ from .update import ModuleUpdate from .remove import ModuleRemove from .info import ModuleInfo +from .mulled import MulledImageNameGenerator diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py new file mode 100644 index 0000000000..b84561a5c1 --- /dev/null +++ b/nf_core/modules/mulled.py @@ -0,0 +1,58 @@ +"""Generate the name of a BioContainers mulled image version 2.""" + + +import logging +import re +from packaging.version import Version, InvalidVersion +from typing import Iterable, Tuple, List + +from galaxy.tool_util.deps.mulled.util import build_target, v2_image_name + + +log = logging.getLogger(__name__) + + +class MulledImageNameGenerator: + """ + Define a service class for generating BioContainers version 2 mulled image names. + + Adapted from https://gist.github.com/natefoo/19cefeedd1942c30f9d88027a61b3f83. + + """ + + _split_pattern = re.compile(r"==?") + + @classmethod + def parse_targets(cls, tools: Iterable[str]) -> List[Tuple[str, str]]: + """ + Parse tool, version pairs from two or more version strings. + + Args: + tools: An iterable of strings that contain tools and their versions. + + """ + result = [] + for spec in tools: + try: + tool, version = cls._split_pattern.split(spec, maxsplit=1) + except ValueError: + raise ValueError( + f"The specification {spec} does not have the expected format or ." + ) from None + try: + Version(version) + except InvalidVersion: + raise ValueError(f"{version} in {spec} is not a PEP440 compliant version specification.") from None + result.append((tool.strip(), version.strip())) + return result + + @classmethod + def generate_image_name(cls, targets: Iterable[Tuple[str, str]]) -> str: + """ + Generate the name of a BioContainers mulled image version 2. + + Args: + targets: One or more tool, version pairs of the multi-tool container image. + + """ + return v2_image_name([build_target(name, version) for name, version in targets]) diff --git a/requirements.txt b/requirements.txt index 97ed43df61..798ed3470b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ click +galaxy-tool-util GitPython jinja2 jsonschema>=3.0 diff --git a/tests/test_mullled.py b/tests/test_mullled.py new file mode 100644 index 0000000000..2206d65eef --- /dev/null +++ b/tests/test_mullled.py @@ -0,0 +1,60 @@ +"""Test the mulled BioContainers image name generation.""" + +import pytest + +from nf_core.modules import MulledImageNameGenerator + + +@pytest.mark.parametrize( + "specs, expected", + [ + (["foo==0.1.2", "bar==1.1"], [("foo", "0.1.2"), ("bar", "1.1")]), + (["foo=0.1.2", "bar=1.1"], [("foo", "0.1.2"), ("bar", "1.1")]), + ], +) +def test_target_parsing(specs, expected): + """""" + assert MulledImageNameGenerator.parse_targets(specs) == expected + + +@pytest.mark.parametrize( + "specs", + [ + ["foo<0.1.2", "bar==1.1"], + ["foo=0.1.2", "bar>1.1"], + ], +) +def test_wrong_specification(specs): + """""" + with pytest.raises(ValueError, match="expected format"): + MulledImageNameGenerator.parse_targets(specs) + + +@pytest.mark.parametrize( + "specs", + [ + ["foo==0a.1.2", "bar==1.1"], + ["foo==0.1.2", "bar==1.b1b"], + ], +) +def test_noncompliant_version(specs): + """""" + with pytest.raises(ValueError, match="PEP440"): + MulledImageNameGenerator.parse_targets(specs) + + +@pytest.mark.parametrize( + "specs, expected", + [ + ( + [("chromap", "0.2.1"), ("samtools", "1.15")], + "mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:bd74d08a359024829a7aec1638a28607bbcd8a58", + ), + ( + [("pysam", "0.16.0.1"), ("biopython", "1.78")], + "mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f", + ), + ], +) +def test_generate_image_name(specs, expected): + assert MulledImageNameGenerator.generate_image_name(specs) == expected From 4dc3c3e4b1c8fc436739e18288a0d43052605097 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Wed, 20 Apr 2022 22:07:29 +0200 Subject: [PATCH 02/37] chore: add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57a4cff8ea..86a8206b91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ ### Modules - Escaped test run output before logging it, to avoid a rich ` MarkupError` +- Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image. ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] From 5c4544071d1a85ef60f9ebb033a710da239007f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 21 Apr 2022 10:35:36 +0200 Subject: [PATCH 03/37] print include statement when installing a module --- nf_core/modules/install.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index 673e1d7f4e..a72c45da76 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -130,6 +130,14 @@ def install(self, module): if not self.download_module_file(module, version, self.modules_repo, install_folder): return False + # Print include statement + if module == "stringtie/stringtie": + # Only with stringtie the process name is STRINGTIE instead of STRINGTIE_STRINGTIE + module_name = module.upper().split("/")[0] + else: + module_name = "_".join(module.upper().split("/")) + log.info(f"Include statement: include {{ {module_name} }} from '.{os.path.join(*install_folder, module)}/main’") + # Update module.json with newly installed module self.update_modules_json(modules_json, self.modules_repo.name, module, version) return True From cd57124cbafe400374fee0daf7b711947b4c9387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 21 Apr 2022 10:59:50 +0200 Subject: [PATCH 04/37] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fc9e1067d..89d265bf04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Fix Prettier formatting bug in completion email HTML template ([#1509](https://github.com/nf-core/tools/issues/1509)) - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself - Add `.prettierignore` file to stop Prettier linting tests from running over test files +- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) ### General From b1e255957470f1ff83b897584f4cc45117b1d2f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 21 Apr 2022 11:04:56 +0200 Subject: [PATCH 05/37] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89d265bf04..378f14f724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,12 @@ - Fix Prettier formatting bug in completion email HTML template ([#1509](https://github.com/nf-core/tools/issues/1509)) - Removed retry strategy for AWS tests CI, as Nextflow now handles spot instance retries itself - Add `.prettierignore` file to stop Prettier linting tests from running over test files -- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) ### General - Bumped the minimum version of `rich` from `v10` to `v10.7.0` - Add an empty line to `modules.json`, `params.json` and `nextflow-schema.json` when dumping them to avoid prettier errors. +- Print include statement to terminal when `modules install` ([#1520](https://github.com/nf-core/tools/pull/1520)) ### Modules From 9fec61b4f2bed1dedde65b7ae4078fe2c1cb23c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 21 Apr 2022 13:52:56 +0200 Subject: [PATCH 06/37] remove remove stringtie special case --- nf_core/modules/install.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index a72c45da76..96f04cb34c 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -131,11 +131,7 @@ def install(self, module): return False # Print include statement - if module == "stringtie/stringtie": - # Only with stringtie the process name is STRINGTIE instead of STRINGTIE_STRINGTIE - module_name = module.upper().split("/")[0] - else: - module_name = "_".join(module.upper().split("/")) + module_name = "_".join(module.upper().split("/")) log.info(f"Include statement: include {{ {module_name} }} from '.{os.path.join(*install_folder, module)}/main’") # Update module.json with newly installed module From 6254c7ca7a0e50b6225b4f82268f672f0e0718c1 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 15:23:45 +0200 Subject: [PATCH 07/37] tests: add docstrings to test cases --- tests/test_mullled.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_mullled.py b/tests/test_mullled.py index 2206d65eef..5eaa41fc81 100644 --- a/tests/test_mullled.py +++ b/tests/test_mullled.py @@ -13,7 +13,7 @@ ], ) def test_target_parsing(specs, expected): - """""" + """Test that valid specifications are correctly parsed into tool, version pairs.""" assert MulledImageNameGenerator.parse_targets(specs) == expected @@ -25,7 +25,7 @@ def test_target_parsing(specs, expected): ], ) def test_wrong_specification(specs): - """""" + """Test that unexpected version constraints fail.""" with pytest.raises(ValueError, match="expected format"): MulledImageNameGenerator.parse_targets(specs) @@ -38,7 +38,7 @@ def test_wrong_specification(specs): ], ) def test_noncompliant_version(specs): - """""" + """Test that version string that do not comply with PEP440 fail.""" with pytest.raises(ValueError, match="PEP440"): MulledImageNameGenerator.parse_targets(specs) @@ -57,4 +57,5 @@ def test_noncompliant_version(specs): ], ) def test_generate_image_name(specs, expected): + """Test that a known image name is generated from given targets.""" assert MulledImageNameGenerator.generate_image_name(specs) == expected From 4ce681d429260c3399aeb0ec3423c2faa4e51f41 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 15:31:07 +0200 Subject: [PATCH 08/37] refactor: rename tools parameter to specifications --- .editorconfig | 1 + nf_core/__main__.py | 6 +++--- nf_core/modules/mulled.py | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.editorconfig b/.editorconfig index c29b6c7eee..0ffa5e3ec1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,6 +7,7 @@ insert_final_newline = true trim_trailing_whitespace = true indent_size = 4 indent_style = space +max_line_length = 120 [*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 569a45d657..49389fd9e6 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -676,8 +676,8 @@ def bump_versions(ctx, tool, dir, all, show_all): # nf-core modules mulled @modules.command() -@click.argument("tools", required=True, nargs=-1, metavar=" <...>") -def mulled(tools): +@click.argument("specifications", required=True, nargs=-1, metavar=" <...>") +def mulled(specifications): """ Generate the name of a BioContainers mulled image version 2. @@ -687,7 +687,7 @@ def mulled(tools): """ print( nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( - nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(tools) + nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications) ) ) diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py index b84561a5c1..197913c99e 100644 --- a/nf_core/modules/mulled.py +++ b/nf_core/modules/mulled.py @@ -23,16 +23,16 @@ class MulledImageNameGenerator: _split_pattern = re.compile(r"==?") @classmethod - def parse_targets(cls, tools: Iterable[str]) -> List[Tuple[str, str]]: + def parse_targets(cls, specifications: Iterable[str]) -> List[Tuple[str, str]]: """ - Parse tool, version pairs from two or more version strings. + Parse tool, version pairs from specification strings. Args: - tools: An iterable of strings that contain tools and their versions. + specifications: An iterable of strings that contain tools and their versions. """ result = [] - for spec in tools: + for spec in specifications: try: tool, version = cls._split_pattern.split(spec, maxsplit=1) except ValueError: From 4c45f6a1471685a70964014defd6a651fc05f33c Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 15:58:01 +0200 Subject: [PATCH 09/37] fix: include image build number --- nf_core/__main__.py | 17 ++++++++++++----- nf_core/modules/mulled.py | 5 +++-- tests/test_mullled.py | 8 ++++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 49389fd9e6..6d78b35845 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -677,7 +677,15 @@ def bump_versions(ctx, tool, dir, all, show_all): # nf-core modules mulled @modules.command() @click.argument("specifications", required=True, nargs=-1, metavar=" <...>") -def mulled(specifications): +@click.option( + "--build-number", + type=int, + default=0, + show_default=True, + metavar="", + help="The build number for this image. This is an incremental value that starts at zero.", +) +def mulled(specifications, build_number): """ Generate the name of a BioContainers mulled image version 2. @@ -685,11 +693,10 @@ def mulled(specifications): that image, this command can generate it for you. """ - print( - nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( - nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications) - ) + image_name = nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( + nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications), build_number=build_number ) + print(image_name) # nf-core schema subcommands diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py index 197913c99e..fe941ee941 100644 --- a/nf_core/modules/mulled.py +++ b/nf_core/modules/mulled.py @@ -47,12 +47,13 @@ def parse_targets(cls, specifications: Iterable[str]) -> List[Tuple[str, str]]: return result @classmethod - def generate_image_name(cls, targets: Iterable[Tuple[str, str]]) -> str: + def generate_image_name(cls, targets: Iterable[Tuple[str, str]], build_number: int = 0) -> str: """ Generate the name of a BioContainers mulled image version 2. Args: targets: One or more tool, version pairs of the multi-tool container image. + build_number: The build number for this image. This is an incremental value that starts at zero. """ - return v2_image_name([build_target(name, version) for name, version in targets]) + return v2_image_name([build_target(name, version) for name, version in targets], image_build=str(build_number)) diff --git a/tests/test_mullled.py b/tests/test_mullled.py index 5eaa41fc81..cf0a4fcfc0 100644 --- a/tests/test_mullled.py +++ b/tests/test_mullled.py @@ -48,11 +48,15 @@ def test_noncompliant_version(specs): [ ( [("chromap", "0.2.1"), ("samtools", "1.15")], - "mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:bd74d08a359024829a7aec1638a28607bbcd8a58", + "mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:bd74d08a359024829a7aec1638a28607bbcd8a58-0", ), ( [("pysam", "0.16.0.1"), ("biopython", "1.78")], - "mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f", + "mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0", + ), + ( + [("samclip", "0.4.0"), ("samtools", "1.15")], + "mulled-v2-d057255d4027721f3ab57f6a599a2ae81cb3cbe3:13051b049b6ae536d76031ba94a0b8e78e364815-0", ), ], ) From 1ff8ade036d8101664f65453c91e8c5275b03e77 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 16:30:44 +0200 Subject: [PATCH 10/37] refactor: check if the generated image name exists --- nf_core/__main__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 6d78b35845..ce3035845e 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1,10 +1,10 @@ #!/usr/bin/env python """ nf-core: Helper tools for use with nf-core Nextflow pipelines. """ - from rich import print import logging import os import re +import requests import rich.console import rich.logging import rich.traceback @@ -697,6 +697,13 @@ def mulled(specifications, build_number): nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications), build_number=build_number ) print(image_name) + response = requests.get(f"https://quay.io/biocontainers/{image_name}/", allow_redirects=True) + if response.status_code != 200: + log.error( + "The generated multi-tool container image does not seem to exist yet. Are you sure that you provided the " + "right combination of tools and versions?" + ) + sys.exit(1) # nf-core schema subcommands From 1882e87656be142aa0a02c098cb6655b5d819400 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 16:32:08 +0200 Subject: [PATCH 11/37] chore: undo accidental commit --- .editorconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 0ffa5e3ec1..c29b6c7eee 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,7 +7,6 @@ insert_final_newline = true trim_trailing_whitespace = true indent_size = 4 indent_style = space -max_line_length = 120 [*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 From 893fc9a187a50d4199183c243e76e0bb0b04e1f6 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Thu, 21 Apr 2022 20:31:41 +0200 Subject: [PATCH 12/37] refactor: move check to class, expand error message --- nf_core/__main__.py | 17 ++++++++++------- nf_core/modules/mulled.py | 8 ++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ce3035845e..ae8b995c99 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -4,7 +4,6 @@ import logging import os import re -import requests import rich.console import rich.logging import rich.traceback @@ -693,15 +692,19 @@ def mulled(specifications, build_number): that image, this command can generate it for you. """ - image_name = nf_core.modules.mulled.MulledImageNameGenerator.generate_image_name( - nf_core.modules.mulled.MulledImageNameGenerator.parse_targets(specifications), build_number=build_number + from nf_core.modules.mulled import MulledImageNameGenerator + + image_name = MulledImageNameGenerator.generate_image_name( + MulledImageNameGenerator.parse_targets(specifications), build_number=build_number ) print(image_name) - response = requests.get(f"https://quay.io/biocontainers/{image_name}/", allow_redirects=True) - if response.status_code != 200: + if not MulledImageNameGenerator.image_exists(image_name): log.error( - "The generated multi-tool container image does not seem to exist yet. Are you sure that you provided the " - "right combination of tools and versions?" + "The generated multi-tool container image name does not seem to exist yet. Please double check that your " + "provided combination of tools and versions exists in the file:\n" + "https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv\n" + "If it does not, please add your desired combination as detailed at:\n" + "https://github.com/BioContainers/multi-package-containers\n" ) sys.exit(1) diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py index fe941ee941..c47f35a559 100644 --- a/nf_core/modules/mulled.py +++ b/nf_core/modules/mulled.py @@ -6,6 +6,7 @@ from packaging.version import Version, InvalidVersion from typing import Iterable, Tuple, List +import requests from galaxy.tool_util.deps.mulled.util import build_target, v2_image_name @@ -57,3 +58,10 @@ def generate_image_name(cls, targets: Iterable[Tuple[str, str]], build_number: i """ return v2_image_name([build_target(name, version) for name, version in targets], image_build=str(build_number)) + + @classmethod + def image_exists(cls, image_name: str) -> bool: + """Check whether a given BioContainers image name exists via a call to the quay.io API.""" + response = requests.get(f"https://quay.io/biocontainers/{image_name}/", allow_redirects=True) + log.debug(response.text) + return response.status_code == 200 From 54d218f2c418a25b892e9c16fcb111aacc8457b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 25 Apr 2022 11:40:08 +0200 Subject: [PATCH 13/37] add new command modules test --- nf_core/__main__.py | 21 ++++++++- nf_core/modules/__init__.py | 1 + nf_core/modules/module_test.py | 81 ++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 nf_core/modules/module_test.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 8affd23bfe..f9118d1322 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -49,7 +49,7 @@ }, { "name": "Developing new modules", - "commands": ["create", "create-test-yml", "lint", "bump-versions"], + "commands": ["create", "create-test-yml", "lint", "bump-versions", "test"], }, ], } @@ -674,6 +674,25 @@ def bump_versions(ctx, tool, dir, all, show_all): sys.exit(1) +# nf-core modules test +@modules.command("test") +@click.pass_context +@click.argument("tool", type=str, required=False, metavar=" or ") +@click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") +def test_module(ctx, tool, no_prompts): + """ + Run module tests locally. + + Given the name of a module, runs the Nextflow test command. + """ + try: + meta_builder = nf_core.modules.ModulesTest(tool, no_prompts) + meta_builder.run() + except UserWarning as e: + log.critical(e) + sys.exit(1) + + # nf-core schema subcommands @nf_core_cli.group() def schema(): diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index f833d52564..c85485c4f1 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -9,3 +9,4 @@ from .update import ModuleUpdate from .remove import ModuleRemove from .info import ModuleInfo +from .module_test import ModulesTest diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py new file mode 100644 index 0000000000..4d9bf95014 --- /dev/null +++ b/nf_core/modules/module_test.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +""" +The ModulesTest class runs the tests locally +""" + +import logging +import questionary +import os + +"""from __future__ import print_function +from rich.syntax import Syntax + +import errno +import gzip +import hashlib +import operator + + +import re +import rich +import shlex +import subprocess +import tempfile +import yaml + +""" +import nf_core.utils +from .modules_repo import ModulesRepo +from .test_yml_builder import ModulesTestYmlBuilder + +log = logging.getLogger(__name__) + + +class ModulesTest(ModulesTestYmlBuilder): + def __init__( + self, + module_name=None, + no_prompts=False, + ): + self.run_tests = True + self.module_name = module_name + self.no_prompts = no_prompts + self.module_dir = None + self.module_test_main = None + self.entry_points = [] + self.tests = [] + self.errors = [] + + def run(self): + """Run test steps""" + if not self.no_prompts: + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" + ) + self.check_inputs_test() + self.scrape_workflow_entry_points() + self.build_all_tests() + if len(self.errors) > 0: + errors = "\n - ".join(self.errors) + raise UserWarning(f"Ran, but found errors:\n - {errors}") + + def check_inputs_test(self): + """Do more complex checks about supplied flags.""" + + # Get the tool name if not specified + if self.module_name is None: + modules_repo = ModulesRepo() + modules_repo.get_modules_file_tree() + self.module_name = questionary.autocomplete( + "Tool name:", + choices=modules_repo.modules_avail_module_names, + style=nf_core.utils.nfcore_question_style, + ).ask() + self.module_dir = os.path.join("modules", *self.module_name.split("/")) + self.module_test_main = os.path.join("tests", "modules", *self.module_name.split("/"), "main.nf") + + # First, sanity check that the module directory exists + if not os.path.isdir(self.module_dir): + raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") + if not os.path.exists(self.module_test_main): + raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") From 5097f04ac95616cc1df7135d9cf14b30804f52db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 10:18:15 +0200 Subject: [PATCH 14/37] run tests with pytest --- nf_core/__main__.py | 5 ++- nf_core/modules/module_test.py | 80 ++++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index f9118d1322..3863ea49c8 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -679,14 +679,15 @@ def bump_versions(ctx, tool, dir, all, show_all): @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") -def test_module(ctx, tool, no_prompts): +@click.option("-a", "--pytest_args", type=str, required=False, multiple=True, help="Additional pytest arguments") +def test_module(ctx, tool, no_prompts, pytest_args): """ Run module tests locally. Given the name of a module, runs the Nextflow test command. """ try: - meta_builder = nf_core.modules.ModulesTest(tool, no_prompts) + meta_builder = nf_core.modules.ModulesTest(tool, no_prompts, pytest_args) meta_builder.run() except UserWarning as e: log.critical(e) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 4d9bf95014..8316d9502d 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -6,45 +6,25 @@ import logging import questionary import os - -"""from __future__ import print_function -from rich.syntax import Syntax - -import errno -import gzip -import hashlib -import operator - - -import re +import pytest +import sys import rich -import shlex -import subprocess -import tempfile -import yaml -""" import nf_core.utils from .modules_repo import ModulesRepo -from .test_yml_builder import ModulesTestYmlBuilder log = logging.getLogger(__name__) - -class ModulesTest(ModulesTestYmlBuilder): +class ModulesTest(object): def __init__( self, module_name=None, no_prompts=False, + pytest_args="", ): - self.run_tests = True self.module_name = module_name self.no_prompts = no_prompts - self.module_dir = None - self.module_test_main = None - self.entry_points = [] - self.tests = [] - self.errors = [] + self.pytest_args = pytest_args def run(self): """Run test steps""" @@ -52,14 +32,11 @@ def run(self): log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" ) - self.check_inputs_test() - self.scrape_workflow_entry_points() - self.build_all_tests() - if len(self.errors) > 0: - errors = "\n - ".join(self.errors) - raise UserWarning(f"Ran, but found errors:\n - {errors}") + self.check_inputs() + self.set_profile() + self.run_pytests() - def check_inputs_test(self): + def check_inputs(self): """Do more complex checks about supplied flags.""" # Get the tool name if not specified @@ -79,3 +56,42 @@ def check_inputs_test(self): raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") if not os.path.exists(self.module_test_main): raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") + + def set_profile(self): + """Set $PROFILE env variable. + The config expects $PROFILE and Nextflow fails if it's not set. + """ + if os.environ.get("PROFILE") is None: + os.environ["PROFILE"] = "" + if self.no_prompts: + log.info( + "Setting env var '$PROFILE' to an empty string as not set.\n" + "Tests will run with Docker by default. " + "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." + ) + else: + question = { + "type": "list", + "name": "profile", + "message": "Choose software profile", + "choices": ["Docker", "Singularity", "Conda"], + } + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) + profile = answer["profile"].lower() + if profile in ["singularity", "conda"]: + os.environ["PROFILE"] = profile + log.info(f"Setting env var '$PROFILE' to '{profile}'") + + def run_pytests(self): + """Given a module name, run tests.""" + # Print nice divider line + console = rich.console.Console() + console.print("[black]" + "─" * console.width) + + # Set pytest arguments + command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd"] + command_args += self.pytest_args + + # Run pytest + log.info(f"Running pytest for module '{self.module_name}'") + sys.exit(pytest.main(command_args)) From 32274990c5c14d226617d60d1981969d56d8f913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 10:29:36 +0200 Subject: [PATCH 15/37] remove test workflow check --- nf_core/modules/module_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 8316d9502d..ee5e39c4f0 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -25,6 +25,7 @@ def __init__( self.module_name = module_name self.no_prompts = no_prompts self.pytest_args = pytest_args + self.module_dir = None def run(self): """Run test steps""" @@ -49,13 +50,10 @@ def check_inputs(self): style=nf_core.utils.nfcore_question_style, ).ask() self.module_dir = os.path.join("modules", *self.module_name.split("/")) - self.module_test_main = os.path.join("tests", "modules", *self.module_name.split("/"), "main.nf") # First, sanity check that the module directory exists if not os.path.isdir(self.module_dir): raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") - if not os.path.exists(self.module_test_main): - raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") def set_profile(self): """Set $PROFILE env variable. From 2e2ea58479e3725b37877e5cbb7ae1e075fb8700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 13:10:10 +0200 Subject: [PATCH 16/37] add code tests --- tests/modules/module_test.py | 15 +++++++++++++++ tests/test_modules.py | 4 ++++ 2 files changed, 19 insertions(+) create mode 100644 tests/modules/module_test.py diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py new file mode 100644 index 0000000000..12a4c85c4d --- /dev/null +++ b/tests/modules/module_test.py @@ -0,0 +1,15 @@ +"""Test the 'modules test' command which runs module pytests.""" +import os +import pytest + +import nf_core.modules + +def test_modules_test_check_inputs(self): + """Test the check_inputs() function - raise UserWarning because module doesn't exist""" + cwd = os.getcwd() + os.chdir(self.nfcore_modules) + meta_builder = nf_core.modules.ModulesTest("none", True, "") + with pytest.raises(UserWarning) as excinfo: + meta_builder.check_inputs() + os.chdir(cwd) + assert "Cannot find directory" in str(excinfo.value) diff --git a/tests/test_modules.py b/tests/test_modules.py index cfa3408e69..00cc89c36d 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -113,3 +113,7 @@ def test_modulesrepo_class(self): test_modules_bump_versions_fail, test_modules_bump_versions_fail_unknown_version, ) + + from .modules.module_test import ( + test_modules_test_check_inputs, + ) From c0b29f5944f86fcf598ce4a6fbf444974b7a53be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 13:29:42 +0200 Subject: [PATCH 17/37] fix linting --- nf_core/modules/module_test.py | 1 + tests/modules/module_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index ee5e39c4f0..5338aeb81e 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -15,6 +15,7 @@ log = logging.getLogger(__name__) + class ModulesTest(object): def __init__( self, diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py index 12a4c85c4d..ac0c36c997 100644 --- a/tests/modules/module_test.py +++ b/tests/modules/module_test.py @@ -4,6 +4,7 @@ import nf_core.modules + def test_modules_test_check_inputs(self): """Test the check_inputs() function - raise UserWarning because module doesn't exist""" cwd = os.getcwd() From bf38c4d03aaf14ea6c72f0f852d268f04ca19e1c Mon Sep 17 00:00:00 2001 From: Nathan Spix <56930974+njspix@users.noreply.github.com> Date: Tue, 26 Apr 2022 09:29:15 -0400 Subject: [PATCH 18/37] Update actions_ci.py In cases where e.g. multiple aligners are specified in the GitHub Actions matrix, trying to pull a 'NXF_VER' key out of the include dictionary can cause line 140 to fail when it shouldn't. The get() method will not error if the key is not present, fixing this problem. --- nf_core/lint/actions_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint/actions_ci.py b/nf_core/lint/actions_ci.py index 89d9becd04..1a02680ece 100644 --- a/nf_core/lint/actions_ci.py +++ b/nf_core/lint/actions_ci.py @@ -137,7 +137,7 @@ def actions_ci(self): # Check that we are testing the minimum nextflow version try: matrix = ciwf["jobs"]["test"]["strategy"]["matrix"]["include"] - assert any([i["NXF_VER"] == self.minNextflowVersion for i in matrix]) + assert any([i.get("NXF_VER") == self.minNextflowVersion for i in matrix]) except (KeyError, TypeError): failed.append("'.github/workflows/ci.yml' does not check minimum NF version") except AssertionError: From 99b5170a5801ab648442157d4a31c4bdcfa0cff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 15:33:13 +0200 Subject: [PATCH 19/37] apply comments from PR review --- nf_core/modules/module_test.py | 39 ++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 5338aeb81e..c6a81f8672 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -17,6 +17,31 @@ class ModulesTest(object): + """ + Class to run module pytests. + + ... + + Attributes + ---------- + module_name : str + name of the tool to run tests for + no_prompts : bool + flat indicating if prompts are used + pytest_args : tuple + additional arguments passed to pytest command + + Methods + ------- + run(): + Run test steps + __check_inputs(): + Check inputs. Ask for module_name if not provided and check that the directory exists + __set_profile(): + Set software profile + __run_pytests(self): + Run pytest + """ def __init__( self, module_name=None, @@ -34,15 +59,17 @@ def run(self): log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" ) - self.check_inputs() - self.set_profile() - self.run_pytests() + self.__check_inputs() + self.__set_profile() + self.__run_pytests() - def check_inputs(self): + def __check_inputs(self): """Do more complex checks about supplied flags.""" # Get the tool name if not specified if self.module_name is None: + if self.no_prompts: + raise UserWarning(f"Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL.") modules_repo = ModulesRepo() modules_repo.get_modules_file_tree() self.module_name = questionary.autocomplete( @@ -56,7 +83,7 @@ def check_inputs(self): if not os.path.isdir(self.module_dir): raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") - def set_profile(self): + def __set_profile(self): """Set $PROFILE env variable. The config expects $PROFILE and Nextflow fails if it's not set. """ @@ -81,7 +108,7 @@ def set_profile(self): os.environ["PROFILE"] = profile log.info(f"Setting env var '$PROFILE' to '{profile}'") - def run_pytests(self): + def __run_pytests(self): """Given a module name, run tests.""" # Print nice divider line console = rich.console.Console() From b80fd622c1de18d583ff302797fdab5bbcb28b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Apr 2022 15:50:51 +0200 Subject: [PATCH 20/37] fix linting --- nf_core/__main__.py | 2 +- nf_core/modules/module_test.py | 53 ++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 51fb5aebac..45fcff9a60 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -708,7 +708,7 @@ def mulled(specifications, build_number): ) sys.exit(1) - + # nf-core modules test @modules.command("test") @click.pass_context diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index c6a81f8672..3d2aa8d58d 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -18,30 +18,31 @@ class ModulesTest(object): """ - Class to run module pytests. - - ... - - Attributes - ---------- - module_name : str - name of the tool to run tests for - no_prompts : bool - flat indicating if prompts are used - pytest_args : tuple - additional arguments passed to pytest command - - Methods - ------- - run(): - Run test steps - __check_inputs(): - Check inputs. Ask for module_name if not provided and check that the directory exists - __set_profile(): - Set software profile - __run_pytests(self): - Run pytest - """ + Class to run module pytests. + + ... + + Attributes + ---------- + module_name : str + name of the tool to run tests for + no_prompts : bool + flat indicating if prompts are used + pytest_args : tuple + additional arguments passed to pytest command + + Methods + ------- + run(): + Run test steps + __check_inputs(): + Check inputs. Ask for module_name if not provided and check that the directory exists + __set_profile(): + Set software profile + __run_pytests(self): + Run pytest + """ + def __init__( self, module_name=None, @@ -69,7 +70,9 @@ def __check_inputs(self): # Get the tool name if not specified if self.module_name is None: if self.no_prompts: - raise UserWarning(f"Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL.") + raise UserWarning( + f"Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." + ) modules_repo = ModulesRepo() modules_repo.get_modules_file_tree() self.module_name = questionary.autocomplete( From 6c006fdf2035e0aa5f0ef9a798ced52ad7257a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 11:25:48 +0200 Subject: [PATCH 21/37] apply comments from PR --- nf_core/modules/module_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 3d2aa8d58d..bb39c27073 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -35,11 +35,11 @@ class ModulesTest(object): ------- run(): Run test steps - __check_inputs(): + _check_inputs(): Check inputs. Ask for module_name if not provided and check that the directory exists - __set_profile(): + _set_profile(): Set software profile - __run_pytests(self): + _run_pytests(self): Run pytest """ @@ -60,11 +60,11 @@ def run(self): log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" ) - self.__check_inputs() - self.__set_profile() - self.__run_pytests() + self._check_inputs() + self._set_profile() + self._run_pytests() - def __check_inputs(self): + def _check_inputs(self): """Do more complex checks about supplied flags.""" # Get the tool name if not specified @@ -86,7 +86,7 @@ def __check_inputs(self): if not os.path.isdir(self.module_dir): raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") - def __set_profile(self): + def _set_profile(self): """Set $PROFILE env variable. The config expects $PROFILE and Nextflow fails if it's not set. """ @@ -111,7 +111,7 @@ def __set_profile(self): os.environ["PROFILE"] = profile log.info(f"Setting env var '$PROFILE' to '{profile}'") - def __run_pytests(self): + def _run_pytests(self): """Given a module name, run tests.""" # Print nice divider line console = rich.console.Console() From 0e4a835fb68ad6e698a161bcd347619bbd3c489e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 12:48:44 +0200 Subject: [PATCH 22/37] fix test --- tests/modules/module_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py index ac0c36c997..fbad1c9fc5 100644 --- a/tests/modules/module_test.py +++ b/tests/modules/module_test.py @@ -11,6 +11,6 @@ def test_modules_test_check_inputs(self): os.chdir(self.nfcore_modules) meta_builder = nf_core.modules.ModulesTest("none", True, "") with pytest.raises(UserWarning) as excinfo: - meta_builder.check_inputs() + meta_builder._check_inputs() os.chdir(cwd) assert "Cannot find directory" in str(excinfo.value) From a8088532f15a2addaed0a2444a0d1312efa84865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 15:26:32 +0200 Subject: [PATCH 23/37] always set profile --- nf_core/modules/module_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index bb39c27073..7f33774776 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -107,9 +107,8 @@ def _set_profile(self): } answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) profile = answer["profile"].lower() - if profile in ["singularity", "conda"]: - os.environ["PROFILE"] = profile - log.info(f"Setting env var '$PROFILE' to '{profile}'") + os.environ["PROFILE"] = profile + log.info(f"Setting env var '$PROFILE' to '{profile}'") def _run_pytests(self): """Given a module name, run tests.""" From 4165a9fecb950488d982af3417a0a1ae44cb94dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 15:29:44 +0200 Subject: [PATCH 24/37] do not abbreviate env var --- nf_core/modules/module_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 7f33774776..db204ba97a 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -94,7 +94,7 @@ def _set_profile(self): os.environ["PROFILE"] = "" if self.no_prompts: log.info( - "Setting env var '$PROFILE' to an empty string as not set.\n" + "Setting environment variable '$PROFILE' to an empty string as not set.\n" "Tests will run with Docker by default. " "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." ) @@ -108,7 +108,7 @@ def _set_profile(self): answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) profile = answer["profile"].lower() os.environ["PROFILE"] = profile - log.info(f"Setting env var '$PROFILE' to '{profile}'") + log.info(f"Setting environment variable '$PROFILE' to '{profile}'") def _run_pytests(self): """Given a module name, run tests.""" From b431f9832b090ad4920349550286ff5190617c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 16:06:53 +0200 Subject: [PATCH 25/37] add new test --- nf_core/modules/module_test.py | 2 +- tests/modules/module_test.py | 11 +++++++++++ tests/test_modules.py | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index db204ba97a..25e4991a4b 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -71,7 +71,7 @@ def _check_inputs(self): if self.module_name is None: if self.no_prompts: raise UserWarning( - f"Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." + "Tool name not provided and prompts deactivated. Please provide the tool name as TOOL/SUBTOOL or TOOL." ) modules_repo = ModulesRepo() modules_repo.get_modules_file_tree() diff --git a/tests/modules/module_test.py b/tests/modules/module_test.py index fbad1c9fc5..a4559ffde5 100644 --- a/tests/modules/module_test.py +++ b/tests/modules/module_test.py @@ -14,3 +14,14 @@ def test_modules_test_check_inputs(self): meta_builder._check_inputs() os.chdir(cwd) assert "Cannot find directory" in str(excinfo.value) + + +def test_modules_test_no_name_no_prompts(self): + """Test the check_inputs() function - raise UserWarning prompts are deactivated and module name is not provided.""" + cwd = os.getcwd() + os.chdir(self.nfcore_modules) + meta_builder = nf_core.modules.ModulesTest(None, True, "") + with pytest.raises(UserWarning) as excinfo: + meta_builder._check_inputs() + os.chdir(cwd) + assert "Tool name not provided and prompts deactivated." in str(excinfo.value) diff --git a/tests/test_modules.py b/tests/test_modules.py index 00cc89c36d..b12333b51d 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -116,4 +116,5 @@ def test_modulesrepo_class(self): from .modules.module_test import ( test_modules_test_check_inputs, + test_modules_test_no_name_no_prompts, ) From 58df6393e4be08cd3de8465a456206aec7d18767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 27 Apr 2022 16:53:46 +0200 Subject: [PATCH 26/37] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a8206b91..306425fd2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Escaped test run output before logging it, to avoid a rich ` MarkupError` - Add a new command `nf-core modules mulled` which can generate the name for a multi-tool container image. +- Add a new command `nf-core modules test` which runs pytests locally. ## [v2.3.2 - Mercury Vulture Fixed Formatting](https://github.com/nf-core/tools/releases/tag/2.3.2) - [2022-03-24] From 530a3839deee338d82a2ecfb3341ed0ca989e3af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 28 Apr 2022 09:57:41 +0200 Subject: [PATCH 27/37] specify test files as test_*.py in pytest.ini --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 57209fee41..652bdf8e53 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,3 +3,4 @@ filterwarnings = ignore::pytest.PytestRemovedIn8Warning:_pytest.nodes:140 testpaths = tests +python_files = test_*.py From b885e5f19184ba5f9e42deae75a01c08a9983b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 28 Apr 2022 14:31:30 +0200 Subject: [PATCH 28/37] apply comments from PR review --- nf_core/modules/module_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 25e4991a4b..57ca2c6932 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -9,6 +9,7 @@ import pytest import sys import rich +from pathlib import Path import nf_core.utils from .modules_repo import ModulesRepo @@ -52,7 +53,6 @@ def __init__( self.module_name = module_name self.no_prompts = no_prompts self.pytest_args = pytest_args - self.module_dir = None def run(self): """Run test steps""" @@ -80,11 +80,11 @@ def _check_inputs(self): choices=modules_repo.modules_avail_module_names, style=nf_core.utils.nfcore_question_style, ).ask() - self.module_dir = os.path.join("modules", *self.module_name.split("/")) + module_dir = Path("modules") / self.module_name # First, sanity check that the module directory exists - if not os.path.isdir(self.module_dir): - raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") + if not module_dir.is_dir(): + raise UserWarning(f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL") def _set_profile(self): """Set $PROFILE env variable. @@ -117,7 +117,7 @@ def _run_pytests(self): console.print("[black]" + "─" * console.width) # Set pytest arguments - command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd"] + command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] command_args += self.pytest_args # Run pytest From f49af39a24209fd08f2b62d6feb91ff503f01f12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 29 Apr 2022 14:12:52 +0200 Subject: [PATCH 29/37] check if profile is available --- nf_core/modules/module_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 57ca2c6932..8ca1027419 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -9,6 +9,8 @@ import pytest import sys import rich +import subprocess +import shlex from pathlib import Path import nf_core.utils @@ -62,6 +64,7 @@ def run(self): ) self._check_inputs() self._set_profile() + self._check_profile() self._run_pytests() def _check_inputs(self): @@ -110,6 +113,14 @@ def _set_profile(self): os.environ["PROFILE"] = profile log.info(f"Setting environment variable '$PROFILE' to '{profile}'") + def _check_profile(self): + """Check if profile is available""" + profile = os.environ.get("PROFILE") + try: + profile_check = subprocess.check_output(shlex.split(f"{profile} --help"), stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + raise UserWarning(f"Error with profile {profile} (exit code {e.returncode})\n[red]{e.output.decode()}") + def _run_pytests(self): """Given a module name, run tests.""" # Print nice divider line From f33f1fb756c1239dfaf6da1c1b37e2c32603de00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 29 Apr 2022 14:41:02 +0200 Subject: [PATCH 30/37] modify linting --- nf_core/modules/module_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 8ca1027419..c2510d3ee9 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -117,7 +117,9 @@ def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") try: - profile_check = subprocess.check_output(shlex.split(f"{profile} --help"), stderr=subprocess.STDOUT, shell=True) + profile_check = subprocess.check_output( + shlex.split(f"{profile} --help"), stderr=subprocess.STDOUT, shell=True + ) except subprocess.CalledProcessError as e: raise UserWarning(f"Error with profile {profile} (exit code {e.returncode})\n[red]{e.output.decode()}") From b732386f60fd8f20c79766163a3bbd137622edab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 12:42:59 +0200 Subject: [PATCH 31/37] Add nf-core/modules folder warning Co-authored-by: Gisela Gabernet --- nf_core/modules/module_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index c2510d3ee9..6507c58219 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -87,7 +87,7 @@ def _check_inputs(self): # First, sanity check that the module directory exists if not module_dir.is_dir(): - raise UserWarning(f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL") + raise UserWarning(f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?") def _set_profile(self): """Set $PROFILE env variable. From c182ef3a39f1865776707ac861c5f21475c6c715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 12:44:22 +0200 Subject: [PATCH 32/37] Add valid profile check Co-authored-by: Fabian Egli --- nf_core/modules/module_test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 6507c58219..65006a6ced 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -117,9 +117,16 @@ def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") try: - profile_check = subprocess.check_output( - shlex.split(f"{profile} --help"), stderr=subprocess.STDOUT, shell=True - ) + + # Make sure the profile read from the environment is a valid Nextflow profile. + valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] + if profile in valid_nextflow_profiles: + profile_check = subprocess.check_output([profile, "--help"], stderr=subprocess.STDOUT) + else: + raise UserWarning( + f"The profile '{profile}' set in the shell environment is not a valid.\n" + f"Valid Nextflow profiles are {valid_nextflow_profiles}." + ) except subprocess.CalledProcessError as e: raise UserWarning(f"Error with profile {profile} (exit code {e.returncode})\n[red]{e.output.decode()}") From 69247e7b72b8682bc6ca030b5393ee19b0ef7140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 13:05:39 +0200 Subject: [PATCH 33/37] fix linting --- nf_core/modules/module_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 65006a6ced..781fb3b2f1 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -87,7 +87,9 @@ def _check_inputs(self): # First, sanity check that the module directory exists if not module_dir.is_dir(): - raise UserWarning(f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?") + raise UserWarning( + f"Cannot find directory '{module_dir}'. Should be TOOL/SUBTOOL or TOOL. Are you running the tests inside the nf-core/modules main directory?" + ) def _set_profile(self): """Set $PROFILE env variable. @@ -117,7 +119,6 @@ def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") try: - # Make sure the profile read from the environment is a valid Nextflow profile. valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] if profile in valid_nextflow_profiles: From 1d8e56788e8d791d22dca847a72270084980cab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 13:14:23 +0200 Subject: [PATCH 34/37] avoid using shell=True when checking profile --- nf_core/modules/module_test.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 781fb3b2f1..75c7cbb333 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -12,6 +12,7 @@ import subprocess import shlex from pathlib import Path +from shutil import which import nf_core.utils from .modules_repo import ModulesRepo @@ -118,18 +119,16 @@ def _set_profile(self): def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") - try: - # Make sure the profile read from the environment is a valid Nextflow profile. - valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] - if profile in valid_nextflow_profiles: - profile_check = subprocess.check_output([profile, "--help"], stderr=subprocess.STDOUT) - else: - raise UserWarning( - f"The profile '{profile}' set in the shell environment is not a valid.\n" - f"Valid Nextflow profiles are {valid_nextflow_profiles}." - ) - except subprocess.CalledProcessError as e: - raise UserWarning(f"Error with profile {profile} (exit code {e.returncode})\n[red]{e.output.decode()}") + # Make sure the profile read from the environment is a valid Nextflow profile. + valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] + if profile in valid_nextflow_profiles: + if not which(profile): + raise UserWarning(f"The PROFILE '{profile}' set in the shell environment is not available.") + else: + raise UserWarning( + f"The PROFILE '{profile}' set in the shell environment is not valid.\n" + f"Valid Nextflow profiles are {valid_nextflow_profiles}." + ) def _run_pytests(self): """Given a module name, run tests.""" From 8e33499cfbf4ca88c24bdf1df67ad08a8777d968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 13:54:45 +0200 Subject: [PATCH 35/37] Apply suggestions from code review Co-authored-by: Phil Ewels --- nf_core/modules/module_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 75c7cbb333..8ca12df6d5 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -120,21 +120,21 @@ def _check_profile(self): """Check if profile is available""" profile = os.environ.get("PROFILE") # Make sure the profile read from the environment is a valid Nextflow profile. - valid_nextflow_profiles = ["docker", "singularity", "podman", "shifter", "charliecloud", "conda"] + valid_nextflow_profiles = ["docker", "singularity", "conda"] if profile in valid_nextflow_profiles: if not which(profile): - raise UserWarning(f"The PROFILE '{profile}' set in the shell environment is not available.") + raise UserWarning(f"Command '{profile}' not found - is it installed?") else: raise UserWarning( f"The PROFILE '{profile}' set in the shell environment is not valid.\n" - f"Valid Nextflow profiles are {valid_nextflow_profiles}." + f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'." ) def _run_pytests(self): """Given a module name, run tests.""" # Print nice divider line console = rich.console.Console() - console.print("[black]" + "─" * console.width) + console.rule(self.module_name, style="black") # Set pytest arguments command_args = ["--tag", f"{self.module_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] From aca376dcf7fc30e1e83c540e5fc04cbee1b92bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 2 May 2022 13:59:54 +0200 Subject: [PATCH 36/37] remove unnecessary libraryes --- nf_core/modules/module_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/modules/module_test.py b/nf_core/modules/module_test.py index 8ca12df6d5..35658dd26b 100644 --- a/nf_core/modules/module_test.py +++ b/nf_core/modules/module_test.py @@ -9,8 +9,6 @@ import pytest import sys import rich -import subprocess -import shlex from pathlib import Path from shutil import which From 8ade0dfe9be6dfddc5555a54479b31688f3b0c4a Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Wed, 4 May 2022 15:40:42 +0200 Subject: [PATCH 37/37] replace a weird with a normal dash --- nf_core/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index ede47d0f22..b0c3e565f7 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -420,7 +420,7 @@ def prompt_param(self, param_id, param_obj, is_required, answers): # If required and got an empty reponse, ask again while type(answer[param_id]) is str and answer[param_id].strip() == "" and is_required: - log.error("'–-{}' is required".format(param_id)) + log.error("'--{}' is required".format(param_id)) answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) # Ignore if empty