From 9dcb778f6dec918765846a238d1293296a6b4a63 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 1 Nov 2022 10:02:06 -0500 Subject: [PATCH 001/246] feat(pipeline-template): Add CHM13 to igenomes --- CHANGELOG.md | 1 + nf_core/pipeline-template/conf/igenomes.config | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e51660e52c..81032c7cfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Fix lint warnings for `samplesheet_check.nf` module - Check that the workflow name provided with a template doesn't contain dashes ([#1822](https://github.com/nf-core/tools/pull/1822)) +- Add initial CHM13 support ([1988](https://github.com/nf-core/tools/issues/1988)) ### Linting diff --git a/nf_core/pipeline-template/conf/igenomes.config b/nf_core/pipeline-template/conf/igenomes.config index 7a1b3ac6d3..3f11437759 100644 --- a/nf_core/pipeline-template/conf/igenomes.config +++ b/nf_core/pipeline-template/conf/igenomes.config @@ -36,6 +36,14 @@ params { macs_gsize = "2.7e9" blacklist = "${projectDir}/assets/blacklists/hg38-blacklist.bed" } + 'CHM13' { + fasta = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/WholeGenomeFasta/genome.fa" + bwa = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/BWAIndex/" + bwamem2 = "${params.igenomes_base}/Homo_sapiens/UCSC/CHM13/Sequence/BWAmem2Index/" + gtf = "${params.igenomes_base}/Homo_sapiens/NCBI/CHM13/Annotation/Genes/genes.gtf" + gff = "ftp://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/009/914/755/GCF_009914755.1_T2T-CHM13v2.0/GCF_009914755.1_T2T-CHM13v2.0_genomic.gff.gz" + mito_name = "chrM" + } 'GRCm38' { fasta = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/WholeGenomeFasta/genome.fa" bwa = "${params.igenomes_base}/Mus_musculus/Ensembl/GRCm38/Sequence/BWAIndex/version0.6.0/" From a50d3520b5aaca09acf4be381f886697985210dc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 19 Dec 2022 13:11:30 +0100 Subject: [PATCH 002/246] bump to 2.8dev --- CHANGELOG.md | 12 ++++++++++++ setup.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 923a6c1ca4..d8c5d89f5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # nf-core/tools: Changelog +# v2.8dev + +### Template + +### Linting + +### Modules + +### Subworkflows + +### General + ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] ### Template diff --git a/setup.py b/setup.py index cc83e90eed..38f36810b7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.7.2" +version = "2.8dev" with open("README.md") as f: readme = f.read() From 6777abb17d17da6887af548c3d00f0c326f4ff20 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 19 Dec 2022 13:35:03 +0100 Subject: [PATCH 003/246] change order in release checklist --- .github/RELEASE_CHECKLIST.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index c5b3464b83..f2d2591f16 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -8,8 +8,8 @@ 6. Create a PR from `dev` to `master` 7. Make sure all CI tests are passing again (additional tests are run on PRs to `master`) 8. Request review (2 approvals required) -9. Merge the PR into `master` -10. Run `rich-codex` to regenerate docs screengrabs (actions `workflow_dispatch` button) +9. Run `rich-codex` to regenerate docs screengrabs (actions `workflow_dispatch` button) +10. Merge the PR into `master` 11. Wait for CI tests on the commit to passed 12. (Optional but a good idea) Run a manual sync on `nf-core/testpipeline` and check that CI is passing on the resulting PR. 13. Create a new release copying the `CHANGELOG` for that release into the description section. From 5d954788ebec7810bc0b77b173ecb2d58d403465 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 23 Dec 2022 10:30:45 +0100 Subject: [PATCH 004/246] Explicitly disable conda in container profiles --- nf_core/pipeline-template/nextflow.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 5cbb076b46..cc591a54e5 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -102,6 +102,7 @@ profiles { docker { docker.enabled = true docker.userEmulation = true + conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false @@ -113,6 +114,7 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true + conda.enabled = false docker.enabled = false podman.enabled = false shifter.enabled = false @@ -120,6 +122,7 @@ profiles { } podman { podman.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false shifter.enabled = false @@ -127,6 +130,7 @@ profiles { } shifter { shifter.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false @@ -134,6 +138,7 @@ profiles { } charliecloud { charliecloud.enabled = true + conda.enabled = false docker.enabled = false singularity.enabled = false podman.enabled = false From bb6ed953e593d4619aa25ceafba2074a464fe69d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 23 Dec 2022 10:32:17 +0100 Subject: [PATCH 005/246] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8c5d89f5d..62eef56c6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Fix the syntax of github_output in GitHub actions ([#2114](https://github.com/nf-core/tools/pull/2114)) - Fix a bug introduced in 2.7 that made pipelines hang ([#2132](https://github.com/nf-core/tools/issues/2132)) +- Explicitly disable `conda` when using container profiles ([#2140](https://github.com/nf-core/tools/pull/2140)) ### Linting From a09c06099aecc9d88c5f9abe399fe810cde238d4 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 23 Dec 2022 10:35:00 +0100 Subject: [PATCH 006/246] Apply suggestions from code review --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62eef56c6c..be6d5820d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - Fix the syntax of github_output in GitHub actions ([#2114](https://github.com/nf-core/tools/pull/2114)) - Fix a bug introduced in 2.7 that made pipelines hang ([#2132](https://github.com/nf-core/tools/issues/2132)) -- Explicitly disable `conda` when using container profiles ([#2140](https://github.com/nf-core/tools/pull/2140)) +- Explicitly disable `conda` when a container profile ([#2140](https://github.com/nf-core/tools/pull/2140)) ### Linting From 316f2b3aa4f692874e4e4b32422be2070fb786dd Mon Sep 17 00:00:00 2001 From: "Maxime U. Garcia" Date: Wed, 4 Jan 2023 13:04:03 +0100 Subject: [PATCH 007/246] Update .editorconfig I believe we removed the CITATIONS.cff file, right? --- nf_core/pipeline-template/.editorconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.editorconfig b/nf_core/pipeline-template/.editorconfig index b78de6e655..b6b3190776 100644 --- a/nf_core/pipeline-template/.editorconfig +++ b/nf_core/pipeline-template/.editorconfig @@ -8,7 +8,7 @@ trim_trailing_whitespace = true indent_size = 4 indent_style = space -[*.{md,yml,yaml,html,css,scss,js,cff}] +[*.{md,yml,yaml,html,css,scss,js}] indent_size = 2 # These files are edited and tested upstream in nf-core/modules From bc9c44258733677db133809fd118fe6df502ba9e Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Sat, 7 Jan 2023 21:57:30 +0000 Subject: [PATCH 008/246] Make sure command runs when an output file is passed --- nf_core/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index aaf0ea9868..15eadc0484 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1446,8 +1446,9 @@ def docs(schema_path, output, format, force, columns): # Assume we're in a pipeline dir root if schema path not set schema_obj.get_schema_path(schema_path) schema_obj.load_schema() + docs = schema_obj.print_documentation(output, format, force, columns.split(",")) if not output: - stdout.print(schema_obj.print_documentation(output, format, force, columns.split(","))) + stdout.print(docs) # nf-core bump-version From ea52c3fbf62c7b9b281ce32bc1323a77da7347b2 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Sat, 7 Jan 2023 21:58:16 +0000 Subject: [PATCH 009/246] Replace newlines from fields in markdown tables with HTML
as otherwise table formatting breaks --- nf_core/schema.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index a50a1ed789..d8046ecadd 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -495,9 +495,11 @@ def schema_to_markdown(self, columns): if column == "parameter": out += f"| `{p_key}` " elif column == "description": - out += f"| {param.get('description', '')} " + desc = param.get("description", "").replace("\n", "
") + out += f"| {desc} " if param.get("help_text", "") != "": - out += f"
Help{param['help_text']}
" + help_txt = param["help_text"].replace("\n", "
") + out += f"
Help{help_txt}
" elif column == "type": out += f"| `{param.get('type', '')}` " else: From e024fffb6d1bdaf6c72a274fef88feccfeb9390e Mon Sep 17 00:00:00 2001 From: Jose Alejandro Romero Herrera Date: Sun, 8 Jan 2023 21:05:36 +0100 Subject: [PATCH 010/246] Add singularity note for offline use --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 73d868bb32..ccb1d33378 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,8 @@ You can run the pipeline by simply providing the directory path for the `workflo nextflow run /path/to/download/nf-core-rnaseq-dev/workflow/ --input mydata.csv --outdir results # usual parameters here ``` +Note that you want to use the downloaded singularity images, you will still need to use `-profile singularity` or have it enabled in your config file. + ### Downloaded nf-core configs The pipeline files are automatically updated (`params.custom_config_base` is set to `../configs`), so that the local copy of institutional configs are available when running the pipeline. From b009559743de5732e0e82db416faad9c7e1e8d74 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sun, 8 Jan 2023 21:48:22 +0100 Subject: [PATCH 011/246] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ccb1d33378..60dc102996 100644 --- a/README.md +++ b/README.md @@ -344,7 +344,7 @@ You can run the pipeline by simply providing the directory path for the `workflo nextflow run /path/to/download/nf-core-rnaseq-dev/workflow/ --input mydata.csv --outdir results # usual parameters here ``` -Note that you want to use the downloaded singularity images, you will still need to use `-profile singularity` or have it enabled in your config file. +> Note that if you downloaded singularity images, you will need to use `-profile singularity` or have it enabled in your config file. ### Downloaded nf-core configs From a528fa8d96d9ba0ee80fb9ec5eb1e205b30c7246 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:32:09 +0000 Subject: [PATCH 012/246] Use technique from create_test_yml command to write prettified docs to file or console --- nf_core/__main__.py | 5 +---- nf_core/schema.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 15eadc0484..b96920b533 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1446,10 +1446,7 @@ def docs(schema_path, output, format, force, columns): # Assume we're in a pipeline dir root if schema path not set schema_obj.get_schema_path(schema_path) schema_obj.load_schema() - docs = schema_obj.print_documentation(output, format, force, columns.split(",")) - if not output: - stdout.print(docs) - + schema_obj.print_documentation(output, format, force, columns.split(",")) # nf-core bump-version @nf_core_cli.command("bump-version") diff --git a/nf_core/schema.py b/nf_core/schema.py index d8046ecadd..b325a6a075 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -6,17 +6,20 @@ import json import logging import os +import tempfile import webbrowser import jinja2 import jsonschema import markdown +import rich.console import yaml from rich.prompt import Confirm +from rich.syntax import Syntax import nf_core.list import nf_core.utils -from nf_core.lint_utils import dump_json_with_prettier +from nf_core.lint_utils import dump_json_with_prettier, run_prettier_on_file log = logging.getLogger(__name__) @@ -464,13 +467,21 @@ def print_documentation( if format == "html": output = self.markdown_to_html(output) - # Print to file - if output_fn: + with tempfile.NamedTemporaryFile(mode="w+") as fh: + fh.write(output) + run_prettier_on_file(fh.name) + fh.seek(0) + prettified_docs = fh.read() + + if not output_fn: + console = rich.console.Console() + console.print("\n", Syntax(prettified_docs, format), "\n") + else: if os.path.exists(output_fn) and not force: log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") return - with open(output_fn, "w") as file: - file.write(output) + with open(output_fn, "w") as fh: + fh.write(prettified_docs) log.info(f"Documentation written to '{output_fn}'") # Return as a string From c090d826024c12b15cbd858df7e91c450b0bdf4f Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 9 Jan 2023 10:35:05 +0000 Subject: [PATCH 013/246] Fix black formatting --- nf_core/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index b96920b533..555783d8d4 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1448,6 +1448,7 @@ def docs(schema_path, output, format, force, columns): schema_obj.load_schema() schema_obj.print_documentation(output, format, force, columns.split(",")) + # nf-core bump-version @nf_core_cli.command("bump-version") @click.argument("new_version", required=True, metavar="") From e32bd8c603388b2c29a30001db08a8573a93502b Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 10 Jan 2023 12:32:52 +0000 Subject: [PATCH 014/246] Fix schema docs to correctly display required field --- nf_core/schema.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/schema.py b/nf_core/schema.py index b325a6a075..1d36d45c12 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -495,6 +495,7 @@ def schema_to_markdown(self, columns): out += f"{self.schema['description']}\n" # Grouped parameters for definition in self.schema.get("definitions", {}).values(): + required = definition.get("required", []) out += f"\n## {definition.get('title', {})}\n\n" out += f"{definition.get('description', '')}\n\n" out += "".join([f"| {column.title()} " for column in columns]) @@ -513,12 +514,15 @@ def schema_to_markdown(self, columns): out += f"
Help{help_txt}
" elif column == "type": out += f"| `{param.get('type', '')}` " + elif column == "required": + out += f"| {p_key in required or ''} " else: out += f"| {param.get(column, '')} " out += "|\n" # Top-level ungrouped parameters if len(self.schema.get("properties", {})) > 0: + required = self.schema.get("required", []) out += "\n## Other parameters\n\n" out += "".join([f"| {column.title()} " for column in columns]) out += "|\n" From 66e6f12663f6f38c7a3586f17d358949837daeb3 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 10 Jan 2023 15:37:19 +0000 Subject: [PATCH 015/246] Refactor param table generation from schema to be its own method --- nf_core/schema.py | 83 ++++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 1d36d45c12..59a47679fe 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -495,54 +495,57 @@ def schema_to_markdown(self, columns): out += f"{self.schema['description']}\n" # Grouped parameters for definition in self.schema.get("definitions", {}).values(): - required = definition.get("required", []) out += f"\n## {definition.get('title', {})}\n\n" out += f"{definition.get('description', '')}\n\n" - out += "".join([f"| {column.title()} " for column in columns]) - out += "|\n" - out += "".join(["|-----------" for columns in columns]) - out += "|\n" - for p_key, param in definition.get("properties", {}).items(): - for column in columns: - if column == "parameter": - out += f"| `{p_key}` " - elif column == "description": - desc = param.get("description", "").replace("\n", "
") - out += f"| {desc} " - if param.get("help_text", "") != "": - help_txt = param["help_text"].replace("\n", "
") - out += f"
Help{help_txt}
" - elif column == "type": - out += f"| `{param.get('type', '')}` " - elif column == "required": - out += f"| {p_key in required or ''} " - else: - out += f"| {param.get(column, '')} " - out += "|\n" + required = definition.get("required", []) + properties = definition.get("properties", {}).items() + param_table = self.markdown_param_table_content(properties, required, columns) + out += param_table # Top-level ungrouped parameters if len(self.schema.get("properties", {})) > 0: - required = self.schema.get("required", []) out += "\n## Other parameters\n\n" - out += "".join([f"| {column.title()} " for column in columns]) - out += "|\n" - out += "".join(["|-----------" for columns in columns]) - out += "|\n" + required = self.schema.get("required", []) + properties = self.schema.get("properties", {}) + param_table = self.markdown_param_table_content(properties, required, columns) + out += param_table + + return out + + def markdown_param_table(properties, required, columns): + """Creates a markdown table for params from jsonschema properties section - for p_key, param in self.schema.get("properties", {}).items(): - for column in columns: - if column == "parameter": - out += f"| `{p_key}` " - elif column == "description": - out += f"| {param.get('description', '')} " - if param.get("help_text", "") != "": - out += f"
Help{param['help_text']}
" - elif column == "type": - out += f"| `{param.get('type', '')}` " - else: - out += f"| {param.get(column, '')} " - out += "|\n" + Args: + properties (dict): A jsonschema properties dictionary + required (list): A list of the required fields. + Should come from the same level of the jsonschema as properties + columns (list): A list of columns to write + Returns: + str: A string with the markdown table + """ + out = "" + out += "".join([f"| {column.title()} " for column in columns]) + out += "|\n" + out += "".join(["|-----------" for _ in columns]) + out += "|\n" + for p_key, param in properties.items(): + for column in columns: + if column == "parameter": + out += f"| `{p_key}` " + elif column == "description": + desc = param.get("description", "").replace("\n", "
") + out += f"| {desc} " + if param.get("help_text", "") != "": + help_txt = param["help_text"].replace("\n", "
") + out += f"
Help{help_txt}
" + elif column == "type": + out += f"| `{param.get('type', '')}` " + elif column == "required": + out += f"| {p_key in required or ''} " + else: + out += f"| {param.get(column, '')} " + out += "|\n" return out def markdown_to_html(self, markdown_str): From b63a407bdb06800e5b62703dafcad4be57831875 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 10 Jan 2023 15:48:00 +0000 Subject: [PATCH 016/246] Fix markdown table method signature --- nf_core/schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 59a47679fe..ba88e762ea 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -498,8 +498,8 @@ def schema_to_markdown(self, columns): out += f"\n## {definition.get('title', {})}\n\n" out += f"{definition.get('description', '')}\n\n" required = definition.get("required", []) - properties = definition.get("properties", {}).items() - param_table = self.markdown_param_table_content(properties, required, columns) + properties = definition.get("properties", {}) + param_table = self.markdown_param_table(properties, required, columns) out += param_table # Top-level ungrouped parameters @@ -507,12 +507,12 @@ def schema_to_markdown(self, columns): out += "\n## Other parameters\n\n" required = self.schema.get("required", []) properties = self.schema.get("properties", {}) - param_table = self.markdown_param_table_content(properties, required, columns) + param_table = self.markdown_param_table(properties, required, columns) out += param_table return out - def markdown_param_table(properties, required, columns): + def markdown_param_table(self, properties, required, columns): """Creates a markdown table for params from jsonschema properties section Args: From 70ab7827d46edfb0172a5d7b9066ad96aef07050 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 20 Jan 2023 13:00:30 +0100 Subject: [PATCH 017/246] Activate auto-work clean up on successful completion for full-tests --- nf_core/pipeline-template/conf/test_full.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index d92692fa94..46b165a910 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -10,6 +10,8 @@ ---------------------------------------------------------------------------------------- */ +cleanup = true + params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' From 67c8aa9d36493fef7b9b1e6075cf50dee4ab7c95 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 20 Jan 2023 13:02:34 +0100 Subject: [PATCH 018/246] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be6d5820d7..856bc9a604 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) + ### Linting ### Modules From b93e8e0e314725392a1bd522f12c8ecdb71b731c Mon Sep 17 00:00:00 2001 From: "Maxime U. Garcia" Date: Mon, 23 Jan 2023 17:49:37 +0100 Subject: [PATCH 019/246] Add dumpHashes in the debug profile to have all possible hashes in the debug profile --- nf_core/pipeline-template/nextflow.config | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index cc591a54e5..9c7ff286e0 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -81,7 +81,10 @@ try { {% endif %} profiles { - debug { process.beforeScript = 'echo $HOSTNAME' } + debug { + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + } conda { conda.enabled = true docker.enabled = false From b57c1ab6c4043d7f5430949ae0c2c74a20c4f632 Mon Sep 17 00:00:00 2001 From: "Maxime U. Garcia" Date: Mon, 23 Jan 2023 17:58:10 +0100 Subject: [PATCH 020/246] Update nf_core/pipeline-template/nextflow.config spacing --- nf_core/pipeline-template/nextflow.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 9c7ff286e0..fe8c6587ab 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -82,8 +82,8 @@ try { profiles { debug { - dumpHashes = true - process.beforeScript = 'echo $HOSTNAME' + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' } conda { conda.enabled = true From 2a4b23b909155adef85dde79adc341db97995d99 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 24 Jan 2023 09:54:05 +0100 Subject: [PATCH 021/246] fix use of component_type --- nf_core/components/remove.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 528f032124..99df757992 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -127,7 +127,7 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals component, silent=True ): log.warning( - f"Could not install the {self.component_type[:-1]} '{component}', please install it manually with 'nf-core {component_type} install {component}'." + f"Could not install the {self.component_type[:-1]} '{component}', please install it manually with 'nf-core {self.component_type} install {component}'." ) removed_components.append(component) return removed From 51f16643b2bcd793292df2bed50244e9f3485db1 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 24 Jan 2023 09:57:33 +0100 Subject: [PATCH 022/246] fix argument name in docstring --- nf_core/modules/modules_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index cda7c827dc..28481a9c3f 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -494,7 +494,7 @@ def reinstall_repo(self, install_dir, remote_url, module_entries): Args: install_dir (str): The name of directory where modules are installed remote_url (str): The git url of the remote repository - modules ([ dict[str, dict[str, str]] ]): Module entries with + module_entries ([ dict[str, dict[str, str]] ]): Module entries with branch and git sha info Returns: From 5c86b52bac04c39f4c057c890780a2eda7ab788a Mon Sep 17 00:00:00 2001 From: fabianegli Date: Tue, 24 Jan 2023 09:59:52 +0100 Subject: [PATCH 023/246] differentiate variable name from builtin function --- nf_core/modules/modules_json.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 28481a9c3f..d82f206e3e 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -419,10 +419,10 @@ def unsynced_components(self): def parse_dirs(self, dirs, missing_installation, component_type): untracked_dirs = [] - for dir in dirs: + for dir_ in dirs: # Check if the module/subworkflows directory exists in modules.json - install_dir = dir.parts[0] - component = str(Path(*dir.parts[1:])) + install_dir = dir_.parts[0] + component = str(Path(*dir_.parts[1:])) component_in_file = False git_url = None for repo in missing_installation: From da6f860e83d6a32959aa9acc4924cf7c2522201b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 24 Jan 2023 19:32:57 +0100 Subject: [PATCH 024/246] Make jobs automatically resubmit for a much wider range of exit codes now `130..145` --- CHANGELOG.md | 1 + nf_core/pipeline-template/conf/base.config | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 856bc9a604..0e8e84504b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) +- Make jobs automatically resubmit for a much wider range of exit codes (now `130..145`) ### Linting diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index c5c691057d..59622840f2 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -15,7 +15,7 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in 130..145 ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' From e5ce2ab833b5b9f160eb76bcb3dadb17951bb702 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 09:05:35 +0100 Subject: [PATCH 025/246] Update usage.md --- nf_core/pipeline-template/docs/usage.md | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 9a171f5aad..acab79dfb7 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -75,6 +75,29 @@ work # Directory containing the nextflow working files # Other nextflow hidden files, eg. history of pipeline runs and old logs. ``` +If you wish to repeatedly use the same parameters for multiple runs, rather than specifying each flag in the command, you can specify these in a params file. + +Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. + +> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. +The above pipeline run specified with a params file in yaml format: + +```bash +nextflow run {{ name }} -profile docker -params-file params.yaml +``` + +with `params.yaml` containing: + +```yaml +input: './samplesheet.csv' +outdir: './results/' +genome: 'GRCh37' +input: 'data' +<...> +``` + +You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). + ### Updating the pipeline When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: @@ -91,6 +114,10 @@ First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releas This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. +To further assist in reproducbility, you can use share and re-use [params files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. + +> 💡 If you wish to share such profile (such as upload as supplementary material for academic publications), make sure to NOT include cluster specific paths to files, nor institutional specific profiles. + ## Core Nextflow arguments > **NB:** These options are part of Nextflow and use a _single_ hyphen (pipeline parameters use a double-hyphen). From f1d3f14e1a8c61481d40e3b24e7459b112e51698 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 09:06:50 +0100 Subject: [PATCH 026/246] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 856bc9a604..759375d4f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ ### Template -- Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) +- Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] +- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133] ### Linting From aa9f6565c31e3f286368bae7e22f7a3add88517c Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 09:07:13 +0100 Subject: [PATCH 027/246] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 759375d4f1..ac57b68d32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] -- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133] +- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] ### Linting From 263a487844be29023ca084cd9513064a7a09c951 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 27 Jan 2023 08:16:48 +0000 Subject: [PATCH 028/246] [automated] Fix code linting --- CHANGELOG.md | 2 +- nf_core/pipeline-template/docs/usage.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac57b68d32..575049e505 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] -- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] +- Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] ### Linting diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index acab79dfb7..5581680ae6 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -80,7 +80,7 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. > ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. -The above pipeline run specified with a params file in yaml format: +> The above pipeline run specified with a params file in yaml format: ```bash nextflow run {{ name }} -profile docker -params-file params.yaml @@ -96,7 +96,7 @@ input: 'data' <...> ``` -You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). +You can also generate such `YAML`/`JSON` files via [nf-core/launch](https://nf-co.re/launch). ### Updating the pipeline From 5e16056a77c53e126494182d2422bf4966da34d9 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 09:40:02 +0100 Subject: [PATCH 029/246] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/docs/usage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 5581680ae6..24ed0fb4bf 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -79,7 +79,7 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. -> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. +> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. > The above pipeline run specified with a params file in yaml format: ```bash @@ -114,7 +114,7 @@ First, go to the [{{ name }} releases page](https://github.com/{{ name }}/releas This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. -To further assist in reproducbility, you can use share and re-use [params files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. +To further assist in reproducbility, you can use share and re-use [parameter files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. > 💡 If you wish to share such profile (such as upload as supplementary material for academic publications), make sure to NOT include cluster specific paths to files, nor institutional specific profiles. From 579a0c3c6d45ed1dc061c64bfacc1e38c900ce99 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 27 Jan 2023 13:24:32 +0100 Subject: [PATCH 030/246] Update nf_core/pipeline-template/docs/usage.md Co-authored-by: Daniel Straub <42973691+d4straub@users.noreply.github.com> --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 24ed0fb4bf..3a87c2f9de 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -79,7 +79,7 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. -> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) and other infrastructural tweaks. +> ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args). > The above pipeline run specified with a params file in yaml format: ```bash From eeba92b6f4c9c774876a2cc19e00696fa09dfaeb Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 30 Jan 2023 10:46:21 +0100 Subject: [PATCH 031/246] Add 104 to retry error codes --- nf_core/pipeline-template/conf/base.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index 59622840f2..f73c5afaa4 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -15,7 +15,7 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - errorStrategy = { task.exitStatus in 130..145 ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' From c0a6ff56bcf5a599dd86928f1652ae816dc54390 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 1 Feb 2023 01:33:58 +0100 Subject: [PATCH 032/246] Update CHANGELOG.md Co-authored-by: Matthieu Muffato --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e8e84504b..691a3a0b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) -- Make jobs automatically resubmit for a much wider range of exit codes (now `130..145`) +- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ### Linting From 9203455d4e31ceb2ff0ca55da9f875a10032bf62 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 2 Feb 2023 16:14:01 +0100 Subject: [PATCH 033/246] modify jinja2 modules template --- nf_core/module-template/modules/main.nf | 18 +++++++++++++-- nf_core/module-template/modules/meta.yml | 28 +++++++++++++++++------- nf_core/module-template/tests/test.yml | 4 ++++ 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index e8f043f083..c9fe8361b5 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,3 +1,4 @@ +{% if not_minimal -%} // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/modules/nf-core/ // You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: @@ -14,33 +15,42 @@ // bwa mem | samtools view -B -T ref.fasta // TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty // list (`[]`) instead of a file can be used to work around this issue. +{%- endif %} process {{ tool_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} label '{{ process_label }}' + {% if not_minimal -%} // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. + {%- endif %} conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': '{{ docker_container if docker_container else 'quay.io/biocontainers/YOUR-TOOL-HERE' }}' }" input: + {% if not_minimal -%} // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" // MUST be provided as an input via a Groovy Map called "meta". // This information may not be required in some instances e.g. indexing reference genome files: // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf // TODO nf-core: Where applicable please provide/convert compressed files as input/output // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. - {{ 'tuple val(meta), path(bam)' if has_meta else 'path bam' }} + {%- endif %} + {{ {'tuple val(meta), path(bam)' if not_minimal else 'tuple val(meta), path(input)'} if has_meta else {'path bam' if not_minimal else 'path input'} }} output: + {% if not_minimal -%} // TODO nf-core: Named file extensions MUST be emitted for ALL output channels - {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam + {%- endif %} + {{ {'tuple val(meta), path("*.bam")' if not_minimal else 'tuple val(meta), path("*")'} if has_meta else {'path "*.bam"' if not_minimal else 'path "*"'} }}, emit: {bam if not_minimal else output} + {% if not_minimal -%} // TODO nf-core: List additional required output channels/values here + {%- endif %} path "versions.yml" , emit: versions when: @@ -51,6 +61,7 @@ process {{ tool_name_underscore|upper }} { {% if has_meta -%} def prefix = task.ext.prefix ?: "${meta.id}" {%- endif %} + {% if not_minimal -%} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf @@ -60,7 +71,9 @@ process {{ tool_name_underscore|upper }} { // using the Nextflow "task" variable e.g. "--threads $task.cpus" // TODO nf-core: Please replace the example samtools command below with your module's command // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) + {%- endif %} """ + {% if not_minimal -%} samtools \\ sort \\ $args \\ @@ -70,6 +83,7 @@ process {{ tool_name_underscore|upper }} { -T $prefix \\ {%- endif %} $bam + {%- endif %} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index c95e3e1d84..fa3011585b 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,11 +1,15 @@ name: "{{ tool_name_underscore }}" +#{% if not_minimal -%} ## TODO nf-core: Add a description of the module and list keywords +#{%- endif %} description: write your description here keywords: - sort tools: - "{{ tool }}": + #{% if not_minimal -%} ## TODO nf-core: Add a description and other details for the software below + #{%- endif %} description: "{{ tool_description }}" homepage: "{{ tool_doc_url }}" documentation: "{{ tool_doc_url }}" @@ -13,7 +17,9 @@ tools: doi: "" licence: "{{ tool_licence }}" +#{% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as input +#{%- endif %} input: #{% if has_meta %} Only when we have meta - meta: @@ -22,13 +28,17 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] #{% endif %} + #{% if not_minimal -%} ## TODO nf-core: Delete / customise this example input - - bam: - type: file - description: BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" + #{%- endif %} + - #{{ 'bam:' if not_minimal else "input:" }} + type: file + description: #{{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} + pattern: #{{ '"*.{bam,cram,sam}"' if not_minimal else "" }} +#{% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as output +#{%- endif %} output: #{% if has_meta -%} Only when we have meta - meta: @@ -41,11 +51,13 @@ output: type: file description: File containing software versions pattern: "versions.yml" + #{% if not_minimal -%} ## TODO nf-core: Delete / customise this example output - - bam: - type: file - description: Sorted BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" + #{%- endif %} + - #{{ 'bam:' if not_minimal else "output:" }} + type: file + description: #{{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} + pattern: #{{ '"*.{bam,cram,sam}"' if not_minimal else "" }} authors: - "{{ author }}" diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index a65b127c97..4095c1108c 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,12 +1,16 @@ +#{% if not_minimal -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} +#{%- endif %} - name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" command: nextflow run ./tests/modules/{{ org }}/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ tool_dir }}/nextflow.config tags: - "{{ tool }}{% if subtool -%}" - "{{ tool }}/{{ subtool }}{%- endif %}" files: + #{% if not_minimal -%} - path: "output/{{ tool }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - path: "output/{{ tool }}/versions.yml" md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b + #{%- endif %} From f13cc5e8b1ac379cee2c9d35788499675ddc35c7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 09:31:50 +0100 Subject: [PATCH 034/246] run isort --- nf_core/components/create.py | 438 +++++++++++++++++++++++++++++++++ nf_core/modules/create.py | 232 +---------------- nf_core/subworkflows/create.py | 112 +-------- 3 files changed, 454 insertions(+), 328 deletions(-) create mode 100644 nf_core/components/create.py diff --git a/nf_core/components/create.py b/nf_core/components/create.py new file mode 100644 index 0000000000..d44eb02c47 --- /dev/null +++ b/nf_core/components/create.py @@ -0,0 +1,438 @@ +""" +The ComponentCreate class handles generating of module and subworkflow templates +""" + +from __future__ import print_function + +import glob +import json +import logging +import os +import re +import subprocess + +import jinja2 +import questionary +import rich +import yaml +from packaging.version import parse as parse_version + +import nf_core +import nf_core.components.components_create +import nf_core.utils +from nf_core.components.components_command import ComponentCommand + +log = logging.getLogger(__name__) + + +class ComponentCreate(ComponentCommand): + def __init__( + self, + component_type, + directory=".", + component="", + author=None, + process_label=None, + has_meta=None, + force=False, + conda_name=None, + conda_version=None, + ): + super().__init__(component_type, directory) + self.directory = directory + self.component = component + self.author = author + self.process_label = process_label + self.has_meta = has_meta + self.force_overwrite = force + self.subtool = None + self.tool_conda_name = conda_name + self.tool_conda_version = conda_version + self.tool_licence = None + self.tool_licence = "" + self.tool_description = "" + self.tool_doc_url = "" + self.tool_dev_url = "" + self.bioconda = None + self.singularity_container = None + self.docker_container = None + self.file_paths = {} + + def create(self): + """ + Create a new DSL2 module or subworkflow from the nf-core template. + + A module should be named just or + e.g fastqc or samtools/sort, respectively. + + The subworkflow should be named as the main file type it operates on and a short description of the task performed + e.g bam_sort or bam_sort_samtools, respectively. + + If is a pipeline, this function creates a file called: + '/modules/local/tool.nf' + OR + '/modules/local/tool_subtool.nf' + OR for subworkflows + '/subworkflows/local/subworkflow_name.nf' + + If is a clone of nf-core/modules, it creates or modifies the following files: + + For modules: + + modules/modules/nf-core/tool/subtool/ + * main.nf + * meta.yml + modules/tests/modules/nf-core/tool/subtool/ + * main.nf + * test.yml + * nextflow.config + tests/config/pytest_modules.yml + + The function will attempt to automatically find a Bioconda package called + and matching Docker / Singularity images from BioContainers. + + For subworkflows: + subworkflows/nf-core/subworkflow_name/ + * main.nf + * meta.yml + tests/subworkflows/nf-core/subworkflow_name/ + * main.nf + * test.yml + * nextflow.config + tests/config/pytest_modules.yml + + """ + + if self.component_type == "modules": + # Check modules directory structure + self.check_modules_structure() + + # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules + log.info(f"Repository type: [blue]{self.repo_type}") + if self.directory != ".": + log.info(f"Base directory: '{self.directory}'") + + log.info( + "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " + "ctrl+click [link=https://youtu.be/dQw4w9WgXcQ]underlined text[/link] to open links." + ) + + # Collect component info via prompt if empty or invalid + self.component, self.subtool = self._collect_name_prompt() + + # Determine the component name + self.component_name = self.component + self.component_dir = self.component + + if self.subtool: + self.component_name = f"{self.component}/{self.subtool}" + self.component_dir = os.path.join(self.component, self.subtool) + + self.component_name_underscore = self.component_name.replace("/", "_") + + # Check existence of directories early for fast-fail + self.file_paths = self._get_component_dirs() + + if self.component_type == "modules": + # Try to find a bioconda package for 'component' + self._get_bioconda_tool() + + # Prompt for GitHub username + self._get_username() + + self._get_module_structure_components() + + # Create component template with jinja2 + self._render_template() + + if self.repo_type == "modules": + # Add entry to pytest_modules.yml + try: + with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: + pytest_modules_yml = yaml.safe_load(fh) + if self.subtool: + pytest_modules_yml[self.component_name] = [ + f"modules/{self.org}/{self.component}/{self.subtool}/**", + f"tests/modules/{self.org}/{self.component}/{self.subtool}/**", + ] + else: + pytest_modules_yml[ + ("" if self.component_type == "modules" else self.component_type + "/") + self.component_name + ] = [ + f"{self.component_type}/{self.org}/{self.component}/**", + f"tests/{self.component_type}/{self.org}/{self.component}/**", + ] + pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) + with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: + yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) + except FileNotFoundError: + raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") + + new_files = list(self.file_paths.values()) + if self.repo_type == "modules": + new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) + log.info("Created / edited following files:\n " + "\n ".join(new_files)) + + def _get_bioconda_tool(self): + """ + Try to find a bioconda package for 'tool' + """ + while True: + try: + if self.tool_conda_name: + anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) + else: + anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) + + if not self.tool_conda_version: + version = anaconda_response.get("latest_version") + if not version: + version = str(max([parse_version(v) for v in anaconda_response["versions"]])) + else: + version = self.tool_conda_version + + self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) + self.tool_description = anaconda_response.get("summary", "") + self.tool_doc_url = anaconda_response.get("doc_url", "") + self.tool_dev_url = anaconda_response.get("dev_url", "") + if self.tool_conda_name: + self.bioconda = "bioconda::" + self.tool_conda_name + "=" + version + else: + self.bioconda = "bioconda::" + self.tool + "=" + version + log.info(f"Using Bioconda package: '{self.bioconda}'") + break + except (ValueError, LookupError) as e: + log.warning( + f"Could not find Conda dependency using the Anaconda API: '{self.tool_conda_name if self.tool_conda_name else self.tool}'" + ) + if rich.prompt.Confirm.ask("[violet]Do you want to enter a different Bioconda package name?"): + self.tool_conda_name = rich.prompt.Prompt.ask("[violet]Name of Bioconda package").strip() + continue + else: + log.warning( + f"{e}\nBuilding module without tool software and meta, you will need to enter this information manually." + ) + break + + # Try to get the container tag (only if bioconda package was found) + if self.bioconda: + try: + if self.tool_conda_name: + self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( + self.tool_conda_name, version + ) + else: + self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( + self.tool, version + ) + log.info(f"Using Docker container: '{self.docker_container}'") + log.info(f"Using Singularity container: '{self.singularity_container}'") + except (ValueError, LookupError) as e: + log.info(f"Could not find a Docker/Singularity container ({e})") + + def _get_module_structure_components(self): + process_label_defaults = ["process_single", "process_low", "process_medium", "process_high", "process_long"] + if self.process_label is None: + log.info( + "Provide an appropriate resource label for the process, taken from the " + "[link=https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config#L29]nf-core pipeline template[/link].\n" + "For example: {}".format(", ".join(process_label_defaults)) + ) + while self.process_label is None: + self.process_label = questionary.autocomplete( + "Process resource label:", + choices=process_label_defaults, + style=nf_core.utils.nfcore_question_style, + default="process_single", + ).unsafe_ask() + + if self.has_meta is None: + log.info( + "Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' " + "MUST be provided as an input via a Groovy Map called 'meta'. " + "This information may [italic]not[/] be required in some instances, for example " + "[link=https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf]indexing reference genome files[/link]." + ) + while self.has_meta is None: + self.has_meta = rich.prompt.Confirm.ask( + "[violet]Will the module require a meta map of sample information?", default=True + ) + + def _render_template(self): + """ + Create new module/subworkflow files with Jinja2. + """ + object_attrs = vars(self) + # Run jinja2 for each file in the template folder + env = jinja2.Environment( + loader=jinja2.PackageLoader("nf_core", f"{self.component_type[:-1]}-template"), keep_trailing_newline=True + ) + for template_fn, dest_fn in self.file_paths.items(): + log.debug(f"Rendering template file: '{template_fn}'") + j_template = env.get_template(template_fn) + object_attrs["nf_core_version"] = nf_core.__version__ + rendered_output = j_template.render(object_attrs) + + # Write output to the target file + os.makedirs(os.path.dirname(dest_fn), exist_ok=True) + with open(dest_fn, "w") as fh: + log.debug(f"Writing output to: '{dest_fn}'") + fh.write(rendered_output) + + # Mirror file permissions + template_stat = os.stat( + os.path.join(os.path.dirname(nf_core.__file__), f"{self.component_type[:-1]}-template", template_fn) + ) + os.chmod(dest_fn, template_stat.st_mode) + + def _collect_name_prompt(self): + """ + Collect module/subworkflow info via prompt if empty or invalid + """ + # Collect module info via prompt if empty or invalid + subname = None + if self.component_type == "modules": + pattern = r"[^a-z\d/]" + elif self.component_type == "subworkflows": + pattern = r"[^a-z\d_/]" + if self.component is None: + self.component = "" + while self.component == "" or re.search(pattern, self.component) or self.component.count("/") > 0: + # Check + auto-fix for invalid chacters + if re.search(pattern, self.component): + if self.component_type == "modules": + log.warning("Tool/subtool name must be lower-case letters only, with no punctuation") + elif self.component_type == "subworkflows": + log.warning("Subworkflow name must be lower-case letters only, with no punctuation") + name_clean = re.sub(r"[^a-z\d/]", "", self.component.lower()) + if rich.prompt.Confirm.ask(f"[violet]Change '{self.component}' to '{name_clean}'?"): + self.component = name_clean + else: + self.component = "" + + if self.component_type == "modules": + # Split into tool and subtool + if self.component.count("/") > 1: + log.warning("Tool/subtool can have maximum one '/' character") + self.component = "" + elif self.component.count("/") == 1: + self.component, subname = self.component.split("/") + else: + subname = None # Reset edge case: entered '/subtool' as name and gone round loop again + + # Prompt for new entry if we reset + if self.component == "": + if self.component_type == "modules": + self.component = rich.prompt.Prompt.ask("[violet]Name of tool/subtool").strip() + elif self.component_type == "subworkflows": + self.component = rich.prompt.Prompt.ask("[violet]Name of subworkflow").strip() + + if self.component_type == "modules": + return self.component, subname + elif self.component_type == "subworkflows": + return self.component + + def _get_component_dirs(self): + """Given a directory and a tool/subtool or subworkflow, set the file paths and check if they already exist + + Returns dict: keys are relative paths to template files, vals are target paths. + """ + file_paths = {} + if self.repo_type == "pipeline": + local_component_dir = os.path.join(self.directory, self.component_type, "local") + # Check whether component file already exists + component_file = os.path.join(local_component_dir, f"{self.component_name}.nf") + if os.path.exists(component_file) and not self.force_overwrite: + raise UserWarning( + f"{self.component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" + ) + + if self.component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + if self.subtool and os.path.exists(os.path.join(local_component_dir, f"{self.component}.nf")): + raise UserWarning( + f"Module '{self.component}' exists already, cannot make subtool '{self.component_name}'" + ) + + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob(f"{local_component_dir}/{self.component}_*.nf") + if not self.subtool and tool_glob: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" + ) + + # Set file paths + file_paths[os.path.join(self.component_type, "main.nf")] = component_file + + if self.repo_type == "modules": + software_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) + test_dir = os.path.join(self.directory, "tests", self.component_type, self.org, self.component_dir) + + # Check if module/subworkflow directories exist already + if os.path.exists(software_dir) and not self.force_overwrite: + raise UserWarning( + f"{self.component_type[:-1]} directory exists: '{software_dir}'. Use '--force' to overwrite" + ) + if os.path.exists(test_dir) and not self.force_overwrite: + raise UserWarning( + f"{self.component_type[:-1]} test directory exists: '{test_dir}'. Use '--force' to overwrite" + ) + + if self.component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + parent_tool_main_nf = os.path.join( + self.directory, self.component_type, self.org, self.component, "main.nf" + ) + parent_tool_test_nf = os.path.join( + self.directory, self.component_type, self.org, self.component, "main.nf" + ) + if self.subtool and os.path.exists(parent_tool_main_nf): + raise UserWarning( + f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" + ) + if self.subtool and os.path.exists(parent_tool_test_nf): + raise UserWarning( + f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{self.component_name}'" + ) + + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob( + f"{os.path.join(self.directory, self.component_type, self.org, self.component)}/*/main.nf" + ) + if not self.subtool and tool_glob: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" + ) + + # Set file paths + # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure + file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(software_dir, "main.nf") + file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml") + file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") + file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") + file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") + + return file_paths + + def _get_username(self): + """ + Prompt for GitHub username + """ + # Try to guess the current user if `gh` is installed + author_default = None + try: + with open(os.devnull, "w") as devnull: + gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) + author_default = f"@{gh_auth_user['login']}" + except Exception as e: + log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") + + # Regex to valid GitHub username: https://github.com/shinnn/github-username-regex + github_username_regex = re.compile(r"^@[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$") + while self.author is None or not github_username_regex.match(self.author): + if self.author is not None and not github_username_regex.match(self.author): + log.warning("Does not look like a valid GitHub username (must start with an '@')!") + self.author = rich.prompt.Prompt.ask( + f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", + default=author_default, + ) diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 019a77c71f..bb744af021 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -1,30 +1,15 @@ -""" -The ModuleCreate class handles generating of module templates -""" - -from __future__ import print_function - import logging -import os -import questionary -import rich -import yaml -from packaging.version import parse as parse_version - -import nf_core -import nf_core.components.components_create -import nf_core.utils -from nf_core.components.components_command import ComponentCommand +from nf_core.components.create import ComponentCreate log = logging.getLogger(__name__) -class ModuleCreate(ComponentCommand): +class ModuleCreate(ComponentCreate): def __init__( self, - directory=".", - tool="", + pipeline_dir, + component="", author=None, process_label=None, has_meta=None, @@ -32,211 +17,6 @@ def __init__( conda_name=None, conda_version=None, ): - super().__init__("modules", directory) - self.directory = directory - self.tool = tool - self.author = author - self.process_label = process_label - self.has_meta = has_meta - self.force_overwrite = force - self.subtool = None - self.tool_conda_name = conda_name - self.tool_conda_version = conda_version - self.tool_licence = None - self.tool_licence = "" - self.tool_description = "" - self.tool_doc_url = "" - self.tool_dev_url = "" - self.bioconda = None - self.singularity_container = None - self.docker_container = None - self.file_paths = {} - - def create(self): - """ - Create a new DSL2 module from the nf-core template. - - Tool should be named just or - e.g fastqc or samtools/sort, respectively. - - If is a pipeline, this function creates a file called: - '/modules/local/tool.nf' - OR - '/modules/local/tool_subtool.nf' - - If is a clone of nf-core/modules, it creates or modifies the following files: - - modules/modules/nf-core/tool/subtool/ - * main.nf - * meta.yml - modules/tests/modules/nf-core/tool/subtool/ - * main.nf - * test.yml - * nextflow.config - tests/config/pytest_modules.yml - - The function will attempt to automatically find a Bioconda package called - and matching Docker / Singularity images from BioContainers. - """ - - # Check modules directory structure - self.check_modules_structure() - - log.info(f"Repository type: [blue]{self.repo_type}") - if self.directory != ".": - log.info(f"Base directory: '{self.directory}'") - - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " - "ctrl+click [link=https://youtu.be/dQw4w9WgXcQ]underlined text[/link] to open links." + super().__init__( + "modules", pipeline_dir, component, author, process_label, has_meta, force, conda_name, conda_version ) - - # Collect module info via prompt if empty or invalid - self.tool, self.subtool = nf_core.components.components_create.collect_name_prompt( - self.tool, self.component_type - ) - - # Determine the tool name - self.tool_name = self.tool - self.tool_dir = self.tool - - if self.subtool: - self.tool_name = f"{self.tool}/{self.subtool}" - self.tool_dir = os.path.join(self.tool, self.subtool) - - self.tool_name_underscore = self.tool_name.replace("/", "_") - - # Check existence of directories early for fast-fail - self.file_paths = nf_core.components.components_create.get_component_dirs( - self.component_type, - self.repo_type, - self.directory, - self.org, - self.tool_name, - self.tool, - self.subtool, - self.tool_dir, - self.force_overwrite, - ) - - # Try to find a bioconda package for 'tool' - self._get_bioconda_tool() - - # Prompt for GitHub username - self.author = nf_core.components.components_create.get_username(self.author) - - self._get_module_structure_components() - - # Create module template with jinja2 - nf_core.components.components_create.render_template(self.component_type, vars(self), self.file_paths) - - if self.repo_type == "modules": - # Add entry to pytest_modules.yml - try: - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: - pytest_modules_yml = yaml.safe_load(fh) - if self.subtool: - pytest_modules_yml[self.tool_name] = [ - f"modules/{self.org}/{self.tool}/{self.subtool}/**", - f"tests/modules/{self.org}/{self.tool}/{self.subtool}/**", - ] - else: - pytest_modules_yml[self.tool_name] = [ - f"modules/{self.org}/{self.tool}/**", - f"tests/modules/{self.org}/{self.tool}/**", - ] - pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: - yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError: - raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") - - new_files = list(self.file_paths.values()) - if self.repo_type == "modules": - new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) - log.info("Created / edited following files:\n " + "\n ".join(new_files)) - - def _get_bioconda_tool(self): - """ - Try to find a bioconda package for 'tool' - """ - while True: - try: - if self.tool_conda_name: - anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) - else: - anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) - - if not self.tool_conda_version: - version = anaconda_response.get("latest_version") - if not version: - version = str(max([parse_version(v) for v in anaconda_response["versions"]])) - else: - version = self.tool_conda_version - - self.tool_licence = nf_core.utils.parse_anaconda_licence(anaconda_response, version) - self.tool_description = anaconda_response.get("summary", "") - self.tool_doc_url = anaconda_response.get("doc_url", "") - self.tool_dev_url = anaconda_response.get("dev_url", "") - if self.tool_conda_name: - self.bioconda = "bioconda::" + self.tool_conda_name + "=" + version - else: - self.bioconda = "bioconda::" + self.tool + "=" + version - log.info(f"Using Bioconda package: '{self.bioconda}'") - break - except (ValueError, LookupError) as e: - log.warning( - f"Could not find Conda dependency using the Anaconda API: '{self.tool_conda_name if self.tool_conda_name else self.tool}'" - ) - if rich.prompt.Confirm.ask("[violet]Do you want to enter a different Bioconda package name?"): - self.tool_conda_name = rich.prompt.Prompt.ask("[violet]Name of Bioconda package").strip() - continue - else: - log.warning( - f"{e}\nBuilding module without tool software and meta, you will need to enter this information manually." - ) - break - - # Try to get the container tag (only if bioconda package was found) - if self.bioconda: - try: - if self.tool_conda_name: - self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( - self.tool_conda_name, version - ) - else: - self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( - self.tool, version - ) - log.info(f"Using Docker container: '{self.docker_container}'") - log.info(f"Using Singularity container: '{self.singularity_container}'") - except (ValueError, LookupError) as e: - log.info(f"Could not find a Docker/Singularity container ({e})") - - def _get_module_structure_components(self): - process_label_defaults = ["process_single", "process_low", "process_medium", "process_high", "process_long"] - if self.process_label is None: - log.info( - "Provide an appropriate resource label for the process, taken from the " - "[link=https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config#L29]nf-core pipeline template[/link].\n" - "For example: {}".format(", ".join(process_label_defaults)) - ) - while self.process_label is None: - self.process_label = questionary.autocomplete( - "Process resource label:", - choices=process_label_defaults, - style=nf_core.utils.nfcore_question_style, - default="process_single", - ).unsafe_ask() - - if self.has_meta is None: - log.info( - "Where applicable all sample-specific information e.g. 'id', 'single_end', 'read_group' " - "MUST be provided as an input via a Groovy Map called 'meta'. " - "This information may [italic]not[/] be required in some instances, for example " - "[link=https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf]indexing reference genome files[/link]." - ) - while self.has_meta is None: - self.has_meta = rich.prompt.Confirm.ask( - "[violet]Will the module require a meta map of sample information?", default=True - ) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index e61f0c6c8d..bd4e5a4fbd 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -1,114 +1,22 @@ -""" -The SubworkflowCreate class handles generating of subworkflow templates -""" - -from __future__ import print_function - import logging -import os - -import yaml -import nf_core -import nf_core.components.components_create -import nf_core.utils -from nf_core.components.components_command import ComponentCommand +from nf_core.components.create import ComponentCreate log = logging.getLogger(__name__) -class SubworkflowCreate(ComponentCommand): +class SubworkflowCreate(ComponentCreate): def __init__( self, - directory=".", - subworkflow="", + pipeline_dir, + component="", author=None, + process_label=None, + has_meta=None, force=False, + conda_name=None, + conda_version=None, ): - super().__init__("subworkflows", directory) - self.directory = directory - self.subworkflow = subworkflow - self.author = author - self.force_overwrite = force - self.file_paths = {} - - def create(self): - """ - Create a new subworkflow from the nf-core template. - - The subworkflow should be named as the main file type it operates on and a short description of the task performed - e.g bam_sort or bam_sort_samtools, respectively. - - If is a pipeline, this function creates a file called: - '/subworkflows/local/subworkflow_name.nf' - - If is a clone of nf-core/modules, it creates or modifies the following files: - - subworkflows/nf-core/subworkflow_name/ - * main.nf - * meta.yml - tests/subworkflows/nf-core/subworkflow_name/ - * main.nf - * test.yml - * nextflow.config - tests/config/pytest_modules.yml - - """ - - # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules - log.info(f"Repository type: [blue]{self.repo_type}") - if self.directory != ".": - log.info(f"Base directory: '{self.directory}'") - - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " - "ctrl+click [link=https://youtu.be/dQw4w9WgXcQ]underlined text[/link] to open links." - ) - - # Collect module info via prompt if empty or invalid - self.subworkflow = nf_core.components.components_create.collect_name_prompt( - self.subworkflow, self.component_type - ) - - # Determine the tool name - self.subworkflow_name = self.subworkflow - self.subworkflow_dir = self.subworkflow - - # Check existence of directories early for fast-fail - self.file_paths = nf_core.components.components_create.get_component_dirs( - self.component_type, - self.repo_type, - self.directory, - self.org, - self.subworkflow_name, - None, - None, - self.subworkflow_dir, - self.force_overwrite, + super().__init__( + "subworkflows", pipeline_dir, component, author, process_label, has_meta, force, conda_name, conda_version ) - - # Prompt for GitHub username - self.author = nf_core.components.components_create.get_username(self.author) - - # Create subworkflow template with jinja2 - nf_core.components.components_create.render_template(self.component_type, vars(self), self.file_paths) - - if self.repo_type == "modules": - # Add entry to pytest_modules.yml - try: - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: - pytest_modules_yml = yaml.safe_load(fh) - pytest_modules_yml["subworkflows/" + self.subworkflow] = [ - f"subworkflows/{self.org}/{self.subworkflow}/**", - f"tests/subworkflows/{self.org}/{self.subworkflow}/**", - ] - pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: - yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError: - raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") - - new_files = list(self.file_paths.values()) - if self.repo_type == "modules": - new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) - log.info("Created / edited following files:\n " + "\n ".join(new_files)) From ccc3530237df9231d5244ca19c7b97537c2e7cb4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 09:32:25 +0100 Subject: [PATCH 035/246] remove unneeded common functions --- nf_core/components/components_create.py | 177 ------------------------ 1 file changed, 177 deletions(-) delete mode 100644 nf_core/components/components_create.py diff --git a/nf_core/components/components_create.py b/nf_core/components/components_create.py deleted file mode 100644 index 86c42c0373..0000000000 --- a/nf_core/components/components_create.py +++ /dev/null @@ -1,177 +0,0 @@ -import glob -import json -import logging -import os -import re -import subprocess - -import jinja2 -import rich - -import nf_core.utils - -log = logging.getLogger(__name__) - - -def render_template(component_type, object_attrs, file_paths): - """ - Create new module/subworkflow files with Jinja2. - """ - # Run jinja2 for each file in the template folder - env = jinja2.Environment( - loader=jinja2.PackageLoader("nf_core", f"{component_type[:-1]}-template"), keep_trailing_newline=True - ) - for template_fn, dest_fn in file_paths.items(): - log.debug(f"Rendering template file: '{template_fn}'") - j_template = env.get_template(template_fn) - object_attrs["nf_core_version"] = nf_core.__version__ - rendered_output = j_template.render(object_attrs) - - # Write output to the target file - os.makedirs(os.path.dirname(dest_fn), exist_ok=True) - with open(dest_fn, "w") as fh: - log.debug(f"Writing output to: '{dest_fn}'") - fh.write(rendered_output) - - # Mirror file permissions - template_stat = os.stat( - os.path.join(os.path.dirname(nf_core.__file__), f"{component_type[:-1]}-template", template_fn) - ) - os.chmod(dest_fn, template_stat.st_mode) - - -def collect_name_prompt(name, component_type): - """ - Collect module/subworkflow info via prompt if empty or invalid - """ - # Collect module info via prompt if empty or invalid - subname = None - if component_type == "modules": - pattern = r"[^a-z\d/]" - elif component_type == "subworkflows": - pattern = r"[^a-z\d_/]" - if name is None: - name = "" - while name == "" or re.search(pattern, name) or name.count("/") > 0: - # Check + auto-fix for invalid chacters - if re.search(pattern, name): - if component_type == "modules": - log.warning("Tool/subtool name must be lower-case letters only, with no punctuation") - elif component_type == "subworkflows": - log.warning("Subworkflow name must be lower-case letters only, with no punctuation") - name_clean = re.sub(r"[^a-z\d/]", "", name.lower()) - if rich.prompt.Confirm.ask(f"[violet]Change '{name}' to '{name_clean}'?"): - name = name_clean - else: - name = "" - - if component_type == "modules": - # Split into tool and subtool - if name.count("/") > 1: - log.warning("Tool/subtool can have maximum one '/' character") - name = "" - elif name.count("/") == 1: - name, subname = name.split("/") - else: - subname = None # Reset edge case: entered '/subtool' as name and gone round loop again - - # Prompt for new entry if we reset - if name == "": - if component_type == "modules": - name = rich.prompt.Prompt.ask("[violet]Name of tool/subtool").strip() - elif component_type == "subworkflows": - name = rich.prompt.Prompt.ask("[violet]Name of subworkflow").strip() - - if component_type == "modules": - return name, subname - elif component_type == "subworkflows": - return name - - -def get_component_dirs(component_type, repo_type, directory, org, name, supername, subname, new_dir, force_overwrite): - """Given a directory and a tool/subtool or subworkflow, set the file paths and check if they already exist - - Returns dict: keys are relative paths to template files, vals are target paths. - """ - file_paths = {} - if repo_type == "pipeline": - local_component_dir = os.path.join(directory, component_type, "local") - # Check whether component file already exists - component_file = os.path.join(local_component_dir, f"{name}.nf") - if os.path.exists(component_file) and not force_overwrite: - raise UserWarning( - f"{component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" - ) - - if component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - if subname and os.path.exists(os.path.join(local_component_dir, f"{supername}.nf")): - raise UserWarning(f"Module '{supername}' exists already, cannot make subtool '{name}'") - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{local_component_dir}/{supername}_*.nf") - if not subname and tool_glob: - raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{name}'") - - # Set file paths - file_paths[os.path.join(component_type, "main.nf")] = component_file - - if repo_type == "modules": - software_dir = os.path.join(directory, component_type, org, new_dir) - test_dir = os.path.join(directory, "tests", component_type, org, new_dir) - - # Check if module/subworkflow directories exist already - if os.path.exists(software_dir) and not force_overwrite: - raise UserWarning(f"{component_type[:-1]} directory exists: '{software_dir}'. Use '--force' to overwrite") - if os.path.exists(test_dir) and not force_overwrite: - raise UserWarning(f"{component_type[:-1]} test directory exists: '{test_dir}'. Use '--force' to overwrite") - - if component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = os.path.join(directory, component_type, org, supername, "main.nf") - parent_tool_test_nf = os.path.join(directory, component_type, org, supername, "main.nf") - if subname and os.path.exists(parent_tool_main_nf): - raise UserWarning(f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{name}'") - if subname and os.path.exists(parent_tool_test_nf): - raise UserWarning(f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{name}'") - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{os.path.join(directory, component_type, org, supername)}/*/main.nf") - if not subname and tool_glob: - raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{name}'") - - # Set file paths - # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths[os.path.join(component_type, "main.nf")] = os.path.join(software_dir, "main.nf") - file_paths[os.path.join(component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml") - file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") - file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") - file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") - - return file_paths - - -def get_username(author): - """ - Prompt for GitHub username - """ - # Try to guess the current user if `gh` is installed - author_default = None - try: - with open(os.devnull, "w") as devnull: - gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) - author_default = f"@{gh_auth_user['login']}" - except Exception as e: - log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") - - # Regex to valid GitHub username: https://github.com/shinnn/github-username-regex - github_username_regex = re.compile(r"^@[a-zA-Z\d](?:[a-zA-Z\d]|-(?=[a-zA-Z\d])){0,38}$") - while author is None or not github_username_regex.match(author): - if author is not None and not github_username_regex.match(author): - log.warning("Does not look like a valid GitHub username (must start with an '@')!") - author = rich.prompt.Prompt.ask( - f"[violet]GitHub Username:[/]{' (@author)' if author_default is None else ''}", - default=author_default, - ) - - return author From 190f5e3089644e2dcdc291d6c5287aa0a9797112 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 10:15:27 +0100 Subject: [PATCH 036/246] linting --- nf_core/__main__.py | 10 ++++++---- nf_core/components/create.py | 24 ++++++++++-------------- nf_core/module-template/modules/main.nf | 8 ++++++-- nf_core/modules/create.py | 12 +++++++++++- nf_core/subworkflows/create.py | 12 +++++++++++- 5 files changed, 44 insertions(+), 22 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 555783d8d4..d8911f9c98 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -678,7 +678,8 @@ def remove(ctx, dir, tool): @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use") @click.option("-p", "--conda-package-version", type=str, default=None, help="Version of conda package to use") -def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version): +@click.option("-i", "--minimal", is_flag=True, default=False, help="Create a minimal version of the template") +def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, minimal): """ Create a new DSL2 module from the nf-core template. @@ -700,7 +701,7 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam # Run function try: module_create = nf_core.modules.ModuleCreate( - dir, tool, author, label, has_meta, force, conda_name, conda_package_version + dir, tool, author, label, has_meta, force, conda_name, conda_package_version, minimal ) module_create.create() except UserWarning as e: @@ -936,7 +937,8 @@ def test_module(ctx, tool, no_prompts, pytest_args): @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-a", "--author", type=str, metavar="", help="Module author's GitHub username prefixed with '@'") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") -def create_subworkflow(ctx, subworkflow, dir, author, force): +@click.option("-i", "--minimap", is_flag=True, default=False, help="Create a minimal version of the template") +def create_subworkflow(ctx, subworkflow, dir, author, force, minimal): """ Create a new subworkflow from the nf-core template. @@ -949,7 +951,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): # Run function try: - subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force) + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force, minimal) subworkflow_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index d44eb02c47..527cb0c556 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -18,7 +18,6 @@ from packaging.version import parse as parse_version import nf_core -import nf_core.components.components_create import nf_core.utils from nf_core.components.components_command import ComponentCommand @@ -37,6 +36,7 @@ def __init__( force=False, conda_name=None, conda_version=None, + minimal=False, ): super().__init__(component_type, directory) self.directory = directory @@ -57,6 +57,7 @@ def __init__( self.singularity_container = None self.docker_container = None self.file_paths = {} + self.not_minimal = not minimal def create(self): """ @@ -118,7 +119,7 @@ def create(self): ) # Collect component info via prompt if empty or invalid - self.component, self.subtool = self._collect_name_prompt() + self._collect_name_prompt() # Determine the component name self.component_name = self.component @@ -182,7 +183,7 @@ def _get_bioconda_tool(self): if self.tool_conda_name: anaconda_response = nf_core.utils.anaconda_package(self.tool_conda_name, ["bioconda"]) else: - anaconda_response = nf_core.utils.anaconda_package(self.tool, ["bioconda"]) + anaconda_response = nf_core.utils.anaconda_package(self.component, ["bioconda"]) if not self.tool_conda_version: version = anaconda_response.get("latest_version") @@ -198,12 +199,12 @@ def _get_bioconda_tool(self): if self.tool_conda_name: self.bioconda = "bioconda::" + self.tool_conda_name + "=" + version else: - self.bioconda = "bioconda::" + self.tool + "=" + version + self.bioconda = "bioconda::" + self.component + "=" + version log.info(f"Using Bioconda package: '{self.bioconda}'") break except (ValueError, LookupError) as e: log.warning( - f"Could not find Conda dependency using the Anaconda API: '{self.tool_conda_name if self.tool_conda_name else self.tool}'" + f"Could not find Conda dependency using the Anaconda API: '{self.tool_conda_name if self.tool_conda_name else self.component}'" ) if rich.prompt.Confirm.ask("[violet]Do you want to enter a different Bioconda package name?"): self.tool_conda_name = rich.prompt.Prompt.ask("[violet]Name of Bioconda package").strip() @@ -223,7 +224,7 @@ def _get_bioconda_tool(self): ) else: self.docker_container, self.singularity_container = nf_core.utils.get_biocontainer_tag( - self.tool, version + self.component, version ) log.info(f"Using Docker container: '{self.docker_container}'") log.info(f"Using Singularity container: '{self.singularity_container}'") @@ -290,7 +291,7 @@ def _collect_name_prompt(self): Collect module/subworkflow info via prompt if empty or invalid """ # Collect module info via prompt if empty or invalid - subname = None + self.subtool = None if self.component_type == "modules": pattern = r"[^a-z\d/]" elif self.component_type == "subworkflows": @@ -316,9 +317,9 @@ def _collect_name_prompt(self): log.warning("Tool/subtool can have maximum one '/' character") self.component = "" elif self.component.count("/") == 1: - self.component, subname = self.component.split("/") + self.component, self.subtool = self.component.split("/") else: - subname = None # Reset edge case: entered '/subtool' as name and gone round loop again + self.subtool = None # Reset edge case: entered '/subtool' as name and gone round loop again # Prompt for new entry if we reset if self.component == "": @@ -327,11 +328,6 @@ def _collect_name_prompt(self): elif self.component_type == "subworkflows": self.component = rich.prompt.Prompt.ask("[violet]Name of subworkflow").strip() - if self.component_type == "modules": - return self.component, subname - elif self.component_type == "subworkflows": - return self.component - def _get_component_dirs(self): """Given a directory and a tool/subtool or subworkflow, set the file paths and check if they already exist diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index c9fe8361b5..3199e46367 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -40,14 +40,18 @@ process {{ tool_name_underscore|upper }} { // https://github.com/nf-core/modules/blob/master/modules/nf-core/bwa/index/main.nf // TODO nf-core: Where applicable please provide/convert compressed files as input/output // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. + {{ 'tuple val(meta), path(bam)' if has_meta else 'path bam' }} + {%- else %} + {{ 'tuple val(meta), path(input)' if has_meta else 'path input' }} {%- endif %} - {{ {'tuple val(meta), path(bam)' if not_minimal else 'tuple val(meta), path(input)'} if has_meta else {'path bam' if not_minimal else 'path input'} }} output: {% if not_minimal -%} // TODO nf-core: Named file extensions MUST be emitted for ALL output channels + {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam + {%- else %} + {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output {%- endif %} - {{ {'tuple val(meta), path("*.bam")' if not_minimal else 'tuple val(meta), path("*")'} if has_meta else {'path "*.bam"' if not_minimal else 'path "*"'} }}, emit: {bam if not_minimal else output} {% if not_minimal -%} // TODO nf-core: List additional required output channels/values here {%- endif %} diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index bb744af021..16d14ef6aa 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -16,7 +16,17 @@ def __init__( force=False, conda_name=None, conda_version=None, + minimal=False, ): super().__init__( - "modules", pipeline_dir, component, author, process_label, has_meta, force, conda_name, conda_version + "modules", + pipeline_dir, + component, + author, + process_label, + has_meta, + force, + conda_name, + conda_version, + minimal, ) diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index bd4e5a4fbd..47181e477d 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -16,7 +16,17 @@ def __init__( force=False, conda_name=None, conda_version=None, + minimal=False, ): super().__init__( - "subworkflows", pipeline_dir, component, author, process_label, has_meta, force, conda_name, conda_version + "subworkflows", + pipeline_dir, + component, + author, + process_label, + has_meta, + force, + conda_name, + conda_version, + minimal, ) From f20614dfa4e425583e34b06ed452146f234b0380 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 10:53:58 +0100 Subject: [PATCH 037/246] refine new lines --- .prettierignore | 3 +- nf_core/module-template/modules/main.nf | 14 ++++---- nf_core/module-template/modules/meta.yml | 42 ++++++++++++------------ nf_core/module-template/tests/main.nf | 6 ++-- nf_core/module-template/tests/test.yml | 10 +++--- 5 files changed, 38 insertions(+), 37 deletions(-) diff --git a/.prettierignore b/.prettierignore index bd1a8bee9c..4cd77bb4ed 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,4 +3,5 @@ adaptivecard.json slackreport.json docs/api/_build testing - +nf_core/module-template/modules/meta.yml +nf_core/module-template/tests/test.yml diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 3199e46367..3736c91655 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -17,7 +17,7 @@ // list (`[]`) instead of a file can be used to work around this issue. {%- endif %} -process {{ tool_name_underscore|upper }} { +process {{ component_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} label '{{ process_label }}' @@ -26,7 +26,7 @@ process {{ tool_name_underscore|upper }} { // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - {%- endif %} + {%- endif -%} conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': @@ -41,7 +41,7 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: Where applicable please provide/convert compressed files as input/output // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. {{ 'tuple val(meta), path(bam)' if has_meta else 'path bam' }} - {%- else %} + {%- else -%} {{ 'tuple val(meta), path(input)' if has_meta else 'path input' }} {%- endif %} @@ -49,10 +49,10 @@ process {{ tool_name_underscore|upper }} { {% if not_minimal -%} // TODO nf-core: Named file extensions MUST be emitted for ALL output channels {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam - {%- else %} + {%- else -%} {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output {%- endif %} - {% if not_minimal -%} + {%- if not_minimal -%} // TODO nf-core: List additional required output channels/values here {%- endif %} path "versions.yml" , emit: versions @@ -65,7 +65,7 @@ process {{ tool_name_underscore|upper }} { {% if has_meta -%} def prefix = task.ext.prefix ?: "${meta.id}" {%- endif %} - {% if not_minimal -%} + {%- if not_minimal -%} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf @@ -77,7 +77,7 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) {%- endif %} """ - {% if not_minimal -%} + {%- if not_minimal -%} samtools \\ sort \\ $args \\ diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index fa3011585b..6a949772ad 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,15 +1,15 @@ -name: "{{ tool_name_underscore }}" -#{% if not_minimal -%} +name: "{{ component_name_underscore }}" +{% if not_minimal -%} ## TODO nf-core: Add a description of the module and list keywords -#{%- endif %} +{%- endif -%} description: write your description here keywords: - sort tools: - "{{ tool }}": - #{% if not_minimal -%} + {% if not_minimal -%} ## TODO nf-core: Add a description and other details for the software below - #{%- endif %} + {%- endif -%} description: "{{ tool_description }}" homepage: "{{ tool_doc_url }}" documentation: "{{ tool_doc_url }}" @@ -17,9 +17,9 @@ tools: doi: "" licence: "{{ tool_licence }}" -#{% if not_minimal -%} +{% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as input -#{%- endif %} +{%- endif -%} input: #{% if has_meta %} Only when we have meta - meta: @@ -27,18 +27,18 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - #{% endif %} - #{% if not_minimal -%} + {% endif %} + {% if not_minimal -%} ## TODO nf-core: Delete / customise this example input - #{%- endif %} - - #{{ 'bam:' if not_minimal else "input:" }} + {%- endif %} + - {{ 'bam:' if not_minimal else "input:" }} type: file - description: #{{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} - pattern: #{{ '"*.{bam,cram,sam}"' if not_minimal else "" }} + description: {{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_minimal else "" }} -#{% if not_minimal -%} +{% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as output -#{%- endif %} +{%- endif -%} output: #{% if has_meta -%} Only when we have meta - meta: @@ -46,18 +46,18 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - #{% endif %} + {% endif %} - versions: type: file description: File containing software versions pattern: "versions.yml" - #{% if not_minimal -%} + {% if not_minimal -%} ## TODO nf-core: Delete / customise this example output - #{%- endif %} - - #{{ 'bam:' if not_minimal else "output:" }} + {%- endif %} + - {{ 'bam:' if not_minimal else "output:" }} type: file - description: #{{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} - pattern: #{{ '"*.{bam,cram,sam}"' if not_minimal else "" }} + description: {{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_minimal else "" }} authors: - "{{ author }}" diff --git a/nf_core/module-template/tests/main.nf b/nf_core/module-template/tests/main.nf index 351de72385..fcb7195fe4 100644 --- a/nf_core/module-template/tests/main.nf +++ b/nf_core/module-template/tests/main.nf @@ -2,9 +2,9 @@ nextflow.enable.dsl = 2 -include { {{ tool_name_underscore|upper }} } from '../../../../{{ "../" if subtool else "" }}modules/{{ org }}/{{ tool_dir }}/main.nf' +include { {{ component_name_underscore|upper }} } from '../../../../{{ "../" if subtool else "" }}modules/{{ org }}/{{ component_dir }}/main.nf' -workflow test_{{ tool_name_underscore }} { +workflow test_{{ component_name_underscore }} { {% if has_meta %} input = [ [ id:'test', single_end:false ], // meta map @@ -14,5 +14,5 @@ workflow test_{{ tool_name_underscore }} { input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) {%- endif %} - {{ tool_name_underscore|upper }} ( input ) + {{ component_name_underscore|upper }} ( input ) } diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index 4095c1108c..5483f3f70f 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,16 +1,16 @@ -#{% if not_minimal -%} +{%- if not_minimal -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} -#{%- endif %} +{%- endif %} - name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" - command: nextflow run ./tests/modules/{{ org }}/{{ tool_dir }} -entry test_{{ tool_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ tool_dir }}/nextflow.config + command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ component_dir }}/nextflow.config tags: - "{{ tool }}{% if subtool -%}" - "{{ tool }}/{{ subtool }}{%- endif %}" files: - #{% if not_minimal -%} + {%- if not_minimal -%} - path: "output/{{ tool }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - path: "output/{{ tool }}/versions.yml" md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b - #{%- endif %} + {%- endif %} From cb8c6122a2f012b4e816a3ef5d1fdb82866c16eb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 12:36:54 +0100 Subject: [PATCH 038/246] run isort --- nf_core/__main__.py | 5 ++--- nf_core/components/create.py | 3 ++- nf_core/module-template/modules/meta.yml | 8 ++++---- nf_core/module-template/tests/test.yml | 14 ++++++++------ nf_core/subworkflows/create.py | 12 +----------- tests/subworkflows/create.py | 2 +- 6 files changed, 18 insertions(+), 26 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d8911f9c98..b02348c753 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -937,8 +937,7 @@ def test_module(ctx, tool, no_prompts, pytest_args): @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-a", "--author", type=str, metavar="", help="Module author's GitHub username prefixed with '@'") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") -@click.option("-i", "--minimap", is_flag=True, default=False, help="Create a minimal version of the template") -def create_subworkflow(ctx, subworkflow, dir, author, force, minimal): +def create_subworkflow(ctx, subworkflow, dir, author, force): """ Create a new subworkflow from the nf-core template. @@ -951,7 +950,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, minimal): # Run function try: - subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force, minimal) + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(dir, subworkflow, author, force) subworkflow_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 527cb0c556..fc31aac306 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -141,7 +141,8 @@ def create(self): # Prompt for GitHub username self._get_username() - self._get_module_structure_components() + if self.component_type == "modules": + self._get_module_structure_components() # Create component template with jinja2 self._render_template() diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 6a949772ad..236fab2aec 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -6,10 +6,10 @@ description: write your description here keywords: - sort tools: - - "{{ tool }}": + - "{{ component }}": {% if not_minimal -%} ## TODO nf-core: Add a description and other details for the software below - {%- endif -%} + {% endif -%} description: "{{ tool_description }}" homepage: "{{ tool_doc_url }}" documentation: "{{ tool_doc_url }}" @@ -19,7 +19,7 @@ tools: {% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as input -{%- endif -%} +{% endif -%} input: #{% if has_meta %} Only when we have meta - meta: @@ -38,7 +38,7 @@ input: {% if not_minimal -%} ## TODO nf-core: Add a description of all of the variables used as output -{%- endif -%} +{% endif -%} output: #{% if has_meta -%} Only when we have meta - meta: diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index 5483f3f70f..deabcf23a9 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -2,15 +2,17 @@ ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} {%- endif %} -- name: "{{ tool }}{{ ' '+subtool if subtool else '' }}" +- name: "{{ component }}{{ ' '+subtool if subtool else '' }}" command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ component_dir }}/nextflow.config tags: - - "{{ tool }}{% if subtool -%}" - - "{{ tool }}/{{ subtool }}{%- endif %}" + - "{{ component }}{% if subtool -%}" + - "{{ component }}/{{ subtool }}{%- endif %}" files: - {%- if not_minimal -%} - - path: "output/{{ tool }}/test.bam" + {% if not_minimal -%} + - path: "output/{{ component }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: "output/{{ tool }}/versions.yml" + - path: "output/{{ component }}/versions.yml" md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b + {% else %} + - path: "" {%- endif %} diff --git a/nf_core/subworkflows/create.py b/nf_core/subworkflows/create.py index 47181e477d..963076455e 100644 --- a/nf_core/subworkflows/create.py +++ b/nf_core/subworkflows/create.py @@ -11,22 +11,12 @@ def __init__( pipeline_dir, component="", author=None, - process_label=None, - has_meta=None, force=False, - conda_name=None, - conda_version=None, - minimal=False, ): super().__init__( "subworkflows", pipeline_dir, component, author, - process_label, - has_meta, - force, - conda_name, - conda_version, - minimal, + force=force, ) diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index eac4929136..60ee6add9a 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -28,7 +28,7 @@ def test_subworkflows_create_fail_exists(self): def test_subworkflows_create_nfcore_modules(self): """Create a subworkflow in nf-core/modules clone""" subworkflow_create = nf_core.subworkflows.SubworkflowCreate( - self.nfcore_modules, "test_subworkflow", "@author", True + self.nfcore_modules, "test_subworkflow", "@author", force=True ) subworkflow_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf")) From c65b36c10d22588e65d9223b3d67432af7270cd9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 13:13:12 +0100 Subject: [PATCH 039/246] fix some more new lines --- nf_core/module-template/modules/main.nf | 4 ++-- nf_core/module-template/tests/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 3736c91655..6bfa9eaa92 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,4 +1,4 @@ -{% if not_minimal -%} +{%- if not_minimal -%} // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/modules/nf-core/ // You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: @@ -15,7 +15,7 @@ // bwa mem | samtools view -B -T ref.fasta // TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty // list (`[]`) instead of a file can be used to work around this issue. -{%- endif %} +{%- endif -%} process {{ component_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index deabcf23a9..61486b9249 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,7 +1,7 @@ {%- if not_minimal -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} -{%- endif %} +{%- endif -%} - name: "{{ component }}{{ ' '+subtool if subtool else '' }}" command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ component_dir }}/nextflow.config tags: @@ -13,6 +13,6 @@ md5sum: e667c7caad0bc4b7ac383fd023c654fc - path: "output/{{ component }}/versions.yml" md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b - {% else %} + {% else -%} - path: "" {%- endif %} From a54b54a9673a2db9a2b7732dcac7bc9960a0c95e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 13:20:43 +0100 Subject: [PATCH 040/246] run black with new version --- nf_core/bump_version.py | 2 -- nf_core/create.py | 1 - nf_core/download.py | 1 - nf_core/launch.py | 3 --- nf_core/lint/__init__.py | 1 - nf_core/lint/files_unchanged.py | 2 -- nf_core/lint/template_strings.py | 1 - nf_core/list.py | 1 - nf_core/modules/bump_versions.py | 1 - nf_core/utils.py | 2 -- tests/test_download.py | 1 - tests/test_test_utils.py | 1 - 12 files changed, 17 deletions(-) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 1f38775bb6..129016fa38 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -133,12 +133,10 @@ def update_file_version(filename, pipeline_obj, patterns): replacements = [] for pattern in patterns: - found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/create.py b/nf_core/create.py index 045c35d1b4..74c9df1b87 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -283,7 +283,6 @@ def render_template(self): # Set the paths to skip according to customization for template_fn_path_obj in template_files: - template_fn_path = str(template_fn_path_obj) # Skip files that are in the self.skip_paths list diff --git a/nf_core/download.py b/nf_core/download.py index 2f964b3afd..9d430e3352 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -506,7 +506,6 @@ def get_singularity_images(self): containers_download = [] containers_pull = [] for container in self.containers: - # Fetch the output and cached filenames for this container out_path, cache_path = self.singularity_image_filenames(container) diff --git a/nf_core/launch.py b/nf_core/launch.py index 87150172f7..648c8775f8 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -98,7 +98,6 @@ def __init__( self.cli_launch = True def launch_pipeline(self): - # Prompt for pipeline if not supplied and no web launch ID if self.pipeline is None and self.web_id is None: launch_type = questionary.select( @@ -461,7 +460,6 @@ def prompt_group(self, group_id, group_obj): answers = {} error_msgs = [] while not while_break: - if len(error_msgs) == 0: self.print_param_header(group_id, group_obj, True) @@ -698,7 +696,6 @@ def build_command(self): # Pipeline parameters if len(self.schema_obj.input_params) > 0: - # Write the user selection to a file and run nextflow with that if self.use_params_file: dump_json_with_prettier(self.params_out, self.schema_obj.input_params) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index b46399eb97..e014a933ea 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -434,7 +434,6 @@ def format_result(test_results): ) def _print_summary(self): - # Summary table summary_colour = "red" if len(self.failed) > 0 else "green" table = Table(box=rich.box.ROUNDED, style=summary_colour) diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index cadced5483..c0be64d0d7 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -147,7 +147,6 @@ def _tf(file_path): # Files that must be completely unchanged from template for files in files_exact: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): @@ -177,7 +176,6 @@ def _tf(file_path): # Files that can be added to, but that must contain the template contents for files in files_partial: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py index 436abe7b2b..fb1f0f32e5 100644 --- a/nf_core/lint/template_strings.py +++ b/nf_core/lint/template_strings.py @@ -24,7 +24,6 @@ def template_strings(self): # Loop through files, searching for string num_matches = 0 for fn in self.files: - # Skip binary files binary_ftypes = ["image", "application/java-archive"] (ftype, encoding) = mimetypes.guess_type(fn) diff --git a/nf_core/list.py b/nf_core/list.py index 53307ac9bd..77a9ac3919 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -328,7 +328,6 @@ def get_local_nf_workflow_details(self): """Get full details about a local cached workflow""" if self.local_path is None: - # Try to guess the local cache directory if len(os.environ.get("NXF_ASSETS", "")) > 0: nf_wfdir = os.path.join(os.environ.get("NXF_ASSETS"), self.full_name) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 7b9702622e..6d61d4c750 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -187,7 +187,6 @@ def bump_module_version(self, module): found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/utils.py b/nf_core/utils.py index b60f61fff0..55703ab450 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -872,7 +872,6 @@ def get_repo_releases_branches(pipeline, wfs): # Repo is a nf-core pipeline for wf in wfs.remote_workflows: if wf.full_name == pipeline or wf.name == pipeline: - # Set to full name just in case it didn't have the nf-core/ prefix pipeline = wf.full_name @@ -883,7 +882,6 @@ def get_repo_releases_branches(pipeline, wfs): # Arbitrary GitHub repo else: if pipeline.count("/") == 1: - # Looks like a GitHub address - try working with this repo log.debug( f"Pipeline '{pipeline}' not in nf-core, but looks like a GitHub address - fetching releases from API" diff --git a/tests/test_download.py b/tests/test_download.py index 4577a83992..e2ae882394 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -18,7 +18,6 @@ class DownloadTest(unittest.TestCase): - # # Tests for 'get_release_hash' # diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index ddf88ef74a..c4e3d49ae0 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -33,7 +33,6 @@ def test_tmp_folder_does_not_exist_after(): def test_set_wd(): - with tempfile.TemporaryDirectory() as tmpdirname: with set_wd(tmpdirname): context_wd = Path().resolve() From c89a94c6eb0f6f861c966c35331e450216234cf3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 13:56:54 +0100 Subject: [PATCH 041/246] make tests pass because of template new lines --- nf_core/module-template/modules/main.nf | 10 +++++----- nf_core/module-template/tests/test.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 6bfa9eaa92..d7a2bb9826 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -15,7 +15,7 @@ // bwa mem | samtools view -B -T ref.fasta // TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty // list (`[]`) instead of a file can be used to work around this issue. -{%- endif -%} +{%- endif %} process {{ component_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} @@ -26,7 +26,7 @@ process {{ component_name_underscore|upper }} { // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. - {%- endif -%} + {% endif -%} conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': @@ -52,7 +52,7 @@ process {{ component_name_underscore|upper }} { {%- else -%} {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output {%- endif %} - {%- if not_minimal -%} + {% if not_minimal -%} // TODO nf-core: List additional required output channels/values here {%- endif %} path "versions.yml" , emit: versions @@ -65,7 +65,7 @@ process {{ component_name_underscore|upper }} { {% if has_meta -%} def prefix = task.ext.prefix ?: "${meta.id}" {%- endif %} - {%- if not_minimal -%} + {% if not_minimal -%} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf @@ -77,7 +77,7 @@ process {{ component_name_underscore|upper }} { // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) {%- endif %} """ - {%- if not_minimal -%} + {% if not_minimal -%} samtools \\ sort \\ $args \\ diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index 61486b9249..21fa60dd5e 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,7 +1,7 @@ {%- if not_minimal -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} -{%- endif -%} +{% endif -%} - name: "{{ component }}{{ ' '+subtool if subtool else '' }}" command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config -c ./tests/modules/{{ org }}/{{ component_dir }}/nextflow.config tags: From 5c5ae91c71fc24dd9ef9a714e16f2fdb40f04cd7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 3 Feb 2023 14:45:26 +0100 Subject: [PATCH 042/246] update CHANGELOG --- CHANGELOG.md | 2 ++ nf_core/__main__.py | 14 +++++++++++--- nf_core/components/create.py | 4 ++-- nf_core/module-template/modules/main.nf | 14 +++++++------- nf_core/module-template/modules/meta.yml | 24 ++++++++++++------------ nf_core/module-template/tests/test.yml | 4 ++-- nf_core/modules/create.py | 4 ++-- 7 files changed, 38 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 575049e505..0c86f897f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ ### Modules +- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175)) + ### Subworkflows ### General diff --git a/nf_core/__main__.py b/nf_core/__main__.py index b02348c753..1372ff7fbc 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -678,8 +678,16 @@ def remove(ctx, dir, tool): @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use") @click.option("-p", "--conda-package-version", type=str, default=None, help="Version of conda package to use") -@click.option("-i", "--minimal", is_flag=True, default=False, help="Create a minimal version of the template") -def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, minimal): +@click.option( + "-i", + "--empty-template", + is_flag=True, + default=False, + help="Create a version of the template without TODOs or examples", +) +def create_module( + ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, empty_template +): """ Create a new DSL2 module from the nf-core template. @@ -701,7 +709,7 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam # Run function try: module_create = nf_core.modules.ModuleCreate( - dir, tool, author, label, has_meta, force, conda_name, conda_package_version, minimal + dir, tool, author, label, has_meta, force, conda_name, conda_package_version, empty_template ) module_create.create() except UserWarning as e: diff --git a/nf_core/components/create.py b/nf_core/components/create.py index fc31aac306..e626de4aaa 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -36,7 +36,7 @@ def __init__( force=False, conda_name=None, conda_version=None, - minimal=False, + empty_template=False, ): super().__init__(component_type, directory) self.directory = directory @@ -57,7 +57,7 @@ def __init__( self.singularity_container = None self.docker_container = None self.file_paths = {} - self.not_minimal = not minimal + self.not_empty_template = not empty_template def create(self): """ diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index d7a2bb9826..895ad8f68c 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,4 +1,4 @@ -{%- if not_minimal -%} +{%- if not_empty_template -%} // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/modules/nf-core/ // You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: @@ -21,7 +21,7 @@ process {{ component_name_underscore|upper }} { tag {{ '"$meta.id"' if has_meta else "'$bam'" }} label '{{ process_label }}' - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. @@ -33,7 +33,7 @@ process {{ component_name_underscore|upper }} { '{{ docker_container if docker_container else 'quay.io/biocontainers/YOUR-TOOL-HERE' }}' }" input: - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" // MUST be provided as an input via a Groovy Map called "meta". // This information may not be required in some instances e.g. indexing reference genome files: @@ -46,13 +46,13 @@ process {{ component_name_underscore|upper }} { {%- endif %} output: - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: Named file extensions MUST be emitted for ALL output channels {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam {%- else -%} {{ 'tuple val(meta), path("*")' if has_meta else 'path "*"' }}, emit: output {%- endif %} - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: List additional required output channels/values here {%- endif %} path "versions.yml" , emit: versions @@ -65,7 +65,7 @@ process {{ component_name_underscore|upper }} { {% if has_meta -%} def prefix = task.ext.prefix ?: "${meta.id}" {%- endif %} - {% if not_minimal -%} + {% if not_empty_template -%} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/modules/nf-core/homer/annotatepeaks/main.nf @@ -77,7 +77,7 @@ process {{ component_name_underscore|upper }} { // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) {%- endif %} """ - {% if not_minimal -%} + {% if not_empty_template -%} samtools \\ sort \\ $args \\ diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 236fab2aec..05ecb6f980 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,5 +1,5 @@ name: "{{ component_name_underscore }}" -{% if not_minimal -%} +{% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords {%- endif -%} description: write your description here @@ -7,7 +7,7 @@ keywords: - sort tools: - "{{ component }}": - {% if not_minimal -%} + {% if not_empty_template -%} ## TODO nf-core: Add a description and other details for the software below {% endif -%} description: "{{ tool_description }}" @@ -17,7 +17,7 @@ tools: doi: "" licence: "{{ tool_licence }}" -{% if not_minimal -%} +{% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as input {% endif -%} input: @@ -28,15 +28,15 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] {% endif %} - {% if not_minimal -%} + {% if not_empty_template -%} ## TODO nf-core: Delete / customise this example input {%- endif %} - - {{ 'bam:' if not_minimal else "input:" }} + - {{ 'bam:' if not_empty_template else "input:" }} type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_minimal else "" }} + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} -{% if not_minimal -%} +{% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as output {% endif -%} output: @@ -51,13 +51,13 @@ output: type: file description: File containing software versions pattern: "versions.yml" - {% if not_minimal -%} + {% if not_empty_template -%} ## TODO nf-core: Delete / customise this example output {%- endif %} - - {{ 'bam:' if not_minimal else "output:" }} + - {{ 'bam:' if not_empty_template else "output:" }} type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_minimal else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_minimal else "" }} + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} authors: - "{{ author }}" diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml index 21fa60dd5e..4f38dec298 100644 --- a/nf_core/module-template/tests/test.yml +++ b/nf_core/module-template/tests/test.yml @@ -1,4 +1,4 @@ -{%- if not_minimal -%} +{%- if not_empty_template -%} ## TODO nf-core: Please run the following command to build this file: # nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} {% endif -%} @@ -8,7 +8,7 @@ - "{{ component }}{% if subtool -%}" - "{{ component }}/{{ subtool }}{%- endif %}" files: - {% if not_minimal -%} + {% if not_empty_template -%} - path: "output/{{ component }}/test.bam" md5sum: e667c7caad0bc4b7ac383fd023c654fc - path: "output/{{ component }}/versions.yml" diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index 16d14ef6aa..b5368130ce 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -16,7 +16,7 @@ def __init__( force=False, conda_name=None, conda_version=None, - minimal=False, + empty_template=False, ): super().__init__( "modules", @@ -28,5 +28,5 @@ def __init__( force, conda_name, conda_version, - minimal, + empty_template, ) From c6cf507be77f5123a8582909b8e333177b96b0b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 3 Feb 2023 15:09:22 +0100 Subject: [PATCH 043/246] Update nf_core/__main__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 1372ff7fbc..841cb9f7e7 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -683,7 +683,7 @@ def remove(ctx, dir, tool): "--empty-template", is_flag=True, default=False, - help="Create a version of the template without TODOs or examples", + help="Create a module from the template without TODOs or examples", ) def create_module( ctx, tool, dir, author, label, meta, no_meta, force, conda_name, conda_package_version, empty_template From 6ff70af488b642d43a27a3d89008fc1cac7ed0e4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 4 Feb 2023 19:33:35 +0100 Subject: [PATCH 044/246] Run new version of Black --- nf_core/bump_version.py | 2 -- nf_core/create.py | 1 - nf_core/download.py | 1 - nf_core/launch.py | 3 --- nf_core/lint/__init__.py | 1 - nf_core/lint/files_unchanged.py | 2 -- nf_core/lint/template_strings.py | 1 - nf_core/list.py | 1 - nf_core/modules/bump_versions.py | 1 - nf_core/utils.py | 2 -- tests/test_download.py | 1 - tests/test_test_utils.py | 1 - 12 files changed, 17 deletions(-) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 1f38775bb6..129016fa38 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -133,12 +133,10 @@ def update_file_version(filename, pipeline_obj, patterns): replacements = [] for pattern in patterns: - found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/create.py b/nf_core/create.py index 045c35d1b4..74c9df1b87 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -283,7 +283,6 @@ def render_template(self): # Set the paths to skip according to customization for template_fn_path_obj in template_files: - template_fn_path = str(template_fn_path_obj) # Skip files that are in the self.skip_paths list diff --git a/nf_core/download.py b/nf_core/download.py index 2f964b3afd..9d430e3352 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -506,7 +506,6 @@ def get_singularity_images(self): containers_download = [] containers_pull = [] for container in self.containers: - # Fetch the output and cached filenames for this container out_path, cache_path = self.singularity_image_filenames(container) diff --git a/nf_core/launch.py b/nf_core/launch.py index 87150172f7..648c8775f8 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -98,7 +98,6 @@ def __init__( self.cli_launch = True def launch_pipeline(self): - # Prompt for pipeline if not supplied and no web launch ID if self.pipeline is None and self.web_id is None: launch_type = questionary.select( @@ -461,7 +460,6 @@ def prompt_group(self, group_id, group_obj): answers = {} error_msgs = [] while not while_break: - if len(error_msgs) == 0: self.print_param_header(group_id, group_obj, True) @@ -698,7 +696,6 @@ def build_command(self): # Pipeline parameters if len(self.schema_obj.input_params) > 0: - # Write the user selection to a file and run nextflow with that if self.use_params_file: dump_json_with_prettier(self.params_out, self.schema_obj.input_params) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index b46399eb97..e014a933ea 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -434,7 +434,6 @@ def format_result(test_results): ) def _print_summary(self): - # Summary table summary_colour = "red" if len(self.failed) > 0 else "green" table = Table(box=rich.box.ROUNDED, style=summary_colour) diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index cadced5483..c0be64d0d7 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -147,7 +147,6 @@ def _tf(file_path): # Files that must be completely unchanged from template for files in files_exact: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): @@ -177,7 +176,6 @@ def _tf(file_path): # Files that can be added to, but that must contain the template contents for files in files_partial: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py index 436abe7b2b..fb1f0f32e5 100644 --- a/nf_core/lint/template_strings.py +++ b/nf_core/lint/template_strings.py @@ -24,7 +24,6 @@ def template_strings(self): # Loop through files, searching for string num_matches = 0 for fn in self.files: - # Skip binary files binary_ftypes = ["image", "application/java-archive"] (ftype, encoding) = mimetypes.guess_type(fn) diff --git a/nf_core/list.py b/nf_core/list.py index 53307ac9bd..77a9ac3919 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -328,7 +328,6 @@ def get_local_nf_workflow_details(self): """Get full details about a local cached workflow""" if self.local_path is None: - # Try to guess the local cache directory if len(os.environ.get("NXF_ASSETS", "")) > 0: nf_wfdir = os.path.join(os.environ.get("NXF_ASSETS"), self.full_name) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 7b9702622e..6d61d4c750 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -187,7 +187,6 @@ def bump_module_version(self, module): found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/utils.py b/nf_core/utils.py index b60f61fff0..55703ab450 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -872,7 +872,6 @@ def get_repo_releases_branches(pipeline, wfs): # Repo is a nf-core pipeline for wf in wfs.remote_workflows: if wf.full_name == pipeline or wf.name == pipeline: - # Set to full name just in case it didn't have the nf-core/ prefix pipeline = wf.full_name @@ -883,7 +882,6 @@ def get_repo_releases_branches(pipeline, wfs): # Arbitrary GitHub repo else: if pipeline.count("/") == 1: - # Looks like a GitHub address - try working with this repo log.debug( f"Pipeline '{pipeline}' not in nf-core, but looks like a GitHub address - fetching releases from API" diff --git a/tests/test_download.py b/tests/test_download.py index 4577a83992..e2ae882394 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -18,7 +18,6 @@ class DownloadTest(unittest.TestCase): - # # Tests for 'get_release_hash' # diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index ddf88ef74a..c4e3d49ae0 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -33,7 +33,6 @@ def test_tmp_folder_does_not_exist_after(): def test_set_wd(): - with tempfile.TemporaryDirectory() as tmpdirname: with set_wd(tmpdirname): context_wd = Path().resolve() From 2b75ef9c57fdc37b0d279583774f833b4df4075e Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 6 Feb 2023 14:30:56 +0100 Subject: [PATCH 045/246] add pre-commit conf to pipeline template, update pre-commit linter versions --- .pre-commit-config.yaml | 7 +++---- nf_core/.pre-commit-prettier-config.yaml | 2 +- nf_core/pipeline-template/.pre-commit-config.yaml | 5 +++++ 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 nf_core/pipeline-template/.pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eddb0f1048..b7aeeb5bc9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,13 @@ repos: - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.1.0 hooks: - id: black - language_version: python3.9 - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.6.2" + rev: "v2.7.1" hooks: - id: prettier diff --git a/nf_core/.pre-commit-prettier-config.yaml b/nf_core/.pre-commit-prettier-config.yaml index 80f75fb22c..0c31cdb99f 100644 --- a/nf_core/.pre-commit-prettier-config.yaml +++ b/nf_core/.pre-commit-prettier-config.yaml @@ -1,5 +1,5 @@ repos: - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.6.2" + rev: "v2.7.1" hooks: - id: prettier diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml new file mode 100644 index 0000000000..0c31cdb99f --- /dev/null +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v2.7.1" + hooks: + - id: prettier From 7c5d0966c955276efe213f38713da9c705f5c184 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Feb 2023 13:00:10 +0100 Subject: [PATCH 046/246] allowing new line before description field --- nf_core/module-template/modules/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 05ecb6f980..9b42bb3bdf 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,7 +1,7 @@ name: "{{ component_name_underscore }}" {% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords -{%- endif -%} +{% endif -%} description: write your description here keywords: - sort From 18277d666c06a57baf3be9e8707c15afed9f4a80 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Feb 2023 14:55:26 +0100 Subject: [PATCH 047/246] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5dd860b3e..b5c5701120 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ ### Modules -- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175)) +- Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175) & [#2177](https://github.com/nf-core/tools/pull/2177)) ### Subworkflows From 53c874fc193ab73f7457d79f0239a53018393850 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Feb 2023 16:38:53 +0100 Subject: [PATCH 048/246] checkout to sha when getting list of available components --- nf_core/components/install.py | 6 +++--- nf_core/modules/modules_repo.py | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index f9a16f73cf..cb6c59aec3 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -169,19 +169,19 @@ def collect_and_verify_name(self, component, modules_repo): if component is None: component = questionary.autocomplete( f"{'Tool' if self.component_type == 'modules' else 'Subworkflow'} name:", - choices=sorted(modules_repo.get_avail_components(self.component_type)), + choices=sorted(modules_repo.get_avail_components(self.component_type, commit=self.sha)), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() # Check that the supplied name is an available module/subworkflow - if component and component not in modules_repo.get_avail_components(self.component_type): + if component and component not in modules_repo.get_avail_components(self.component_type, commit=self.sha): log.error( f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}." ) log.info(f"Use the command 'nf-core {self.component_type} list' to view available software") return False - if not modules_repo.component_exists(component, self.component_type): + if not modules_repo.component_exists(component, self.component_type, commit=self.sha): warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})" log.warning(warn_msg) return False diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 606514e55e..5f77148867 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -302,7 +302,7 @@ def checkout(self, commit): """ self.repo.git.checkout(commit) - def component_exists(self, component_name, component_type, checkout=True): + def component_exists(self, component_name, component_type, checkout=True, commit=None): """ Check if a module/subworkflow exists in the branch of the repo @@ -312,7 +312,7 @@ def component_exists(self, component_name, component_type, checkout=True): Returns: (bool): Whether the module/subworkflow exists in this branch of the repository """ - return component_name in self.get_avail_components(component_type, checkout=checkout) + return component_name in self.get_avail_components(component_type, checkout=checkout, commit=commit) def get_component_dir(self, component_name, component_type): """ @@ -449,7 +449,7 @@ def get_commit_info(self, sha): return message, date raise LookupError(f"Commit '{sha}' not found in the '{self.remote_url}'") - def get_avail_components(self, component_type, checkout=True): + def get_avail_components(self, component_type, checkout=True, commit=None): """ Gets the names of the modules/subworkflows in the repository. They are detected by checking which directories have a 'main.nf' file @@ -459,6 +459,8 @@ def get_avail_components(self, component_type, checkout=True): """ if checkout: self.checkout_branch() + if commit is not None: + self.checkout(commit) # Get directory if component_type == "modules": directory = self.modules_dir From 7ea4b6e63481a08396212c0ba05468e8f5e32bf6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 8 Feb 2023 09:17:27 +0100 Subject: [PATCH 049/246] update gitpod dockerfile based on https://github.com/nextflow-io/training/blob/master/.github/gitpod.Dockerfile --- nf_core/gitpod/gitpod.Dockerfile | 38 ++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 07beed96ec..fa435c3612 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -2,6 +2,19 @@ FROM gitpod/workspace-base USER root +# Install util tools. +RUN apt-get update --quiet && \ + apt-get install --quiet --yes \ + apt-transport-https \ + apt-utils \ + sudo \ + git \ + less \ + wget \ + curl \ + tree \ + graphviz + # Install Conda RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ @@ -18,23 +31,20 @@ RUN chown -R gitpod:gitpod /opt/conda /usr/src/nf_core # Change user to gitpod USER gitpod - # Install nextflow, nf-core, Mamba, and pytest-workflow -RUN conda update -n base -c defaults conda && \ - conda config --add channels defaults && \ +RUN conda config --add channels defaults && \ conda config --add channels bioconda && \ conda config --add channels conda-forge && \ - conda install \ - openjdk=17.0.3 \ - nextflow=22.10.1 \ - nf-test=0.7.1-0 \ - pytest-workflow=1.6.0 \ - mamba=0.27.0 \ - pip=22.3 \ - black=22.10.0 \ - prettier=2.7.1 \ - -n base && \ - conda clean --all -f -y + conda config --set channel_priority strict && \ + conda install --quiet --yes --name base mamba && \ + mamba install --quiet --yes --name base \ + nextflow \ + nf-core \ + nf-test \ + black \ + prettier \ + pytest-workflow && \ + mamba clean --all -f -y # Install nf-core RUN python -m pip install . From b2f3698631721a09e83d4d315a5b29cec5b683c2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Feb 2023 12:35:25 +0100 Subject: [PATCH 050/246] skip modules whose name has changed --- nf_core/components/update.py | 15 ++++++++++++--- tests/subworkflows/update.py | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index ef645a5a1d..e640827594 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -386,7 +386,7 @@ def get_single_component_info(self, component): ) # Check that the supplied name is an available module/subworkflow - if component and component not in self.modules_repo.get_avail_components(self.component_type): + if component and component not in self.modules_repo.get_avail_components(self.component_type, commit=self.sha): raise LookupError( f"{self.component_type[:-1].title()} '{component}' not found in list of available {self.component_type}." f"Use the command 'nf-core {self.component_type} list remote' to view available software" @@ -879,8 +879,17 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up if m_update in updated: continue original_component_type, original_update_all = self._change_component_type("modules") - self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) - self._reset_component_type(original_component_type, original_update_all) + try: + self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist) + except LookupError as e: + # If the module to be updated is not available, check if there has been a name change + if "not found in list of available" in str(e): + # Skip update, we check for name changes with manage_changes_in_linked_components in line #261 + pass + else: + raise + finally: + self._reset_component_type(original_component_type, original_update_all) def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update): """Check for linked components added or removed in the new subworkflow version""" diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 29e6cb1179..1d437f00a4 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -8,6 +8,7 @@ import nf_core.utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, NF_CORE_MODULES_REMOTE +from nf_core.modules.remove import ModuleRemove from nf_core.modules.update import ModuleUpdate from nf_core.subworkflows.update import SubworkflowUpdate From 39810e1d37149ead2905b7ee3b651fa9cb776266 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Feb 2023 12:47:12 +0100 Subject: [PATCH 051/246] change manage_changes order --- nf_core/components/update.py | 22 +++++++++++----------- tests/subworkflows/update.py | 4 ++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index e640827594..ab4a6eedbb 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -896,17 +896,6 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw if self.component_type == "subworkflows": subworkflow_directory = Path(self.dir, self.component_type, self.modules_repo.repo_path, component) included_modules, included_subworkflows = get_components_to_install(subworkflow_directory) - # If a new module/subworkflow is included in the subworklfow and wasn't included before - for module in included_modules: - if module not in modules_to_update: - log.info(f"Installing newly included module '{module}' for '{component}'") - install_module_object = ComponentInstall(self.dir, "modules", installed_by=component) - install_module_object.install(module, silent=True) - for subworkflow in included_subworkflows: - if subworkflow not in subworkflows_to_update: - log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'") - install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component) - install_subworkflow_object.install(subworkflow, silent=True) # If a module/subworkflow has been removed from the subworkflow for module in modules_to_update: if module not in included_modules: @@ -918,6 +907,17 @@ def manage_changes_in_linked_components(self, component, modules_to_update, subw log.info(f"Removing subworkflow '{subworkflow}' which is not included in '{component}' anymore.") remove_subworkflow_object = ComponentRemove("subworkflows", self.dir) remove_subworkflow_object.remove(subworkflow, removed_by=component) + # If a new module/subworkflow is included in the subworklfow and wasn't included before + for module in included_modules: + if module not in modules_to_update: + log.info(f"Installing newly included module '{module}' for '{component}'") + install_module_object = ComponentInstall(self.dir, "modules", installed_by=component) + install_module_object.install(module, silent=True) + for subworkflow in included_subworkflows: + if subworkflow not in subworkflows_to_update: + log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'") + install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component) + install_subworkflow_object.install(subworkflow, silent=True) def _change_component_type(self, new_component_type): original_component_type = self.component_type diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 1d437f00a4..698086e186 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -322,6 +322,10 @@ def test_update_change_of_included_modules(self): # Check that tabix/tabix is there assert "tabix/tabix" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "tabix/tabix").is_dir() + # Check that ensemblevep is not there but instead we have ensemblevep/vep (due to a file re-naming) + assert "ensemblvep" not in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] + assert "ensemblvep/vep" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME] + assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "ensemblvep/vep").is_dir() def cmp_component(dir1, dir2): From 297b2c8fd6398b0e1c16981616392308b2e4380a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Feb 2023 12:54:11 +0100 Subject: [PATCH 052/246] update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5c5701120..3b3d15f18a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] - Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] -- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) +- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ([#2170](https://github.com/nf-core/tools/pull/2170)) ### Linting @@ -16,6 +16,8 @@ ### Subworkflows +- Fixing problem when a module included in a subworkflow had a name change from TOOL to TOOL/SUBTOOL ([#2177](https://github.com/nf-core/tools/pull/2177)) + ### General ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] From 53af36c2cccd3c87c9afc6de3af4a708890fa6d4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 9 Feb 2023 11:09:04 +0100 Subject: [PATCH 053/246] add 'subworkflows/' to pytest tag --- nf_core/components/components_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index eadb5ba0cb..c2b9abf569 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -180,7 +180,10 @@ def _run_pytests(self): console.rule(self.component_name, style="black") # Set pytest arguments - command_args = ["--tag", f"{self.component_name}", "--symlink", "--keep-workflow-wd", "--git-aware"] + tag = self.component_name + if self.component_type == "subworkflows": + tag = "subworkflows/" + tag + command_args = ["--tag", f"{tag}", "--symlink", "--keep-workflow-wd", "--git-aware"] command_args += self.pytest_args # Run pytest From 2250919643065125f406c69ca28937df0b5b3c0d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 9 Feb 2023 11:11:18 +0100 Subject: [PATCH 054/246] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b3d15f18a..ac67ab7bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ ### Subworkflows - Fixing problem when a module included in a subworkflow had a name change from TOOL to TOOL/SUBTOOL ([#2177](https://github.com/nf-core/tools/pull/2177)) +- Fix `nf-core subworkflows test` not running subworkflow tests ([#2181](https://github.com/nf-core/tools/pull/2181)) ### General From cab29c86aa70a838c18f3f5a1f05b62cf7328b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 9 Feb 2023 11:40:26 +0100 Subject: [PATCH 055/246] Update nf_core/components/update.py --- nf_core/components/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index ab4a6eedbb..5f8a2129c5 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -884,7 +884,7 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up except LookupError as e: # If the module to be updated is not available, check if there has been a name change if "not found in list of available" in str(e): - # Skip update, we check for name changes with manage_changes_in_linked_components in line #261 + # Skip update, we check for name changes with manage_changes_in_linked_components pass else: raise From 4604e408a91bd3f37f2d3b480d0544e5ec9c18de Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 9 Feb 2023 11:57:11 +0100 Subject: [PATCH 056/246] add include statement to info command for installed components fix incorrect handling of locally installed modules --- nf_core/components/info.py | 40 ++++++++++++++++++++++++++++++++++- nf_core/components/install.py | 5 ++--- tests/modules/info.py | 2 +- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/nf_core/components/info.py b/nf_core/components/info.py index b89baaf99e..69b9e19b87 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -8,6 +8,7 @@ from rich.console import Group from rich.markdown import Markdown from rich.panel import Panel +from rich.syntax import Syntax from rich.table import Table from rich.text import Text @@ -128,6 +129,18 @@ def init_mod_name(self, component): choices=components, style=nf_core.utils.nfcore_question_style, ).unsafe_ask() + else: + if self.repo_type == "pipeline": + # check if the module is locally installed + self.local_path = self.modules_json.get_all_components(self.component_type).get( + self.modules_repo.remote_url, {} + ) + for directory, comp in self.local_path: + if comp == component: + self.local_path = Path(self.component_type, directory, component) + break + if self.local_path: + self.local = True return component @@ -283,7 +296,7 @@ def generate_component_info_help(self): renderables.append(outputs_table) # Installation command - if self.remote_location: + if self.remote_location and not self.local: cmd_base = f"nf-core {self.component_type}" if self.remote_location != NF_CORE_MODULES_REMOTE: cmd_base = f"nf-core {self.component_type} --git-remote {self.remote_location}" @@ -291,4 +304,29 @@ def generate_component_info_help(self): Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.component}\n") ) + # Print include statement + if self.local_path: + install_folder = Path(self.dir, self.component_type, self.modules_repo.repo_path) + component_name = "_".join(self.component.upper().split("/")) + renderables.append( + Text.from_markup(f"\n [blue]Use the following statement to include this {self.component_type[:-1]}:") + ) + renderables.append( + Syntax( + f"include {{ {component_name} }} from '../{Path(install_folder, self.component).relative_to(self.dir)}/main'", + "groovy", + theme="ansi_dark", + padding=1, + ) + ) + if self.component_type == "subworkflows": + subworkflow_config = Path(install_folder, self.component, "nextflow.config").relative_to(self.dir) + if os.path.isfile(subworkflow_config): + renderables.append( + Text.from_markup("\n [blue]Add the following config statement to use this subworkflow:") + ) + renderables.append( + Syntax(f"includeConfig '{subworkflow_config}'", "groovy", theme="ansi_dark", padding=1) + ) + return Group(*renderables) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index f9a16f73cf..90cdf7b6a9 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -1,6 +1,5 @@ import logging import os -import re from pathlib import Path import questionary @@ -74,10 +73,10 @@ def install(self, component, silent=False): ) # Set the install folder based on the repository name - install_folder = os.path.join(self.dir, self.component_type, self.modules_repo.repo_path) + install_folder = Path(self.dir, self.component_type, self.modules_repo.repo_path) # Compute the component directory - component_dir = os.path.join(install_folder, component) + component_dir = Path(install_folder, component) # Check that the component is not already installed component_not_installed = self.check_component_installed( diff --git a/tests/modules/info.py b/tests/modules/info.py index 6c5b1063f1..2dbd48b240 100644 --- a/tests/modules/info.py +++ b/tests/modules/info.py @@ -38,7 +38,6 @@ def test_modules_info_local(self): """Test getting info about a locally installed module""" self.mods_install.install("trimgalore") mods_info = nf_core.modules.ModuleInfo(self.pipeline_dir, "trimgalore") - mods_info.local = True mods_info_output = mods_info.get_component_info() console = Console(record=True) console.print(mods_info_output) @@ -47,6 +46,7 @@ def test_modules_info_local(self): assert "Module: trimgalore" in output assert "Inputs" in output assert "Outputs" in output + assert "Location" in output def test_modules_info_in_modules_repo(self): From 2d04d7294ad997a9787cd168be8850f7ea4ac4ea Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 9 Feb 2023 12:09:35 +0100 Subject: [PATCH 057/246] update changelog, use same way to generate self.local_path as in the rest of the code --- CHANGELOG.md | 2 ++ nf_core/components/info.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5dd860b3e..16900b7a59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ ### General +- `nf-core modules/subworkflows info` now prints the include statement for the module/subworkflow ([#2182](https://github.com/nf-core/tools/pull/2182)). + ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] ### Template diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 69b9e19b87..365e017ab1 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -132,12 +132,13 @@ def init_mod_name(self, component): else: if self.repo_type == "pipeline": # check if the module is locally installed - self.local_path = self.modules_json.get_all_components(self.component_type).get( + local_paths = self.modules_json.get_all_components(self.component_type).get( self.modules_repo.remote_url, {} ) - for directory, comp in self.local_path: + for directory, comp in local_paths: if comp == component: - self.local_path = Path(self.component_type, directory, component) + component_base_path = Path(self.dir, self.component_type) + self.local_path = Path(component_base_path, directory, self.component) break if self.local_path: self.local = True From 6ce1b3c72b146ebf296493facde2ac1cbf0df312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 9 Feb 2023 12:54:03 +0100 Subject: [PATCH 058/246] Update nf_core/components/info.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/components/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 365e017ab1..e4d8038b87 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -138,7 +138,7 @@ def init_mod_name(self, component): for directory, comp in local_paths: if comp == component: component_base_path = Path(self.dir, self.component_type) - self.local_path = Path(component_base_path, directory, self.component) + self.local_path = Path(component_base_path, directory, component) break if self.local_path: self.local = True From 11253e759ca54841957eb05ec35ecdf8e39f954a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 16 Feb 2023 14:27:41 +0100 Subject: [PATCH 059/246] add stale GHA in tools --- .github/workflows/stale.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000000..8e121b39fa --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,24 @@ +name: "Close stale issues and stale PRs" +on: + schedule: + - cron: "30 1 * * 7" # Once a week + +jobs: + stale: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v7 + with: + stale-issue-message: "This issue is stale because it has been inactive for more than 30 days. More information is required. Remove stale label or comment or this will be closed in 20 days." + stale-pr-message: "This PR is stale because it has been open more than 45 days with no activity. Remove stale label or comment if it is still useful. In any case a PR will be automatically closed." + close-issue-message: "This issue was closed because it has been stalled for 20 days with no activity." + days-before-stale: 30 + days-before-close: 20 + days-before-pr-close: -1 + any-of-labels: "awaiting-changes,awaiting-feedback" + exempt-issue-labels: "WIP" + exempt-pr-labels: "WIP" + repo-token: ${{ secrets.GITHUB_TOKEN }} From 8cd4990f6f80fd93fe46836cb5edb384ac329008 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 16 Feb 2023 14:30:24 +0100 Subject: [PATCH 060/246] add stale action to pipeline template, not mandatory --- .../.github/workflows/stale.yml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 nf_core/pipeline-template/.github/workflows/stale.yml diff --git a/nf_core/pipeline-template/.github/workflows/stale.yml b/nf_core/pipeline-template/.github/workflows/stale.yml new file mode 100644 index 0000000000..8e121b39fa --- /dev/null +++ b/nf_core/pipeline-template/.github/workflows/stale.yml @@ -0,0 +1,24 @@ +name: "Close stale issues and stale PRs" +on: + schedule: + - cron: "30 1 * * 7" # Once a week + +jobs: + stale: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v7 + with: + stale-issue-message: "This issue is stale because it has been inactive for more than 30 days. More information is required. Remove stale label or comment or this will be closed in 20 days." + stale-pr-message: "This PR is stale because it has been open more than 45 days with no activity. Remove stale label or comment if it is still useful. In any case a PR will be automatically closed." + close-issue-message: "This issue was closed because it has been stalled for 20 days with no activity." + days-before-stale: 30 + days-before-close: 20 + days-before-pr-close: -1 + any-of-labels: "awaiting-changes,awaiting-feedback" + exempt-issue-labels: "WIP" + exempt-pr-labels: "WIP" + repo-token: ${{ secrets.GITHUB_TOKEN }} From 501931231f8c0c6e419d26593262daa273d4773e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 16 Feb 2023 14:35:58 +0100 Subject: [PATCH 061/246] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67e6ada896..f36e6b84ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] - Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] - Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ([#2170](https://github.com/nf-core/tools/pull/2170)) +- Add a stale GHA wich stale + close issues and stale PRs with specific labels ([#2183](https://github.com/nf-core/tools/pull/2183)) ### Linting @@ -22,6 +23,7 @@ ### General - `nf-core modules/subworkflows info` now prints the include statement for the module/subworkflow ([#2182](https://github.com/nf-core/tools/pull/2182)). +- Add a stale GHA wich stale + close issues and stale PRs with specific labels ([#2183](https://github.com/nf-core/tools/pull/2183)) ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] From a1645dacd369c04857051116aa4fc43a5b795319 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 17 Feb 2023 13:13:37 +0100 Subject: [PATCH 062/246] try fixing pytest GHA --- .github/workflows/pytest-frozen-ubuntu-20.04.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytest-frozen-ubuntu-20.04.yml b/.github/workflows/pytest-frozen-ubuntu-20.04.yml index 6d49145ed7..b015376633 100644 --- a/.github/workflows/pytest-frozen-ubuntu-20.04.yml +++ b/.github/workflows/pytest-frozen-ubuntu-20.04.yml @@ -33,6 +33,7 @@ jobs: - name: Downgrade git to the Ubuntu official repository's version run: | + sudo apt update sudo apt remove git git-man sudo add-apt-repository --remove ppa:git-core/ppa sudo apt install git From 5743140920d7f5de7c6244a04554770b8ff58da6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 17 Feb 2023 16:00:19 +0100 Subject: [PATCH 063/246] update required rich version --- CHANGELOG.md | 1 + requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67e6ada896..3608e20e48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ ### General - `nf-core modules/subworkflows info` now prints the include statement for the module/subworkflow ([#2182](https://github.com/nf-core/tools/pull/2182)). +- update minimum version of rich to 13.3.1 ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] diff --git a/requirements.txt b/requirements.txt index b3d1f251bf..9dda69b407 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,6 @@ questionary>=1.8.0 refgenie requests requests_cache -rich-click>=1.0.0 -rich>=10.7.0 +rich-click>=1.6.1 +rich>=13.3.1 tabulate From 62a26ac0dcfcd56b371afb7806fe38c62cc08072 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 17 Feb 2023 16:16:13 +0100 Subject: [PATCH 064/246] add PR to changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3608e20e48..ff5014a3dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ ### General - `nf-core modules/subworkflows info` now prints the include statement for the module/subworkflow ([#2182](https://github.com/nf-core/tools/pull/2182)). -- update minimum version of rich to 13.3.1 +- update minimum version of rich to 13.3.1 ([#2185](https://github.com/nf-core/tools/pull/2185)) ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] From 7915b47b4672b8bb27d0bd996f7f2b5eb2b363a8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 22 Feb 2023 09:38:57 +0100 Subject: [PATCH 065/246] add repository_type to .nf-core.yml --- README.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 60dc102996..5c47e815d9 100644 --- a/README.md +++ b/README.md @@ -1032,6 +1032,8 @@ To list subworkflows installed in a local pipeline directory you can use `nf-cor @@ -1056,6 +1058,8 @@ A subworkflow installed this way will be installed to the `./subworkflows/nf-cor ![`nf-core subworkflows install bam_rseqc`](docs/images/nf-core-subworkflows-install.svg) @@ -1075,6 +1079,8 @@ You can update subworkflows installed from a remote repository in your pipeline ![`nf-core subworkflows update --all --no-preview`](docs/images/nf-core-subworkflows-update.svg) @@ -1134,6 +1140,8 @@ To delete a subworkflow from your pipeline, run `nf-core subworkflows remove`. ![`nf-core subworkflows remove bam_rseqc`](docs/images/nf-core-subworkflows-remove.svg) @@ -1162,10 +1170,10 @@ The `nf-core subworkflows create` command will prompt you with the relevant ques -![`cd modules && nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --label process_low --meta --force`](docs/images/nf-core-subworkflows-create.svg) +![`cd modules && nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force`](docs/images/nf-core-subworkflows-create.svg) ### Create a subworkflow test config file @@ -1177,6 +1185,8 @@ After you have written a minimal Nextflow script to test your subworkflow in `/t working_dir: tmp/subworkflows extra_env: PROFILE: 'conda' +before_command: > + echo "repository_type: modules" >> .nf-core.yml --> ![`nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force`](docs/images/nf-core-subworkflows-create-test.svg) @@ -1192,6 +1202,8 @@ working_dir: tmp/subworkflows timeout: 30 extra_env: PROFILE: 'conda' +before_command: > + echo "repository_type: pipeline" >> .nf-core.yml --> ![`nf-core subworkflows test bam_rseqc --no-prompts`](docs/images/nf-core-subworkflows-test.svg) From c569daaed17d9a1e67a3887a92d45c8b4965904b Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 22 Feb 2023 09:44:40 +0100 Subject: [PATCH 066/246] add repository_type to .nf-core.yml --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5c47e815d9..fa4fc96d0e 100644 --- a/README.md +++ b/README.md @@ -1047,6 +1047,8 @@ This shows documentation about the subworkflow on the command line, similar to w ![`nf-core subworkflows info bam_rseqc`](docs/images/nf-core-subworkflows-info.svg) From 6d68f11d47a4603c5ca484d43154f068981008b2 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 22 Feb 2023 13:01:06 +0000 Subject: [PATCH 067/246] Dump template yml to file in pipeline repo after creation --- nf_core/create.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 74c9df1b87..1c6a88bc50 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -56,7 +56,7 @@ def __init__( plain=False, default_branch=None, ): - self.template_params, skip_paths_keys = self.create_param_dict( + self.template_params, skip_paths_keys, self.template_yaml = self.create_param_dict( name, description, author, version, template_yaml_path, plain ) @@ -178,7 +178,7 @@ def create_param_dict(self, name, description, author, version, template_yaml_pa if not re.match(r"^[a-z]+$", param_dict["short_name"]): raise UserWarning("[red]Invalid workflow name: must be lowercase without punctuation.") - return param_dict, skip_paths + return param_dict, skip_paths, template_yaml def customize_template(self, template_areas): """Customizes the template parameters. @@ -348,6 +348,10 @@ def render_template(self): # Update the .nf-core.yml with linting configurations self.fix_linting() + log.debug("Dumping pipeline template yml to file") + with open(self.outdir / "pipeline_template.yml", "w") as fh: + yaml.safe_dump(self.template_yaml, fh) + def update_nextflow_schema(self): """ Removes unused parameters from the nextflow schema. From 834c0ecb6d5a6d0c160ee2aed53c0bd21d3fc2c1 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 22 Feb 2023 13:04:20 +0000 Subject: [PATCH 068/246] Make the dump conditional on template yaml having content --- nf_core/create.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index 1c6a88bc50..0e9edf59c9 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -349,8 +349,9 @@ def render_template(self): self.fix_linting() log.debug("Dumping pipeline template yml to file") - with open(self.outdir / "pipeline_template.yml", "w") as fh: - yaml.safe_dump(self.template_yaml, fh) + if self.template_yaml: + with open(self.outdir / "pipeline_template.yml", "w") as fh: + yaml.safe_dump(self.template_yaml, fh) def update_nextflow_schema(self): """ From 12197f09676d698f041486a0d7dc37698b3c16a9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 22 Feb 2023 16:02:30 +0100 Subject: [PATCH 069/246] add schema based validation for meta.ymls to linting --- nf_core/modules/lint/meta_yml.py | 46 +++++++++++++++++++------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index d6ec296999..9e78670bcc 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -1,5 +1,7 @@ +import json from pathlib import Path +import jsonschema.validators import yaml from nf_core.modules.modules_differ import ModulesDiffer @@ -10,17 +12,15 @@ def meta_yml(module_lint_object, module): Lint a ``meta.yml`` file The lint test checks that the module has - a ``meta.yml`` file and that it contains - the required keys: ``name``, input`` and - ``output``. + a ``meta.yml`` file and that it follows the + JSON schema defined in the ``yaml-schema.json`` + file in the nf-core/modules repository. In addition it checks that the module name and module input is consistent between the ``meta.yml`` and the ``main.nf``. """ - required_keys = ["name", "output"] - required_keys_lists = ["input", "output"] # Check if we have a patch file, get original file in that case meta_yaml = None if module.is_patched: @@ -42,21 +42,31 @@ def meta_yml(module_lint_object, module): module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) return - # Confirm that all required keys are given - contains_required_keys = True - all_list_children = True - for rk in required_keys: - if rk not in meta_yaml.keys(): - module.failed.append(("meta_required_keys", f"`{rk}` not specified in YAML", module.meta_yml)) - contains_required_keys = False - elif rk in meta_yaml.keys() and not isinstance(meta_yaml[rk], list) and rk in required_keys_lists: - module.failed.append(("meta_required_keys", f"`{rk}` is not a list", module.meta_yml)) - all_list_children = False - if contains_required_keys: - module.passed.append(("meta_required_keys", "`meta.yml` contains all required keys", module.meta_yml)) + # Confirm that the meta.yml file is valid according to the JSON schema + valid_meta_yml = True + try: + with open(Path(module_lint_object.modules_repo.local_repo_dir, "yaml-schema.json"), "r") as fh: + schema = json.load(fh) + jsonschema.validators.validate(instance=meta_yaml, schema=schema) + module.passed.append(("meta_yml_valid", "Module `meta.yml` is valid", module.meta_yml)) + except jsonschema.exceptions.ValidationError as e: + valid_meta_yml = False + hint = "" + if len(e.path) > 0: + hint = f"\nCheck the entry for `{e.path[0]}`." + if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: + hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + module.failed.append( + ( + "meta_yml_valid", + f"The `meta.yml` of the module {module.module_name} is not valid: {e.message}.{hint}", + module.meta_yml, + ) + ) + return # Confirm that all input and output channels are specified - if contains_required_keys and all_list_children: + if valid_meta_yml: if "input" in meta_yaml: meta_input = [list(x.keys())[0] for x in meta_yaml["input"]] for input in module.inputs: From 6c0bcf59f00e9fbdb56108c51d3268a1ed6fbbda Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 24 Feb 2023 20:27:46 +0100 Subject: [PATCH 070/246] Remove problematic sniffer code resulting in no header found even when it exists See https://github.com/nf-core/funcscan/pull/227 --- nf_core/pipeline-template/bin/check_samplesheet.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nf_core/pipeline-template/bin/check_samplesheet.py b/nf_core/pipeline-template/bin/check_samplesheet.py index 11b155723a..4a758fe003 100755 --- a/nf_core/pipeline-template/bin/check_samplesheet.py +++ b/nf_core/pipeline-template/bin/check_samplesheet.py @@ -158,9 +158,6 @@ def sniff_format(handle): peek = read_head(handle) handle.seek(0) sniffer = csv.Sniffer() - if not sniffer.has_header(peek): - logger.critical("The given sample sheet does not appear to contain a header.") - sys.exit(1) dialect = sniffer.sniff(peek) return dialect From c6e9878809fe7ab4abf72f0dbfee0734707904e1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 24 Feb 2023 20:31:28 +0100 Subject: [PATCH 071/246] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67e6ada896..b9fe88e980 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] - Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] - Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ([#2170](https://github.com/nf-core/tools/pull/2170)) +- Remove problematic sniffer code in samplesheet_check.py that could give false positive 'missing header' errors ([https://github.com/nf-core/tools/pull/2194]) [Contributed by @jfy133] ### Linting From 6de786c53cf1d7c751de9c109cad6e76e101fd84 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 24 Feb 2023 20:31:41 +0100 Subject: [PATCH 072/246] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9fe88e980..2f7204d37d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) [Contributed by @jfy133] - Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] - Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ([#2170](https://github.com/nf-core/tools/pull/2170)) -- Remove problematic sniffer code in samplesheet_check.py that could give false positive 'missing header' errors ([https://github.com/nf-core/tools/pull/2194]) [Contributed by @jfy133] +- Remove problematic sniffer code in samplesheet_check.py that could give false positive 'missing header' errors ([https://github.com/nf-core/tools/pull/2194]) [Contributed by @Midnighter, @jfy133] ### Linting From 7a27da2b5858f2b11cc21e729cc642a877d44f8e Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Feb 2023 08:34:40 +0100 Subject: [PATCH 073/246] add schema definition to template --- nf_core/module-template/modules/meta.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 9b42bb3bdf..5def7dd301 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,3 +1,5 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/yaml-schema.json name: "{{ component_name_underscore }}" {% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords From f7da0ccbf28e9ffc0a7f61db91989e7c151d9300 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Feb 2023 09:35:00 +0100 Subject: [PATCH 074/246] add gitpod nextflow version and use dev container --- .github/RELEASE_CHECKLIST.md | 4 ++-- .gitpod.yml | 2 +- nf_core/gitpod/gitpod.Dockerfile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/RELEASE_CHECKLIST.md b/.github/RELEASE_CHECKLIST.md index f2d2591f16..f7cab98c55 100644 --- a/.github/RELEASE_CHECKLIST.md +++ b/.github/RELEASE_CHECKLIST.md @@ -3,7 +3,7 @@ 1. Check issue milestones to see outstanding issues to resolve if possible or transfer to the milestones for the next release e.g. [`v1.9`](https://github.com/nf-core/tools/issues?q=is%3Aopen+is%3Aissue+milestone%3A1.9) 2. Most importantly, pick an undeniably outstanding [name](http://www.codenamegenerator.com/) for the release where _Prefix_ = _Metal_ and _Dictionary_ = _Animal_. 3. Check the [pipeline health page](https://nf-co.re/pipeline_health) to make sure that all repos look sane (missing `TEMPLATE` branches etc) -4. Create a PR to `dev` to bump the version in `CHANGELOG.md` and `setup.py`. +4. Create a PR to `dev` to bump the version in `CHANGELOG.md` and `setup.py` and change the gitpod container to `nfcore/gitpod:latest`. 5. Make sure all CI tests are passing! 6. Create a PR from `dev` to `master` 7. Make sure all CI tests are passing again (additional tests are run on PRs to `master`) @@ -19,4 +19,4 @@ 1. Check the automated template synchronisation has been triggered properly. This should automatically open PRs directly to individual pipeline repos with the appropriate changes to update the pipeline template. 2. Check that the automatic `PyPi` deployment has worked: [pypi.org/project/nf-core](https://pypi.org/project/nf-core/) 3. Check `BioConda` has an automated PR to bump the version, and merge. eg. [bioconda/bioconda-recipes #20065](https://github.com/bioconda/bioconda-recipes/pull/20065) -4. Create a tools PR to `dev` to bump back to the next development version in `CHANGELOG.md` and `setup.py` +4. Create a tools PR to `dev` to bump back to the next development version in `CHANGELOG.md` and `setup.py` and change the gitpod container to `nfcore/gitpod:dev`. diff --git a/.gitpod.yml b/.gitpod.yml index 263fcc41db..a93e660516 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index fa435c3612..417208a20d 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -38,7 +38,7 @@ RUN conda config --add channels defaults && \ conda config --set channel_priority strict && \ conda install --quiet --yes --name base mamba && \ mamba install --quiet --yes --name base \ - nextflow \ + nextflow=22.10.1 \ nf-core \ nf-test \ black \ From 953f47482295ca66ab9e03982441d2f04bbe9cd9 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Feb 2023 09:39:27 +0100 Subject: [PATCH 075/246] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67e6ada896..7ca913078b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ ### General - `nf-core modules/subworkflows info` now prints the include statement for the module/subworkflow ([#2182](https://github.com/nf-core/tools/pull/2182)). +- Add the Nextflow version to Gitpod container matching the minimal Nextflow version for nf-core (according to `nextflow.config`) ([#2196](https://github.com/nf-core/tools/pull/2196)) +- Use `nfcore/gitpod:dev` container in the dev branch ([#2196](https://github.com/nf-core/tools/pull/2196)) ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] From 9f349f045c9eadcadb89a6070d7d38d4a17953e5 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Sun, 12 Mar 2023 18:00:35 +0100 Subject: [PATCH 076/246] refactor: remove the mulled command and all related code --- nf_core/__main__.py | 45 +---------------------- nf_core/modules/__init__.py | 1 - nf_core/modules/mulled.py | 72 ------------------------------------- requirements.txt | 1 - tests/test_mulled.py | 65 --------------------------------- 5 files changed, 1 insertion(+), 183 deletions(-) delete mode 100644 nf_core/modules/mulled.py delete mode 100644 tests/test_mulled.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 841cb9f7e7..4df0dd2d93 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -52,7 +52,7 @@ }, { "name": "Developing new modules", - "commands": ["create", "create-test-yml", "lint", "bump-versions", "mulled", "test"], + "commands": ["create", "create-test-yml", "lint", "bump-versions", "test"], }, ], "nf-core subworkflows": [ @@ -875,49 +875,6 @@ def bump_versions(ctx, tool, dir, all, show_all): sys.exit(1) -# nf-core modules mulled -@modules.command() -@click.argument("specifications", required=True, nargs=-1, metavar=" <...>") -@click.option( - "--build-number", - type=int, - default=0, - show_default=True, - metavar="", - help="The build number for this image. This is an incremental value that starts at zero.", -) -def mulled(specifications, build_number): - """ - Generate the name of a BioContainers mulled image version 2. - - When you know the specific dependencies and their versions of a multi-tool container image and you need the name of - that image, this command can generate it for you. - - """ - from nf_core.modules.mulled import MulledImageNameGenerator - - try: - image_name = MulledImageNameGenerator.generate_image_name( - MulledImageNameGenerator.parse_targets(specifications), build_number=build_number - ) - except ValueError as e: - log.error(e) - sys.exit(1) - if not MulledImageNameGenerator.image_exists(image_name): - log.error("The generated multi-tool container image name does not seem to exist yet.") - log.info( - "Please double check that your provided combination of tools and versions exists in the file: " - "[link=https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv]BioContainers/multi-package-containers 'combinations/hash.tsv'[/link]" - ) - log.info( - "If it does not, please add your desired combination as detailed at: " - "https://github.com/BioContainers/multi-package-containers" - ) - sys.exit(1) - log.info("Mulled container hash:") - stdout.print(image_name) - - # nf-core modules test @modules.command("test") @click.pass_context diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index 060b39124b..47af637d02 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -8,7 +8,6 @@ from .modules_repo import ModulesRepo from .modules_test import ModulesTest from .modules_utils import ModuleException -from .mulled import MulledImageNameGenerator from .patch import ModulePatch from .remove import ModuleRemove from .test_yml_builder import ModulesTestYmlBuilder diff --git a/nf_core/modules/mulled.py b/nf_core/modules/mulled.py deleted file mode 100644 index fc1d1a3555..0000000000 --- a/nf_core/modules/mulled.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Generate the name of a BioContainers mulled image version 2.""" - - -import logging -import re -from typing import Iterable, List, Tuple - -import requests -from galaxy.tool_util.deps.mulled.util import build_target, v2_image_name -from packaging.version import InvalidVersion, Version - -log = logging.getLogger(__name__) - - -class MulledImageNameGenerator: - """ - Define a service class for generating BioContainers version 2 mulled image names. - - Adapted from https://gist.github.com/natefoo/19cefeedd1942c30f9d88027a61b3f83. - - """ - - _split_pattern = re.compile(r"==?") - - @classmethod - def parse_targets(cls, specifications: Iterable[str]) -> List[Tuple[str, str]]: - """ - Parse tool, version pairs from specification strings. - - Args: - specifications: An iterable of strings that contain tools and their versions. - - """ - result = [] - for spec in specifications: - try: - tool, version = cls._split_pattern.split(spec, maxsplit=1) - except ValueError: - raise ValueError( - f"The specification {spec} does not have the expected format or ." - ) from None - try: - Version(version) - except InvalidVersion: - raise ValueError(f"Not a PEP440 version spec: '{version}' in '{spec}'") from None - result.append((tool.strip(), version.strip())) - return result - - @classmethod - def generate_image_name(cls, targets: Iterable[Tuple[str, str]], build_number: int = 0) -> str: - """ - Generate the name of a BioContainers mulled image version 2. - - Args: - targets: One or more tool, version pairs of the multi-tool container image. - build_number: The build number for this image. This is an incremental value that starts at zero. - - """ - return v2_image_name([build_target(name, version) for name, version in targets], image_build=str(build_number)) - - @classmethod - def image_exists(cls, image_name: str) -> bool: - """Check whether a given BioContainers image name exists via a call to the quay.io API.""" - quay_url = f"https://quay.io/biocontainers/{image_name}/" - response = requests.get(quay_url, allow_redirects=True) - log.debug(f"Got response code '{response.status_code}' for URL {quay_url}") - if response.status_code == 200: - log.info(f"Found [link={quay_url}]docker image[/link] on quay.io! :sparkles:") - return True - else: - log.error(f"Was not able to find [link={quay_url}]docker image[/link] on quay.io") - return False diff --git a/requirements.txt b/requirements.txt index b3d1f251bf..8b85e605ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ click filetype -galaxy-tool-util GitPython jinja2 jsonschema>=3.0 diff --git a/tests/test_mulled.py b/tests/test_mulled.py deleted file mode 100644 index cf0a4fcfc0..0000000000 --- a/tests/test_mulled.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Test the mulled BioContainers image name generation.""" - -import pytest - -from nf_core.modules import MulledImageNameGenerator - - -@pytest.mark.parametrize( - "specs, expected", - [ - (["foo==0.1.2", "bar==1.1"], [("foo", "0.1.2"), ("bar", "1.1")]), - (["foo=0.1.2", "bar=1.1"], [("foo", "0.1.2"), ("bar", "1.1")]), - ], -) -def test_target_parsing(specs, expected): - """Test that valid specifications are correctly parsed into tool, version pairs.""" - assert MulledImageNameGenerator.parse_targets(specs) == expected - - -@pytest.mark.parametrize( - "specs", - [ - ["foo<0.1.2", "bar==1.1"], - ["foo=0.1.2", "bar>1.1"], - ], -) -def test_wrong_specification(specs): - """Test that unexpected version constraints fail.""" - with pytest.raises(ValueError, match="expected format"): - MulledImageNameGenerator.parse_targets(specs) - - -@pytest.mark.parametrize( - "specs", - [ - ["foo==0a.1.2", "bar==1.1"], - ["foo==0.1.2", "bar==1.b1b"], - ], -) -def test_noncompliant_version(specs): - """Test that version string that do not comply with PEP440 fail.""" - with pytest.raises(ValueError, match="PEP440"): - MulledImageNameGenerator.parse_targets(specs) - - -@pytest.mark.parametrize( - "specs, expected", - [ - ( - [("chromap", "0.2.1"), ("samtools", "1.15")], - "mulled-v2-1f09f39f20b1c4ee36581dc81cc323c70e661633:bd74d08a359024829a7aec1638a28607bbcd8a58-0", - ), - ( - [("pysam", "0.16.0.1"), ("biopython", "1.78")], - "mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0", - ), - ( - [("samclip", "0.4.0"), ("samtools", "1.15")], - "mulled-v2-d057255d4027721f3ab57f6a599a2ae81cb3cbe3:13051b049b6ae536d76031ba94a0b8e78e364815-0", - ), - ], -) -def test_generate_image_name(specs, expected): - """Test that a known image name is generated from given targets.""" - assert MulledImageNameGenerator.generate_image_name(specs) == expected From 3a4b67dd890dd4445b75c63f1863edd3b55fbfa9 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Sun, 12 Mar 2023 18:16:21 +0100 Subject: [PATCH 077/246] docs: remove mulled command from readme --- README.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/README.md b/README.md index fa4fc96d0e..284f6d69c6 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,6 @@ A python package with helper tools for the nf-core community. - [`modules lint` - Check a module against nf-core guidelines](#check-a-module-against-nf-core-guidelines) - [`modules test` - Run the tests for a module](#run-the-tests-for-a-module-using-pytest) - [`modules bump-versions` - Bump software versions of modules](#bump-bioconda-and-container-versions-of-modules-in) - - [`modules mulled` - Generate the name for a multi-tool container image](#generate-the-name-for-a-multi-tool-container-image) - [`nf-core subworkflows` - commands for dealing with subworkflows](#subworkflows) - [`subworkflows list` - List available subworkflows](#list-subworkflows) @@ -961,17 +960,6 @@ bump-versions: star/align: "2.6.1d" ``` -### Generate the name for a multi-tool container image - -When you want to use an image of a multi-tool container and you know the specific dependencies and their versions of that container, for example, by looking them up in the [BioContainers hash.tsv](https://github.com/BioContainers/multi-package-containers/blob/master/combinations/hash.tsv), you can use the `nf-core modules mulled` helper tool. This tool generates the name of a BioContainers mulled image. - - - -![`nf-core modules mulled pysam==0.16.0.1 biopython==1.78`](docs/images/nf-core-modules-mulled.svg) - ## Subworkflows After the launch of nf-core modules, we can provide now also nf-core subworkflows to fully utilize the power of DSL2 modularization. From 1bde22236d89b4c40a848b97447f9bd370f0833e Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Sun, 12 Mar 2023 18:19:16 +0100 Subject: [PATCH 078/246] docs: describe removal in changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34276b0321..386f097f97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ### Modules - Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175) & [#2177](https://github.com/nf-core/tools/pull/2177)) +- Removed the `nf-core modules mulled` command and all its code dependencies ([2199](https://github.com/nf-core/tools/pull/2199)). ### Subworkflows From 3364e7ede13161145467d23a39ce4a68c4dbd0c0 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 14 Mar 2023 15:37:25 +0100 Subject: [PATCH 079/246] Condense main readme --- nf_core/pipeline-template/docs/usage.md | 101 ++---------------------- 1 file changed, 8 insertions(+), 93 deletions(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 9a171f5aad..416cdf0900 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -145,104 +145,19 @@ Specify the path to a specific config file (this is a core Nextflow command). Se Whilst the default requirements set within the pipeline will hopefully work for most people and with most input data, you may find that you want to customise the compute resources that the pipeline requests. Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with any of the error codes specified [here](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L18) it will automatically be resubmitted with higher requests (2 x original, then 3 x original). If it still fails after the third attempt then the pipeline execution is stopped. -For example, if the nf-core/rnaseq pipeline is failing after multiple re-submissions of the `STAR_ALIGN` process due to an exit code of `137` this would indicate that there is an out of memory issue: +To optimise resource requests away from the pipeline 'base' defaults, please see the [max resources](https://nf-co.re/docs/usage/configuration#max-resources) and [tuning workflow resources](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) section of the nf-core website. -```console -[62/149eb0] NOTE: Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) -- Execution is retried (1) -Error executing process > 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)' - -Caused by: - Process `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN (WT_REP1)` terminated with an error exit status (137) - -Command executed: - STAR \ - --genomeDir star \ - --readFilesIn WT_REP1_trimmed.fq.gz \ - --runThreadN 2 \ - --outFileNamePrefix WT_REP1. \ - - -Command exit status: - 137 - -Command output: - (empty) - -Command error: - .command.sh: line 9: 30 Killed STAR --genomeDir star --readFilesIn WT_REP1_trimmed.fq.gz --runThreadN 2 --outFileNamePrefix WT_REP1. -Work dir: - /home/pipelinetest/work/9d/172ca5881234073e8d76f2a19c88fb - -Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run` -``` - -#### For beginners - -A first step to bypass this error, you could try to increase the amount of CPUs, memory, and time for the whole pipeline. Therefor you can try to increase the resource for the parameters `--max_cpus`, `--max_memory`, and `--max_time`. Based on the error above, you have to increase the amount of memory. Therefore you can go to the [parameter documentation of rnaseq](https://nf-co.re/rnaseq/3.9/parameters) and scroll down to the `show hidden parameter` button to get the default value for `--max_memory`. In this case 128GB, you than can try to run your pipeline again with `--max_memory 200GB -resume` to skip all process, that were already calculated. If you can not increase the resource of the complete pipeline, you can try to adapt the resource for a single process as mentioned below. - -#### Advanced option on process level - -To bypass this error you would need to find exactly which resources are set by the `STAR_ALIGN` process. The quickest way is to search for `process STAR_ALIGN` in the [nf-core/rnaseq Github repo](https://github.com/nf-core/rnaseq/search?q=process+STAR_ALIGN). -We have standardised the structure of Nextflow DSL2 pipelines such that all module files will be present in the `modules/` directory and so, based on the search results, the file we want is `modules/nf-core/star/align/main.nf`. -If you click on the link to that file you will notice that there is a `label` directive at the top of the module that is set to [`label process_high`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/modules/nf-core/software/star/align/main.nf#L9). -The [Nextflow `label`](https://www.nextflow.io/docs/latest/process.html#label) directive allows us to organise workflow processes in separate groups which can be referenced in a configuration file to select and configure subset of processes having similar computing requirements. -The default values for the `process_high` label are set in the pipeline's [`base.config`](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L33-L37) which in this case is defined as 72GB. -Providing you haven't set any other standard nf-core parameters to **cap** the [maximum resources](https://nf-co.re/usage/configuration#max-resources) used by the pipeline then we can try and bypass the `STAR_ALIGN` process failure by creating a custom config file that sets at least 72GB of memory, in this case increased to 100GB. -The custom config below can then be provided to the pipeline via the [`-c`](#-c) parameter as highlighted in previous sections. - -```nextflow -process { - withName: 'NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN' { - memory = 100.GB - } -} -``` - -> **NB:** We specify the full process name i.e. `NFCORE_RNASEQ:RNASEQ:ALIGN_STAR:STAR_ALIGN` in the config file because this takes priority over the short name (`STAR_ALIGN`) and allows existing configuration using the full process name to be correctly overridden. -> -> If you get a warning suggesting that the process selector isn't recognised check that the process name has been specified correctly. - -### Updating containers (advanced users) - -The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. If for some reason you need to use a different version of a particular tool with the pipeline then you just need to identify the `process` name and override the Nextflow `container` definition for that process using the `withName` declaration. For example, in the [nf-core/viralrecon](https://nf-co.re/viralrecon) pipeline a tool called [Pangolin](https://github.com/cov-lineages/pangolin) has been used during the COVID-19 pandemic to assign lineages to SARS-CoV-2 genome sequenced samples. Given that the lineage assignments change quite frequently it doesn't make sense to re-release the nf-core/viralrecon everytime a new version of Pangolin has been released. However, you can override the default container used by the pipeline by creating a custom config file and passing it as a command-line argument via `-c custom.config`. - -1. Check the default version used by the pipeline in the module file for [Pangolin](https://github.com/nf-core/viralrecon/blob/a85d5969f9025409e3618d6c280ef15ce417df65/modules/nf-core/software/pangolin/main.nf#L14-L19) -2. Find the latest version of the Biocontainer available on [Quay.io](https://quay.io/repository/biocontainers/pangolin?tag=latest&tab=tags) -3. Create the custom config accordingly: - - - For Docker: - - ```nextflow - process { - withName: PANGOLIN { - container = 'quay.io/biocontainers/pangolin:3.0.5--pyhdfd78af_0' - } - } - ``` - - - For Singularity: - - ```nextflow - process { - withName: PANGOLIN { - container = 'https://depot.galaxyproject.org/singularity/pangolin:3.0.5--pyhdfd78af_0' - } - } - ``` +### Custom Containers - - For Conda: +In some cases you may wish to change which container or conda environment a step of the pipeline uses for a particular tool. By default nf-core pipelines use containers and software from the [biocontainers](https://biocontainers.pro/) or [bioconda](https://bioconda.github.io/) projects. However in some cases the pipeline specified version maybe out of date. - ```nextflow - process { - withName: PANGOLIN { - conda = 'bioconda::pangolin=3.0.5' - } - } - ``` +To use a different container from the default container or conda environment specified in a pipeline, please see the [max resources](https://nf-co.re/docs/usage/configuration#updating-tool-versions) section of the nf-core website. -> **NB:** If you wish to periodically update individual tool-specific results (e.g. Pangolin) generated by the pipeline then you must ensure to keep the `work/` directory otherwise the `-resume` ability of the pipeline will be compromised and it will restart from scratch. +### Custom Tool Arguments + +A pipeline might not always support every possible argument or option of a particular tool used in pipeline. Fortunately, nf-core pipelines provide some freedom to users to insert additional parameters that the pipeline does not include by default. -{% if nf_core_configs -%} +To learn how to provide additional arguments to a particular tool of the pipeline, please see the [customising tool arguments](https://nf-co.re/docs/usage/configuration#customising-tool-arguments) section of the nf-core website. ### nf-core/configs From 42d0aa923f14e215996ab07447293614bf6c90dd Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 14 Mar 2023 14:47:23 +0000 Subject: [PATCH 080/246] [automated] Fix code linting --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 776c72af96..97b22b93d3 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -181,7 +181,7 @@ In some cases you may wish to change which container or conda environment a step To use a different container from the default container or conda environment specified in a pipeline, please see the [max resources](https://nf-co.re/docs/usage/configuration#updating-tool-versions) section of the nf-core website. ### Custom Tool Arguments - + A pipeline might not always support every possible argument or option of a particular tool used in pipeline. Fortunately, nf-core pipelines provide some freedom to users to insert additional parameters that the pipeline does not include by default. To learn how to provide additional arguments to a particular tool of the pipeline, please see the [customising tool arguments](https://nf-co.re/docs/usage/configuration#customising-tool-arguments) section of the nf-core website. From 69074a2b8fde8dc8931d2528191c002fa8255106 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 16 Mar 2023 09:39:57 +0100 Subject: [PATCH 081/246] fix GITHUB_HEAD_REF equals dev check --- .github/workflows/branch.yml | 2 +- nf_core/pipeline-template/.github/workflows/branch.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 1f3d241d5f..dd64ffa3e5 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -13,7 +13,7 @@ jobs: - name: Check PRs if: github.repository == 'nf-core/tools' run: | - { [[ ${{github.event.pull_request.head.repo.full_name}} == nf-core/tools ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] + { [[ ${{github.event.pull_request.head.repo.full_name}} == nf-core/tools ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] # If the above check failed, post a comment on the PR explaining the failure - name: Post PR comment diff --git a/nf_core/pipeline-template/.github/workflows/branch.yml b/nf_core/pipeline-template/.github/workflows/branch.yml index e22310ba6f..8edfa540c9 100644 --- a/nf_core/pipeline-template/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/.github/workflows/branch.yml @@ -13,7 +13,7 @@ jobs: - name: Check PRs if: github.repository == '{{ name }}' run: | - { [[ {% raw %}${{github.event.pull_request.head.repo.full_name }}{% endraw %} == {{ name }} ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] + { [[ {% raw %}${{github.event.pull_request.head.repo.full_name }}{% endraw %} == {{ name }} ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] # If the above check failed, post a comment on the PR explaining the failure {%- raw %} # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets From f0b11cf4df556dc3800a91ba2ee9848f27a07733 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Thu, 16 Mar 2023 09:59:09 +0100 Subject: [PATCH 082/246] consistent syntax for branch ckecks in PRs --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34276b0321..bb9fe40caa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Add documentation to `usage.md` on how to use `params.yml` files, based on nf-core/ampliseq text ([#2173](https://github.com/nf-core/tools/pull/2173/)) [Contributed by @jfy133, @d4straub] - Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ([#2170](https://github.com/nf-core/tools/pull/2170)) - Remove problematic sniffer code in samplesheet_check.py that could give false positive 'missing header' errors ([https://github.com/nf-core/tools/pull/2194]) [Contributed by @Midnighter, @jfy133] +- Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) ### Linting From 5fe0cb3d32ea48a359eb344572bf2bf92ecb5682 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 23 Mar 2023 10:21:26 +0100 Subject: [PATCH 083/246] Update usage.md --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 97b22b93d3..381a7ec69d 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -134,7 +134,7 @@ Several generic profiles are bundled with the pipeline which instruct the pipeli The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). {% else %} -{% endif %} +{% endif -%} Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. From 638cc677cd7468a9ebe7c959717ca905b2fb36bd Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 23 Mar 2023 10:21:40 +0100 Subject: [PATCH 084/246] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 381a7ec69d..f4868512e6 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -172,7 +172,7 @@ Specify the path to a specific config file (this is a core Nextflow command). Se Whilst the default requirements set within the pipeline will hopefully work for most people and with most input data, you may find that you want to customise the compute resources that the pipeline requests. Each step in the pipeline has a default set of requirements for number of CPUs, memory and time. For most of the steps in the pipeline, if the job exits with any of the error codes specified [here](https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L18) it will automatically be resubmitted with higher requests (2 x original, then 3 x original). If it still fails after the third attempt then the pipeline execution is stopped. -To optimise resource requests away from the pipeline 'base' defaults, please see the [max resources](https://nf-co.re/docs/usage/configuration#max-resources) and [tuning workflow resources](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) section of the nf-core website. +To change the resource requests, please see the [max resources](https://nf-co.re/docs/usage/configuration#max-resources) and [tuning workflow resources](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources) section of the nf-core website. ### Custom Containers From 411327e461f4d76c68d4e1a6171f9d8769a78998 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 23 Mar 2023 11:00:41 +0100 Subject: [PATCH 085/246] Apply suggestions from code review --- nf_core/pipeline-template/docs/usage.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index f4868512e6..562daae7f9 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -134,7 +134,7 @@ Several generic profiles are bundled with the pipeline which instruct the pipeli The pipeline also dynamically loads configurations from [https://github.com/nf-core/configs](https://github.com/nf-core/configs) when it runs, making multiple config profiles for various institutional clusters available at run time. For more information and to see if your system is available in these configs please see the [nf-core/configs documentation](https://github.com/nf-core/configs#documentation). {% else %} -{% endif -%} +{% endif %} Note that multiple profiles can be loaded, for example: `-profile test,docker` - the order of arguments is important! They are loaded in sequence, so later profiles can overwrite earlier profiles. @@ -184,6 +184,7 @@ To use a different container from the default container or conda environment spe A pipeline might not always support every possible argument or option of a particular tool used in pipeline. Fortunately, nf-core pipelines provide some freedom to users to insert additional parameters that the pipeline does not include by default. +{% if nf_core_configs -%} To learn how to provide additional arguments to a particular tool of the pipeline, please see the [customising tool arguments](https://nf-co.re/docs/usage/configuration#customising-tool-arguments) section of the nf-core website. ### nf-core/configs From 4bc3f6072ed15458901aae3e79f21148dff0885d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 23 Mar 2023 11:09:00 +0100 Subject: [PATCH 086/246] Apply suggestions from code review --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 562daae7f9..47270b7432 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -178,7 +178,7 @@ To change the resource requests, please see the [max resources](https://nf-co.re In some cases you may wish to change which container or conda environment a step of the pipeline uses for a particular tool. By default nf-core pipelines use containers and software from the [biocontainers](https://biocontainers.pro/) or [bioconda](https://bioconda.github.io/) projects. However in some cases the pipeline specified version maybe out of date. -To use a different container from the default container or conda environment specified in a pipeline, please see the [max resources](https://nf-co.re/docs/usage/configuration#updating-tool-versions) section of the nf-core website. +To use a different container from the default container or conda environment specified in a pipeline, please see the [updating tool versions](https://nf-co.re/docs/usage/configuration#updating-tool-versions) section of the nf-core website. ### Custom Tool Arguments From addcace82cd11d65ecb4e32731131dd51bf1de23 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 24 Mar 2023 11:26:22 +0100 Subject: [PATCH 087/246] Fixed minor Jinja2 templating bug that caused the PR template to miss a newline --- CHANGELOG.md | 1 + nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb9fe40caa..29038c456f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ([#2170](https://github.com/nf-core/tools/pull/2170)) - Remove problematic sniffer code in samplesheet_check.py that could give false positive 'missing header' errors ([https://github.com/nf-core/tools/pull/2194]) [Contributed by @Midnighter, @jfy133] - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) +- Fixed minor Jinja2 templating bug that caused the PR template to miss a newline ### Linting diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index 3278a33b1e..0f81ebaa4c 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -16,8 +16,9 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) - {%- if branded -%} -- [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.{% endif %} + {%- if branded %} +- [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. + {%- endif %} - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. From d88756bba22d4ccd7e5fb8d4e0c69120d8866ce8 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 27 Mar 2023 10:35:36 +0100 Subject: [PATCH 088/246] Add test for pipeline init with template yml --- tests/data/pipeline_create_template.yml | 1 + tests/test_create.py | 65 ++++++++++++++++++++----- 2 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 tests/data/pipeline_create_template.yml diff --git a/tests/data/pipeline_create_template.yml b/tests/data/pipeline_create_template.yml new file mode 100644 index 0000000000..12e48e9c27 --- /dev/null +++ b/tests/data/pipeline_create_template.yml @@ -0,0 +1 @@ +prefix: testprefix diff --git a/tests/test_create.py b/tests/test_create.py index baac509d74..2e709ef94b 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -2,6 +2,7 @@ """ import os import unittest +from pathlib import Path import git @@ -9,17 +10,38 @@ from .utils import with_temporary_folder +TEST_DATA_DIR = Path(__file__).parent / "data" +PIPELINE_TEMPLATE_YML = TEST_DATA_DIR / "pipeline_create_template.yml" + class NfcoreCreateTest(unittest.TestCase): - @with_temporary_folder - def setUp(self, tmp_path): + def setUp(self): self.pipeline_name = "nf-core/test" self.pipeline_description = "just for 4w3s0m3 tests" self.pipeline_author = "Chuck Norris" self.pipeline_version = "1.0.0" self.default_branch = "default" - self.pipeline = nf_core.create.PipelineCreate( + def test_pipeline_creation(self): + pipeline = nf_core.create.PipelineCreate( + name=self.pipeline_name, + description=self.pipeline_description, + author=self.pipeline_author, + version=self.pipeline_version, + no_git=False, + force=True, + plain=True, + default_branch=self.default_branch, + ) + + assert pipeline.template_params["name"] == self.pipeline_name + assert pipeline.template_params["description"] == self.pipeline_description + assert pipeline.template_params["author"] == self.pipeline_author + assert pipeline.template_params["version"] == self.pipeline_version + + @with_temporary_folder + def test_pipeline_creation_initiation(self, tmp_path): + pipeline = nf_core.create.PipelineCreate( name=self.pipeline_name, description=self.pipeline_description, author=self.pipeline_author, @@ -30,14 +52,31 @@ def setUp(self, tmp_path): plain=True, default_branch=self.default_branch, ) + pipeline.init_pipeline() + assert os.path.isdir(os.path.join(pipeline.outdir, ".git")) + assert f" {self.default_branch}\n" in git.Repo.init(pipeline.outdir).git.branch() + assert not os.path.exists(os.path.join(pipeline.outdir, "pipeline_template.yml")) - def test_pipeline_creation(self): - assert self.pipeline.template_params["name"] == self.pipeline_name - assert self.pipeline.template_params["description"] == self.pipeline_description - assert self.pipeline.template_params["author"] == self.pipeline_author - assert self.pipeline.template_params["version"] == self.pipeline_version - - def test_pipeline_creation_initiation(self): - self.pipeline.init_pipeline() - assert os.path.isdir(os.path.join(self.pipeline.outdir, ".git")) - assert f" {self.default_branch}\n" in git.Repo.init(self.pipeline.outdir).git.branch() + @with_temporary_folder + def test_pipeline_creation_initiation_with_yml(self, tmp_path): + pipeline = nf_core.create.PipelineCreate( + name=self.pipeline_name, + description=self.pipeline_description, + author=self.pipeline_author, + version=self.pipeline_version, + no_git=False, + force=True, + outdir=tmp_path, + template_yaml_path=PIPELINE_TEMPLATE_YML, + plain=True, + default_branch=self.default_branch, + ) + pipeline.init_pipeline() + assert os.path.isdir(os.path.join(pipeline.outdir, ".git")) + assert f" {self.default_branch}\n" in git.Repo.init(pipeline.outdir).git.branch() + + # Check pipeline yml has been dumped and matches input + pipeline_template = os.path.join(pipeline.outdir, "pipeline_template.yml") + assert os.path.exists(pipeline_template) + with open(pipeline_template) as fh: + assert fh.read() == PIPELINE_TEMPLATE_YML.read_text() From 637e9678db8265e1ced0a90b9c2e248b2e24d9fd Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:38:16 +0100 Subject: [PATCH 089/246] Add test mocking cli input --- tests/test_create.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_create.py b/tests/test_create.py index 2e709ef94b..e454f0f85d 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -3,6 +3,7 @@ import os import unittest from pathlib import Path +from unittest import mock import git @@ -80,3 +81,27 @@ def test_pipeline_creation_initiation_with_yml(self, tmp_path): assert os.path.exists(pipeline_template) with open(pipeline_template) as fh: assert fh.read() == PIPELINE_TEMPLATE_YML.read_text() + + @mock.patch.object(nf_core.create.PipelineCreate, "customize_template") + @mock.patch.object(nf_core.create.questionary, "confirm") + @with_temporary_folder + def test_pipeline_creation_initiation_customize_template(self, mock_questionary, mock_customize, tmp_path): + mock_questionary.unsafe_ask.return_value = True + mock_customize.return_value = {"prefix": "testprefix"} + pipeline = nf_core.create.PipelineCreate( + name=self.pipeline_name, + description=self.pipeline_description, + author=self.pipeline_author, + version=self.pipeline_version, + no_git=False, + force=True, + outdir=tmp_path, + default_branch=self.default_branch, + ) + pipeline.init_pipeline() + assert os.path.isdir(os.path.join(pipeline.outdir, ".git")) + assert f" {self.default_branch}\n" in git.Repo.init(pipeline.outdir).git.branch() + + # Check pipeline yml has been dumped and matches input + pipeline_template = os.path.join(pipeline.outdir, "pipeline_template.yml") + assert os.path.exists(pipeline_template) From 326e600d8ddbd5fb5914bf6781f141ce91716186 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:38:54 +0100 Subject: [PATCH 090/246] Add check to ensure output yml matches expected --- tests/test_create.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_create.py b/tests/test_create.py index e454f0f85d..cc6bf8ba47 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -105,3 +105,5 @@ def test_pipeline_creation_initiation_customize_template(self, mock_questionary, # Check pipeline yml has been dumped and matches input pipeline_template = os.path.join(pipeline.outdir, "pipeline_template.yml") assert os.path.exists(pipeline_template) + with open(pipeline_template) as fh: + assert fh.read() == PIPELINE_TEMPLATE_YML.read_text() From a8384a034314c9c319823a0c87ec413d54ba3f57 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 27 Mar 2023 13:14:04 +0100 Subject: [PATCH 091/246] Initial commit of system exit lint check --- nf_core/lint/__init__.py | 2 ++ nf_core/lint/system_exit.py | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 nf_core/lint/system_exit.py diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index e014a933ea..a998c964a0 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -181,6 +181,7 @@ class PipelineLint(nf_core.utils.Pipeline): from .schema_description import schema_description from .schema_lint import schema_lint from .schema_params import schema_params + from .system_exit import system_exit from .template_strings import template_strings from .version_consistency import version_consistency @@ -223,6 +224,7 @@ def _get_all_lint_tests(release_mode): "template_strings", "schema_lint", "schema_params", + "system_exit", "schema_description", "actions_schema_validation", "merge_markers", diff --git a/nf_core/lint/system_exit.py b/nf_core/lint/system_exit.py new file mode 100644 index 0000000000..a6764a920f --- /dev/null +++ b/nf_core/lint/system_exit.py @@ -0,0 +1,26 @@ +import logging +from pathlib import Path + +log = logging.getLogger(__name__) + + +def system_exit(self): + passed = [] + warned = [] + + root_dir = Path(self.wf_path) + + # Get all groovy and nf files + groovy_files = [f for f in root_dir.rglob("*.groovy")] + nf_files = [f for f in root_dir.rglob("*.nf")] + to_check = nf_files + groovy_files + + for file in to_check: + with file.open() as fh: + for l in fh.readlines(): + if "System.exit" in l: + warned.append(f"`System.exit` in {file.name}: _{l.strip()}_") + if len(warned) == 0: + passed.append("No `System.exit` calls found") + + return {"passed": passed, "warned": warned} From f6a19ebad32119446aa683f299bf7536c475baf9 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 27 Mar 2023 13:15:31 +0200 Subject: [PATCH 092/246] deprecate Python 3.7 support --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/create-test-wf.yml | 4 ++-- .github/workflows/deploy-pypi.yml | 4 ++-- .github/workflows/pytest.yml | 2 +- .github/workflows/tools-api-docs-dev.yml | 4 ++-- .github/workflows/tools-api-docs-release.yml | 4 ++-- CHANGELOG.md | 1 + nf_core/modules/test_yml_builder.py | 13 +++---------- nf_core/subworkflows/test_yml_builder.py | 13 +++---------- setup.py | 2 +- 10 files changed, 18 insertions(+), 31 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 6cf8ffd921..1bd2e27fb0 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -38,4 +38,4 @@ body: * Executor _(eg. slurm, local, awsbatch)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ * Version of nf-core/tools _(eg. 1.1, 1.5, 1.8.2)_ - * Python version _(eg. 3.7, 3.8)_ + * Python version _(eg. 3.10, 3.11)_ diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 2da7901d16..8f3c5fdb47 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -29,10 +29,10 @@ jobs: - uses: actions/checkout@v3 name: Check out source-code repository - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Install python dependencies run: | diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 8657283417..d762154f7e 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v3 name: Check out source-code repository - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Install python dependencies run: | diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 00cd3c813e..d03b1c33de 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index 0c84d38aea..3ec4dc2e10 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -22,10 +22,10 @@ jobs: - name: Check out source-code repository uses: actions/checkout@v3 - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Install python dependencies run: | diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index 98c4f997fb..71acf6baa1 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -21,10 +21,10 @@ jobs: - name: Check out source-code repository uses: actions/checkout@v3 - - name: Set up Python 3.7 + - name: Set up Python 3.8 uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Install python dependencies run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 29038c456f..0c67714ca9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ - Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) - Deprecate `--enable_conda` parameter. Use `conda.enable` instead ([#2131](https://github.com/nf-core/tools/pull/2131)) - Handle `json.load()` exceptions ([#2134](https://github.com/nf-core/tools/pull/2134)) +- Deprecate Python 3.7 support because it reaches EOL ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py index c9a6273a58..7ab9e464d4 100644 --- a/nf_core/modules/test_yml_builder.py +++ b/nf_core/modules/test_yml_builder.py @@ -213,16 +213,9 @@ def check_if_empty_file(self, fname): g_f = gzip.GzipFile(fileobj=fh, mode="rb") if g_f.read() == b"": return True - except Exception as e: - # Python 3.8+ - if hasattr(gzip, "BadGzipFile"): - if isinstance(e, gzip.BadGzipFile): - pass - # Python 3.7 - elif isinstance(e, OSError): - pass - else: - raise e + except gzip.BadGzipFile: + pass + return False def _md5(self, fname): diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index 3090b22c05..39877ca241 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -230,16 +230,9 @@ def check_if_empty_file(self, fname): g_f = gzip.GzipFile(fileobj=fh, mode="rb") if g_f.read() == b"": return True - except Exception as e: - # Python 3.8+ - if hasattr(gzip, "BadGzipFile"): - if isinstance(e, gzip.BadGzipFile): - pass - # Python 3.7 - elif isinstance(e, OSError): - pass - else: - raise e + except gzip.BadGzipFile: + pass + return False def _md5(self, fname): diff --git a/setup.py b/setup.py index 38f36810b7..b5c756c7f8 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ "console_scripts": ["nf-core=nf_core.__main__:run_nf_core"], "refgenie.hooks.post_update": ["nf-core-refgenie=nf_core.refgenie:update_config"], }, - python_requires=">=3.7, <4", + python_requires=">=3.8, <4", install_requires=required, packages=find_packages(exclude=("docs")), include_package_data=True, From f3b5d1e2c7d1a5d4959535947c73954070d4ba18 Mon Sep 17 00:00:00 2001 From: fabianegli Date: Mon, 27 Mar 2023 13:35:09 +0200 Subject: [PATCH 093/246] add link to PR --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c67714ca9..c0dd174e3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,7 +44,7 @@ - Only check that a pipeline name doesn't contain dashes if the name is provided by prompt of `--name`. Don't check if a template file is used. ([#2123](https://github.com/nf-core/tools/pull/2123)) - Deprecate `--enable_conda` parameter. Use `conda.enable` instead ([#2131](https://github.com/nf-core/tools/pull/2131)) - Handle `json.load()` exceptions ([#2134](https://github.com/nf-core/tools/pull/2134)) -- Deprecate Python 3.7 support because it reaches EOL +- Deprecate Python 3.7 support because it reaches EOL ([#2210](https://github.com/nf-core/tools/pull/2210)) ## [v2.7.1 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.1) - [2022-12-08] From 1422b3444e50c98aa30fc283b781c914658ead95 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 27 Mar 2023 14:38:19 +0100 Subject: [PATCH 094/246] Add a small docstring. Add line number to lint message --- nf_core/lint/system_exit.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/nf_core/lint/system_exit.py b/nf_core/lint/system_exit.py index a6764a920f..419f0cd63d 100644 --- a/nf_core/lint/system_exit.py +++ b/nf_core/lint/system_exit.py @@ -5,6 +5,13 @@ def system_exit(self): + """Check for System.exit calls in groovy/nextflow code + + Calls to System.exit(1) should be replaced by throwing errors + + This lint test looks for all calls to `System.exit` + in any file with the `.nf` or `.groovy` extension + """ passed = [] warned = [] @@ -16,10 +23,14 @@ def system_exit(self): to_check = nf_files + groovy_files for file in to_check: - with file.open() as fh: - for l in fh.readlines(): - if "System.exit" in l: - warned.append(f"`System.exit` in {file.name}: _{l.strip()}_") + try: + with file.open() as fh: + for i, l in enumerate(fh.readlines(), start=1): + if "System.exit" in l: + warned.append(f"`System.exit` in {file.name}: _{l.strip()}_ [line {i}]") + except FileNotFoundError: + log.debug(f"Could not open file {file.name} in system_exit lint test") + if len(warned) == 0: passed.append("No `System.exit` calls found") From 8d70a776e0f9d0522c066f97d49ef8b2eb79c1f0 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 27 Mar 2023 15:20:25 +0100 Subject: [PATCH 095/246] Add docs .md stub --- docs/api/_src/pipeline_lint_tests/system_exit.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/api/_src/pipeline_lint_tests/system_exit.md diff --git a/docs/api/_src/pipeline_lint_tests/system_exit.md b/docs/api/_src/pipeline_lint_tests/system_exit.md new file mode 100644 index 0000000000..3d0ac20f8d --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/system_exit.md @@ -0,0 +1,5 @@ +# system_exit + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.system_exit +``` From 9626ae5f18d3660da28eca3cd3523ded1e9069fa Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 27 Mar 2023 16:49:02 +0100 Subject: [PATCH 096/246] Allow System.exit(0) and update template --- nf_core/lint/system_exit.py | 2 +- nf_core/pipeline-template/lib/NfcoreSchema.groovy | 3 ++- nf_core/pipeline-template/lib/WorkflowMain.groovy | 5 +++-- nf_core/pipeline-template/lib/WorkflowPipeline.groovy | 8 ++++---- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/nf_core/lint/system_exit.py b/nf_core/lint/system_exit.py index 419f0cd63d..56a526d97b 100644 --- a/nf_core/lint/system_exit.py +++ b/nf_core/lint/system_exit.py @@ -26,7 +26,7 @@ def system_exit(self): try: with file.open() as fh: for i, l in enumerate(fh.readlines(), start=1): - if "System.exit" in l: + if "System.exit" in l and not "System.exit(0)" in l: warned.append(f"`System.exit` in {file.name}: _{l.strip()}_ [line {i}]") except FileNotFoundError: log.debug(f"Could not open file {file.name} in system_exit lint test") diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index 33cd4f6e8d..4d29681431 100755 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -2,6 +2,7 @@ // This file holds several functions used to perform JSON parameter validation, help and summary rendering for the nf-core pipeline template. // +import nextflow.Nextflow import org.everit.json.schema.Schema import org.everit.json.schema.loader.SchemaLoader import org.everit.json.schema.ValidationException @@ -177,7 +178,7 @@ class NfcoreSchema { } if (has_error) { - System.exit(1) + Nextflow.error('Exiting!') } } diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 05db418b2d..2024e95c79 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -2,6 +2,8 @@ // This file holds several functions specific to the main.nf workflow in the {{ name }} pipeline // +import nextflow.Nextflow + class WorkflowMain { // @@ -85,8 +87,7 @@ class WorkflowMain { // Check input has been provided if (!params.input) { - log.error "Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'" - System.exit(1) + Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'") } } {% if igenomes -%} diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index 252f127d80..600a655439 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -2,6 +2,7 @@ // This file holds several functions specific to the workflow/{{ short_name }}.nf in the {{ name }} pipeline // +import nextflow.Nextflow import groovy.text.SimpleTemplateEngine class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { @@ -15,8 +16,7 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { {% endif %} if (!params.fasta) { - log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." - System.exit(1) + Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." } } @@ -70,12 +70,12 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { // private static void genomeExistsError(params, log) { if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - log.error "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + " Currently, the available genome keys are:\n" + " ${params.genomes.keySet().join(", ")}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - System.exit(1) + Nextflow.error(error_string) } } {% endif -%}} From 671c1e59dec927fba1473a2ce29f280eb0cc3957 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 27 Mar 2023 14:58:17 -0400 Subject: [PATCH 097/246] Add failed lint condition when params.enable_conda is in a module --- nf_core/modules/lint/main_nf.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index d44fe90f1e..1bfbb6825e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -257,9 +257,15 @@ def check_process_section(self, lines, fix_version, progress_bar): self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) for i, l in enumerate(lines): url = None + l = l.strip(" '\"") if _container_type(l) == "bioconda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] - l = l.strip(" '\"") + if _container_type(l) == "bioconda" or _container_type(l) == "conda": + match = re.search(r"params\.enable_conda",l) + if match is None: + self.passed.append(("deprecated_enable_conda", f"Found deprecated parameter 'params.enable_conda' in the conda definition", self.main_nf)) + else: + self.failed.append(("deprecated_enable_conda", f"Deprecated parameter 'params.enable_conda' correctly not found in the conda definition", self.main_nf)) if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 @@ -518,6 +524,8 @@ def _container_type(line): """Returns the container type of a build.""" if re.search("bioconda::", line): return "bioconda" + if line.startswith("conda"): + return "conda" if line.startswith("https://containers") or line.startswith("https://depot"): # Look for a http download URL. # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980 From d78b9263d3521777fe620848562926f229d4f01c Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 27 Mar 2023 15:38:39 -0400 Subject: [PATCH 098/246] Empty-Commit From 97fda70a315cc5b5c19a61a4cb7784a5f0f6d294 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 27 Mar 2023 15:47:35 -0400 Subject: [PATCH 099/246] fix code formatting with black --- nf_core/modules/lint/main_nf.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 1bfbb6825e..c9bc9f4526 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -261,11 +261,23 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "bioconda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] if _container_type(l) == "bioconda" or _container_type(l) == "conda": - match = re.search(r"params\.enable_conda",l) + match = re.search(r"params\.enable_conda", l) if match is None: - self.passed.append(("deprecated_enable_conda", f"Found deprecated parameter 'params.enable_conda' in the conda definition", self.main_nf)) + self.passed.append( + ( + "deprecated_enable_conda", + f"Found deprecated parameter 'params.enable_conda' in the conda definition", + self.main_nf, + ) + ) else: - self.failed.append(("deprecated_enable_conda", f"Deprecated parameter 'params.enable_conda' correctly not found in the conda definition", self.main_nf)) + self.failed.append( + ( + "deprecated_enable_conda", + f"Deprecated parameter 'params.enable_conda' correctly not found in the conda definition", + self.main_nf, + ) + ) if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 From 30a6dc3dafd8a81d28920093a85f2526e78d84a4 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 27 Mar 2023 16:14:12 -0400 Subject: [PATCH 100/246] add tests to account for params.enable_conda failures --- tests/modules/lint.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 476481a109..70fa6bff2b 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -66,7 +66,7 @@ def test_modules_lint_gitlab_modules(self): self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) module_lint.lint(print_results=False, all_modules=True) - assert len(module_lint.failed) == 0 + assert len(module_lint.failed) == 1 assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -77,7 +77,7 @@ def test_modules_lint_multiple_remotes(self): self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) module_lint.lint(print_results=False, all_modules=True) - assert len(module_lint.failed) == 0 + assert len(module_lint.failed) == 1 assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -103,6 +103,6 @@ def test_modules_lint_patched_modules(self): all_modules=True, ) - assert len(module_lint.failed) == 0 + assert len(module_lint.failed) == 1 assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 From 95594951a9fbebabcbb9c26a828e1ba3e615e815 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 27 Mar 2023 16:31:16 -0400 Subject: [PATCH 101/246] Add enable_conda linting to CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0dd174e3d..112c05d8bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ ### Linting +- Update modules lint test to fail if enable_conda is found ([#2213](https://github.com/nf-core/tools/pull/2213)) + ### Modules - Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175) & [#2177](https://github.com/nf-core/tools/pull/2177)) From c2ab6fee5975280d2e99815ca1b23abb6456073e Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 27 Mar 2023 16:43:50 -0400 Subject: [PATCH 102/246] Fix number of fails in assertion test --- tests/modules/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 70fa6bff2b..18c0dc4dab 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -66,7 +66,7 @@ def test_modules_lint_gitlab_modules(self): self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) module_lint.lint(print_results=False, all_modules=True) - assert len(module_lint.failed) == 1 + assert len(module_lint.failed) == 2 assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 From d19f64b516faa7b2b3a8cff64d178b8cbbe418ae Mon Sep 17 00:00:00 2001 From: jenmuell Date: Tue, 28 Mar 2023 11:36:44 +0200 Subject: [PATCH 103/246] API spec of the different JSON schema fields --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 73d868bb32..966edfb00b 100644 --- a/README.md +++ b/README.md @@ -608,7 +608,18 @@ If you want to add a parameter to the schema, you first have to add the paramete The graphical interface is oganzised in groups and within the groups the single parameters are stored. For a better overview you can collapse all groups with the `Collapse groups` button, then your new parameters will be the only remaining one at the bottom of the page. Now you can either create a new group with the `Add group` button or drag and drop the paramters in an existing group. Therefor the group has to be expanded. The group title will be displayed, if you run your pipeline with the `--help` flag and its description apears on the parameter page of your pipeline. -Now you can start to change the parameter itself. The description is a short explanation about the parameter, that apears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For intergers you can define a min and max value, and for strings the file extension can be specified. +Now you can start to change the parameter itself. The `ID` of a new parameter should be defined in small lettersm without whitespaces. The description is a short free text explanation about the parameter, that apears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. Usually, they contain a small paragraph about the parameter settings or a used datasource, like databases or references. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For intergers you can define a min and max value, and for strings the file extension can be specified. + +The `type` field is one of the most important points in your pipeline schema, since it define the datatype of your input and how it will be interpreted. This allows extensive testing before even starting the pipeline. + +The basic datatypes for a pipeline schema are: + +- `string` +- `number` +- `integer` +- `boolean` + +For the `string` type you have three different options in the settings (nut icon). First, `enumerated values`, which allow you to specify a list of specific input values. The list has to be sepearated with a pipe. The `pattern` and `format` setting can depend on each other. The `format` has to be a directory path, a file path or a path in general. Dependent on the `format`, it could be importent to specify the `pattern` setting. Especially for `file paths`, to specify the file extension can save you a lot of effort. For the `number` and `integer` type you have the same setting options. Similar to string you have `enumerated values` and additionally you can specify a `min` and `max` value. For the `boolean` type you have no further settings and usually the default value is `false`, which allows you to switch it to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. After you filled your schema, click on the `Finished` button in the top rigth corner, this will automatically update your `nextflow_schema.json`. If this is not working you can copy the schema from the graphical interface and paste it in your `nextflow_schema.json` file. From 46595026dfd95ef3fe5df8c2dec7717835e4c311 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 28 Mar 2023 11:36:31 +0100 Subject: [PATCH 104/246] Initial commit of one method for patch removal --- nf_core/__main__.py | 8 +++- nf_core/modules/modules_json.py | 10 +++++ nf_core/modules/patch.py | 68 +++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 841cb9f7e7..0b29f1769c 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -618,7 +618,8 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, update_d default=".", help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) -def patch(ctx, tool, dir): +@click.option("-r", "--remove", is_flag=True, default=False) +def patch(ctx, tool, dir, remove): """ Create a patch file for minor changes in a module @@ -632,7 +633,10 @@ def patch(ctx, tool, dir): ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], ) - module_patch.patch(tool) + if remove: + module_patch.remove(tool) + else: + module_patch.patch(tool) except (UserWarning, LookupError) as e: log.error(e) sys.exit(1) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 431ad1d657..13e6d4e32b 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -745,6 +745,16 @@ def add_patch_entry(self, module_name, repo_url, install_dir, patch_filename, wr if write_file: self.dump() + def remove_patch_entry(self, module_name, repo_url, install_dir, write_file=True): + if self.modules_json is None: + self.load() + try: + del self.modules_json["repos"][repo_url]["modules"][install_dir][module_name]["patch"] + except KeyError: + log.warning("No patch entry in 'modules.json' to remove") + if write_file: + self.dump() + def get_patch_fn(self, module_name, repo_url, install_dir): """ Get the patch filename of a module diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index fa51640c06..8b8068bafb 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -130,3 +130,71 @@ def patch(self, module=None): # Finally move the created patch file to its final location shutil.move(patch_temp_path, patch_path) log.info(f"Patch file of '{module_fullname}' written to '{patch_path}'") + + def remove(self, module): + # Check modules directory structure + self.check_modules_structure() + + self.modules_json.check_up_to_date() + self.param_check(module) + modules = self.modules_json.get_all_components(self.component_type)[self.modules_repo.remote_url] + + if module is None: + choices = [ + module if directory == self.modules_repo.repo_path else f"{directory}/{module}" + for directory, module in modules + ] + module = questionary.autocomplete( + "Tool:", + choices, + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + module_dir = [dir for dir, m in modules if m == module][0] + module_fullname = str(Path("modules", module_dir, module)) + + # Verify that the module has an entry in the modules.json file + if not self.modules_json.module_present(module, self.modules_repo.remote_url, module_dir): + raise UserWarning( + f"The '{module_fullname}' module does not have an entry in the 'modules.json' file. Cannot compute patch" + ) + + module_version = self.modules_json.get_module_version(module, self.modules_repo.remote_url, module_dir) + if module_version is None: + raise UserWarning( + f"The '{module_fullname}' module does not have a valid version in the 'modules.json' file. Cannot compute patch" + ) + # Get the module branch and reset it in the ModulesRepo object + module_branch = self.modules_json.get_component_branch( + self.component_type, module, self.modules_repo.remote_url, module_dir + ) + if module_branch != self.modules_repo.branch: + self.modules_repo.setup_branch(module_branch) + + # Set the diff filename based on the module name + patch_filename = f"{module.replace('/', '-')}.diff" + module_relpath = Path("modules", module_dir, module) + patch_relpath = Path(module_relpath, patch_filename) + module_current_dir = Path(self.dir, module_relpath) + patch_path = Path(self.dir, patch_relpath) + + if patch_path.exists(): + remove = questionary.confirm( + f"Patch exists for module '{module_fullname}'. Are you sure you want to remove?", + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + if not remove: + return + + # Try to apply the patch in reverse and move resulting files to module dir + temp_module_dir = self.modules_json.try_apply_patch_reverse( + module, self.modules_repo.repo_path, patch_relpath, module_relpath + ) + for file in Path(temp_module_dir).glob("*"): + file.rename(module_relpath.joinpath(file.name)) + os.rmdir(temp_module_dir) + + # Remove patch file + patch_path.unlink() + # Write changes to modules.json + self.modules_json.remove_patch_entry(module, self.modules_repo.remote_url, module_dir) + log.info(f"Patch for {module} reverted!") From a6623b1618d5b4ab785a8e480698c26317b26f84 Mon Sep 17 00:00:00 2001 From: Areda Elezi <73264439+aelezi01@users.noreply.github.com> Date: Tue, 28 Mar 2023 13:08:58 +0100 Subject: [PATCH 105/246] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e221694355..63bf235bc7 100644 --- a/README.md +++ b/README.md @@ -610,9 +610,9 @@ If you want to add a parameter to the schema, you first have to add the paramete The graphical interface is oganzised in groups and within the groups the single parameters are stored. For a better overview you can collapse all groups with the `Collapse groups` button, then your new parameters will be the only remaining one at the bottom of the page. Now you can either create a new group with the `Add group` button or drag and drop the paramters in an existing group. Therefor the group has to be expanded. The group title will be displayed, if you run your pipeline with the `--help` flag and its description apears on the parameter page of your pipeline. -Now you can start to change the parameter itself. The `ID` of a new parameter should be defined in small lettersm without whitespaces. The description is a short free text explanation about the parameter, that apears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. Usually, they contain a small paragraph about the parameter settings or a used datasource, like databases or references. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For intergers you can define a min and max value, and for strings the file extension can be specified. +Now you can start to change the parameter itself. The `ID` of a new parameter should be defined in small letters without whitespaces. The description is a short free text explanation about the parameter, that appears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. Usually, they contain a small paragraph about the parameter settings or a used datasource, like databases or references. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For integers you can define a min and max value, and for strings the file extension can be specified. -The `type` field is one of the most important points in your pipeline schema, since it define the datatype of your input and how it will be interpreted. This allows extensive testing before even starting the pipeline. +The `type` field is one of the most important points in your pipeline schema, since it defines the datatype of your input and how it will be interpreted. This allows extensive testing prior to starting the pipeline. The basic datatypes for a pipeline schema are: @@ -621,13 +621,13 @@ The basic datatypes for a pipeline schema are: - `integer` - `boolean` -For the `string` type you have three different options in the settings (nut icon). First, `enumerated values`, which allow you to specify a list of specific input values. The list has to be sepearated with a pipe. The `pattern` and `format` setting can depend on each other. The `format` has to be a directory path, a file path or a path in general. Dependent on the `format`, it could be importent to specify the `pattern` setting. Especially for `file paths`, to specify the file extension can save you a lot of effort. For the `number` and `integer` type you have the same setting options. Similar to string you have `enumerated values` and additionally you can specify a `min` and `max` value. For the `boolean` type you have no further settings and usually the default value is `false`, which allows you to switch it to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. +For the `string` type you have three different options in the settings (nut icon): `enumerated values`, `pattern` and `format`. The first option, `enumerated values`, allows you to specify a list of specific input values. The list has to be separated with a pipe. The `pattern` and `format` settings can depend on each other. The `format` has to be either a directory or a file path. Depending on the `format` setting selected, specifying the `pattern` setting can be the most efficient and time saving option, especially for `file paths`. The `number` and `integer` types share the same settings. Similarly to `string`, there is an `enumerated values` option with the possibility of specifying a `min` and `max` value. For the `boolean` there is no further settings and the default value is usually `false`. The `boolean` value can be switched to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. -After you filled your schema, click on the `Finished` button in the top rigth corner, this will automatically update your `nextflow_schema.json`. If this is not working you can copy the schema from the graphical interface and paste it in your `nextflow_schema.json` file. +After filling the schema, click on the `Finished` button in the top right corner, this will automatically update your `nextflow_schema.json`. If this is not working, the schema can be copied from the graphical interface and pasted in your `nextflow_schema.json` file. ### Update existing pipeline schema -Important for the update of a pipeline schema is, that if you want to change the default value of a parameter, you should change it in the `nextflow.config` file, since the value in the config file overwrites the value in the pipeline schema. To change any other parameter use `nf-core schema build --web-only` to open the graphical interface without rebuilding the pipeline schema. Now, you can change your parameters as mentioned above but keep in mind that changing the parameter datatype is depending on the default value you specified in the `nextflow.config` file. +It's important to change the default value of a parameter in the `nextflow.config` file first and then in the pipeline schema, because the value in the config file overwrites the value in the pipeline schema. To change any other parameter use `nf-core schema build --web-only` to open the graphical interface without rebuilding the pipeline schema. Now, the parameters can be changed as mentioned above but keep in mind that changing the parameter datatype depends on the default value specified in the `nextflow.config` file. ### Linting a pipeline schema From a3fa9d52a5a1d19073bace2e59b2b2c9b6ff3753 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 28 Mar 2023 14:33:22 +0200 Subject: [PATCH 106/246] Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action` --- CHANGELOG.md | 1 + nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 2 +- nf_core/pipeline-template/.github/workflows/awstest.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0dd174e3d..dd2c26df1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Remove problematic sniffer code in samplesheet_check.py that could give false positive 'missing header' errors ([https://github.com/nf-core/tools/pull/2194]) [Contributed by @Midnighter, @jfy133] - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) - Fixed minor Jinja2 templating bug that caused the PR template to miss a newline +- Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action` ### Linting diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 4b96fa80ee..4942167b12 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: nf-core/tower-action@v3 + uses: seqeralabs/action-tower-launch@v1 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters {%- raw %} diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 0f261fcb42..7f80cf1bb5 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -12,7 +12,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action {%- raw %} - name: Launch workflow via tower - uses: nf-core/tower-action@v3 + uses: seqeralabs/action-tower-launch@v1 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} From 8242026f48d135a668c007295d7d01c6b8196d4b Mon Sep 17 00:00:00 2001 From: ash Date: Tue, 28 Mar 2023 14:22:50 +0100 Subject: [PATCH 107/246] Make `nf-core modules lint` use new config file --- nf_core/components/components_command.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 31ab1a71fb..775b205cf5 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -6,6 +6,7 @@ import yaml +import nf_core.utils from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ModulesRepo @@ -162,24 +163,13 @@ def install_component_files(self, component_name, component_version, modules_rep def load_lint_config(self): """Parse a pipeline lint config file. - Look for a file called either `.nf-core-lint.yml` or - `.nf-core-lint.yaml` in the pipeline root directory and parse it. - (`.yml` takes precedence). + Load the '.nf-core.yml' config file and extract + the lint config from it Add parsed config to the `self.lint_config` class attribute. """ - config_fn = os.path.join(self.dir, ".nf-core-lint.yml") - - # Pick up the file if it's .yaml instead of .yml - if not os.path.isfile(config_fn): - config_fn = os.path.join(self.dir, ".nf-core-lint.yaml") - - # Load the YAML - try: - with open(config_fn, "r") as fh: - self.lint_config = yaml.safe_load(fh) - except FileNotFoundError: - log.debug(f"No lint config file found: {config_fn}") + _, tools_config = nf_core.utils.load_tools_config(self.dir) + self.lint_config = tools_config.get("lint", {}) def check_modules_structure(self): """ From 8d3404dbad4fce136076a4ec292d8745cd3ae960 Mon Sep 17 00:00:00 2001 From: ash Date: Tue, 28 Mar 2023 14:23:36 +0100 Subject: [PATCH 108/246] Correct old reference to .nf-core-lint.yaml --- nf_core/__main__.py | 2 +- nf_core/lint/nextflow_config.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 841cb9f7e7..2cf3805902 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -341,7 +341,7 @@ def lint(ctx, dir, release, fix, key, show_passed, fail_ignored, fail_warned, ma meets the nf-core guidelines. Documentation of all lint tests can be found on the nf-core website: [link=https://nf-co.re/tools-docs/]https://nf-co.re/tools-docs/[/] - You can ignore tests using a file called [blue].nf-core-lint.yaml[/] [i](if you have a good reason!)[/]. + You can ignore tests using a file called [blue].nf-core.yml[/] [i](if you have a good reason!)[/]. See the documentation for details. """ diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 79bce3e7f1..af018331f0 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -92,20 +92,22 @@ def nextflow_config(self): * Process-level configuration syntax still using the old Nextflow syntax, for example: ``process.$fastqc`` instead of ``process withName:'fastqc'``. .. tip:: You can choose to ignore tests for the presence or absence of specific config variables - by creating a file called ``.nf-core-lint.yml`` in the root of your pipeline and creating + by creating a file called ``.nf-core.yml`` in the root of your pipeline and creating a list the config variables that should be ignored. For example: .. code-block:: yaml - nextflow_config: - - params.input + lint: + nextflow_config: + - params.input The other checks in this test (depreciated syntax etc) can not be individually identified, but you can skip the entire test block if you wish: .. code-block:: yaml - nextflow_config: False + lint: + nextflow_config: False """ passed = [] warned = [] From cf45d4941e8189f1c78062e428777d754c7a59b9 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Tue, 28 Mar 2023 09:43:53 -0400 Subject: [PATCH 109/246] Remove container determination of bioconda in favor of determining conda --- nf_core/modules/lint/main_nf.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index c9bc9f4526..1caa67ac4f 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -258,9 +258,8 @@ def check_process_section(self, lines, fix_version, progress_bar): for i, l in enumerate(lines): url = None l = l.strip(" '\"") - if _container_type(l) == "bioconda": + if _container_type(l) == "conda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] - if _container_type(l) == "bioconda" or _container_type(l) == "conda": match = re.search(r"params\.enable_conda", l) if match is None: self.passed.append( @@ -534,8 +533,6 @@ def _get_build(response): def _container_type(line): """Returns the container type of a build.""" - if re.search("bioconda::", line): - return "bioconda" if line.startswith("conda"): return "conda" if line.startswith("https://containers") or line.startswith("https://depot"): From 2242cf4f0e6098dab8c11831bec62a3c521b6c0a Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Tue, 28 Mar 2023 09:44:35 -0400 Subject: [PATCH 110/246] Fix bug where lint messages are swapped --- nf_core/modules/lint/main_nf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 1caa67ac4f..0b82ad2bad 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -265,7 +265,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.passed.append( ( "deprecated_enable_conda", - f"Found deprecated parameter 'params.enable_conda' in the conda definition", + f"Deprecated parameter 'params.enable_conda' correctly not found in the conda definition", self.main_nf, ) ) @@ -273,7 +273,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append( ( "deprecated_enable_conda", - f"Deprecated parameter 'params.enable_conda' correctly not found in the conda definition", + f"Found deprecated parameter 'params.enable_conda' in the conda definition", self.main_nf, ) ) From dfe2057fe514ab73b5586e3e98a114f2161aa972 Mon Sep 17 00:00:00 2001 From: ash Date: Tue, 28 Mar 2023 14:55:09 +0100 Subject: [PATCH 111/246] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0dd174e3d..007901d476 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ ### Linting +- Read module lint configuration from `.nf-core.yml`, not `.nf-core-lint.yml` ([#2221](https://github.com/nf-core/tools/pull/2221)) + ### Modules - Add an `--empty-template` option to create a module without TODO statements or examples ([#2175](https://github.com/nf-core/tools/pull/2175) & [#2177](https://github.com/nf-core/tools/pull/2177)) From 65c2603bcc323c64e7d7428d783fc796d6d5b2c7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Fri, 20 Jan 2023 13:39:17 -0600 Subject: [PATCH 112/246] build: Replace requests-mock with responses --- requirements-dev.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 42ce780ce4..360f6ff87f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,6 @@ isort myst_parser pytest-cov pytest-datafiles -requests-mock +responses Sphinx sphinx-rtd-theme -requests_mock From ebbce34f52475353f2e6f6767e6a22bd0be637f7 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sat, 21 Jan 2023 14:48:01 -0600 Subject: [PATCH 113/246] refactor: Replace requests_mock calls with responses --- tests/modules/create.py | 18 +++++++++--------- tests/test_modules.py | 6 +++--- tests/test_subworkflows.py | 5 +++-- tests/utils.py | 10 +++++----- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/modules/create.py b/tests/modules/create.py index 61a8777b14..9e018e97b4 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -1,7 +1,7 @@ import os import pytest -import requests_mock +import responses import nf_core.modules from tests.utils import mock_api_calls @@ -9,8 +9,8 @@ def test_modules_create_succeed(self): """Succeed at creating the TrimGalore! module""" - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "trim-galore", "0.6.7") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" ) @@ -20,8 +20,8 @@ def test_modules_create_succeed(self): def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "trim-galore", "0.6.7") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" ) @@ -33,8 +33,8 @@ def test_modules_create_fail_exists(self): def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "fastqc", "0.11.9") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "fastqc", "0.11.9") module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "fastqc", "@author", "process_low", False, False ) @@ -45,8 +45,8 @@ def test_modules_create_nfcore_modules(self): def test_modules_create_nfcore_modules_subtool(self): """Create a tool/subtool module in a nf-core/modules clone""" - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "star", "2.8.10a") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "star", "2.8.10a") module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "star/index", "@author", "process_medium", False, False ) diff --git a/tests/test_modules.py b/tests/test_modules.py index c50c1f2ba8..81d3f69c49 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -6,7 +6,7 @@ import tempfile import unittest -import requests_mock +import responses import nf_core.create import nf_core.modules @@ -35,8 +35,8 @@ def create_modules_repo_dummy(tmp_dir): fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) # mock biocontainers and anaconda response - with requests_mock.Mocker() as mock: - mock_api_calls(mock, "bpipe", "0.9.11--hdfd78af_0") + with responses.RequestsMock() as rsps: + mock_api_calls(rsps, "bpipe", "0.9.11--hdfd78af_0") # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) module_create.create() diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 552a2ab176..c07e5d92d8 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -6,7 +6,7 @@ import tempfile import unittest -import requests_mock +import responses import nf_core.create import nf_core.modules @@ -30,7 +30,8 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - with requests_mock.Mocker() as mock: + # FIXME This mock isn't used + with responses.RequestsMock() as rsps: subworkflow_create = nf_core.subworkflows.SubworkflowCreate(root_dir, "test_subworkflow", "@author", True) subworkflow_create.create() diff --git a/tests/utils.py b/tests/utils.py index 77e94be464..cb48ac347e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -8,6 +8,8 @@ from contextlib import contextmanager from pathlib import Path +import responses + OLD_TRIMGALORE_SHA = "06348dffce2a732fc9e656bdc5c64c3e02d302cb" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" @@ -68,14 +70,13 @@ def set_wd(path: Path): os.chdir(start_wd) -def mock_api_calls(mock, module, version): +def mock_api_calls(rsps: responses.RequestsMock, module, version): """Mock biocontainers and anaconda api calls for module""" biocontainers_api_url = ( f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/{module}/versions/{module}-{version.split('--')[0]}" ) anaconda_api_url = f"https://api.anaconda.org/package/bioconda/{module}" anaconda_mock = { - "status_code": 200, "latest_version": version.split("--")[0], "summary": "", "doc_url": "", @@ -84,7 +85,6 @@ def mock_api_calls(mock, module, version): "license": "", } biocontainers_mock = { - "status_code": 200, "images": [ { "image_type": "Singularity", @@ -98,5 +98,5 @@ def mock_api_calls(mock, module, version): }, ], } - mock.register_uri("GET", anaconda_api_url, json=anaconda_mock) - mock.register_uri("GET", biocontainers_api_url, json=biocontainers_mock) + rsps.get(anaconda_api_url, json=anaconda_mock, status=200) + rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) From 887827284f86e7802132f0f72c73970f5eaa3a03 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 22 Jan 2023 14:48:53 -0600 Subject: [PATCH 114/246] refactor: Split biocontainer and anaconda mocks Responses actually throws an error if all the mocks aren't used. That allowed me to clean up a few tests that weren't actually using the mocks and find a few. --- tests/modules/create.py | 14 +++++++++----- tests/test_modules.py | 11 ++++------- tests/utils.py | 16 ++++++++++------ 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/modules/create.py b/tests/modules/create.py index 9e018e97b4..33764159f7 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -4,13 +4,14 @@ import responses import nf_core.modules -from tests.utils import mock_api_calls +from tests.utils import mock_anaconda_api_calls, mock_biocontainers_api_calls def test_modules_create_succeed(self): """Succeed at creating the TrimGalore! module""" with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "trim-galore", "0.6.7") + mock_anaconda_api_calls(rsps, "trim-galore", "0.6.7") + mock_biocontainers_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" ) @@ -21,7 +22,8 @@ def test_modules_create_succeed(self): def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "trim-galore", "0.6.7") + mock_anaconda_api_calls(rsps, "trim-galore", "0.6.7") + mock_biocontainers_api_calls(rsps, "trim-galore", "0.6.7") module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" ) @@ -34,7 +36,8 @@ def test_modules_create_fail_exists(self): def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "fastqc", "0.11.9") + mock_anaconda_api_calls(rsps, "fastqc", "0.11.9") + mock_biocontainers_api_calls(rsps, "fastqc", "0.11.9") module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "fastqc", "@author", "process_low", False, False ) @@ -46,7 +49,8 @@ def test_modules_create_nfcore_modules(self): def test_modules_create_nfcore_modules_subtool(self): """Create a tool/subtool module in a nf-core/modules clone""" with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "star", "2.8.10a") + mock_anaconda_api_calls(rsps, "star", "2.8.10a") + mock_biocontainers_api_calls(rsps, "star", "2.8.10a") module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "star/index", "@author", "process_medium", False, False ) diff --git a/tests/test_modules.py b/tests/test_modules.py index 81d3f69c49..8d4a9c1c23 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -18,7 +18,6 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, - mock_api_calls, ) @@ -34,12 +33,10 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - # mock biocontainers and anaconda response - with responses.RequestsMock() as rsps: - mock_api_calls(rsps, "bpipe", "0.9.11--hdfd78af_0") - # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules - module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) - module_create.create() + # FIXME Should use mock? + # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules + module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) + module_create.create() return root_dir diff --git a/tests/utils.py b/tests/utils.py index cb48ac347e..0dd60dd051 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -70,11 +70,8 @@ def set_wd(path: Path): os.chdir(start_wd) -def mock_api_calls(rsps: responses.RequestsMock, module, version): - """Mock biocontainers and anaconda api calls for module""" - biocontainers_api_url = ( - f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/{module}/versions/{module}-{version.split('--')[0]}" - ) +def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): + """Mock anaconda api calls for module""" anaconda_api_url = f"https://api.anaconda.org/package/bioconda/{module}" anaconda_mock = { "latest_version": version.split("--")[0], @@ -84,6 +81,14 @@ def mock_api_calls(rsps: responses.RequestsMock, module, version): "files": [{"version": version.split("--")[0]}], "license": "", } + rsps.get(anaconda_api_url, json=anaconda_mock, status=200) + + +def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module, version): + """Mock biocontainers api calls for module""" + biocontainers_api_url = ( + f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/{module}/versions/{module}-{version.split('--')[0]}" + ) biocontainers_mock = { "images": [ { @@ -98,5 +103,4 @@ def mock_api_calls(rsps: responses.RequestsMock, module, version): }, ], } - rsps.get(anaconda_api_url, json=anaconda_mock, status=200) rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) From 9c2537fb1898344fb6ad540ebccb698f316bf125 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 27 Mar 2023 12:17:16 -0500 Subject: [PATCH 115/246] test(modules): Turn off requests_cache for module creation --- tests/modules/create.py | 16 +++++++++++----- tests/test_modules.py | 15 +++++++++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/modules/create.py b/tests/modules/create.py index 33764159f7..98e498c1b0 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -1,6 +1,7 @@ import os import pytest +import requests_cache import responses import nf_core.modules @@ -15,7 +16,8 @@ def test_modules_create_succeed(self): module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", True, True, conda_name="trim-galore" ) - module_create.create() + with requests_cache.disabled(): + module_create.create() assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore.nf")) @@ -27,9 +29,11 @@ def test_modules_create_fail_exists(self): module_create = nf_core.modules.ModuleCreate( self.pipeline_dir, "trimgalore", "@author", "process_single", False, False, conda_name="trim-galore" ) - module_create.create() - with pytest.raises(UserWarning) as excinfo: + with requests_cache.disabled(): module_create.create() + with pytest.raises(UserWarning) as excinfo: + with requests_cache.disabled(): + module_create.create() assert "Module file exists already" in str(excinfo.value) @@ -41,7 +45,8 @@ def test_modules_create_nfcore_modules(self): module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "fastqc", "@author", "process_low", False, False ) - module_create.create() + with requests_cache.disabled(): + module_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "fastqc", "main.nf")) assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "fastqc", "main.nf")) @@ -54,6 +59,7 @@ def test_modules_create_nfcore_modules_subtool(self): module_create = nf_core.modules.ModuleCreate( self.nfcore_modules, "star/index", "@author", "process_medium", False, False ) - module_create.create() + with requests_cache.disabled(): + module_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "star", "index", "main.nf")) assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "star", "index", "main.nf")) diff --git a/tests/test_modules.py b/tests/test_modules.py index 8d4a9c1c23..21c003112e 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -6,6 +6,7 @@ import tempfile import unittest +import requests_cache import responses import nf_core.create @@ -18,6 +19,8 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, + mock_anaconda_api_calls, + mock_biocontainers_api_calls, ) @@ -33,10 +36,14 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - # FIXME Should use mock? - # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules - module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) - module_create.create() + # mock biocontainers and anaconda response + with responses.RequestsMock() as rsps: + mock_anaconda_api_calls(rsps, "bpipe", "0.9.11--hdfd78af_0") + mock_biocontainers_api_calls(rsps, "bpipe", "0.9.11--hdfd78af_0") + # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules + module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_single", False, False) + with requests_cache.disabled(): + module_create.create() return root_dir From cb60b1d2e10f80f2290f06f0285624dfc9b57dde Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 27 Mar 2023 18:32:18 -0500 Subject: [PATCH 116/246] test(subworkflows): Remove unused mock --- tests/test_subworkflows.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index c07e5d92d8..c5d768ab10 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -30,10 +30,9 @@ def create_modules_repo_dummy(tmp_dir): with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - # FIXME This mock isn't used - with responses.RequestsMock() as rsps: - subworkflow_create = nf_core.subworkflows.SubworkflowCreate(root_dir, "test_subworkflow", "@author", True) - subworkflow_create.create() + # TODO Add a mock here + subworkflow_create = nf_core.subworkflows.SubworkflowCreate(root_dir, "test_subworkflow", "@author", True) + subworkflow_create.create() return root_dir From f4747bf32559fbe75a356c13ee70a2f43eba6d9a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 27 Mar 2023 21:50:44 -0500 Subject: [PATCH 117/246] chore: Add #2165 to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2102b1c5a..4797f020e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - `nf-core modules/subworkflows info` now prints the include statement for the module/subworkflow ([#2182](https://github.com/nf-core/tools/pull/2182)). - Add the Nextflow version to Gitpod container matching the minimal Nextflow version for nf-core (according to `nextflow.config`) ([#2196](https://github.com/nf-core/tools/pull/2196)) - Use `nfcore/gitpod:dev` container in the dev branch ([#2196](https://github.com/nf-core/tools/pull/2196)) +- Replace requests_mock with responses in test mocks ([#2165](https://github.com/nf-core/tools/pull/2165)). ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] From e8baf1928c171f6d7ee0aeffa8ae681c9fc0260e Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Tue, 28 Mar 2023 15:17:05 -0400 Subject: [PATCH 118/246] bugfix --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 0b82ad2bad..dd565c9a4e 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -488,7 +488,7 @@ def _fix_module_version(self, current_version, latest_version, singularity_tag, for line in lines: l = line.strip(" '\"") build_type = _container_type(l) - if build_type == "bioconda": + if build_type == "conda": new_lines.append(re.sub(rf"{current_version}", f"{latest_version}", line)) elif build_type in ("singularity", "docker"): # Check that the new url is valid From 7e40458a471e62f158d3a9bd8b44a047b5924d37 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Mon, 27 Mar 2023 14:19:42 +0200 Subject: [PATCH 119/246] Test subworkflow yml dumper --- tests/subworkflows/create_test_yml.py | 24 ++++++++++++++++++++++++ tests/test_subworkflows.py | 3 +++ 2 files changed, 27 insertions(+) create mode 100644 tests/subworkflows/create_test_yml.py diff --git a/tests/subworkflows/create_test_yml.py b/tests/subworkflows/create_test_yml.py new file mode 100644 index 0000000000..1654117bb6 --- /dev/null +++ b/tests/subworkflows/create_test_yml.py @@ -0,0 +1,24 @@ +import os +from unittest import mock + +import pytest + +import nf_core.subworkflows + +from ..utils import with_temporary_folder + + +@with_temporary_folder +def test_subworkflows_custom_yml_dumper(self, out_dir): + """Try to create a yml file with the custom yml dumper""" + yml_output_path = os.path.join(out_dir, "test.yml") + meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( + subworkflow="test/tool", + directory=self.pipeline_dir, + test_yml_output_path=yml_output_path, + no_prompts=True, + ) + meta_builder.test_yml_output_path = yml_output_path + meta_builder.tests = [{"testname": "myname"}] + meta_builder.print_test_yml() + assert os.path.isfile(yml_output_path) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index c5d768ab10..b11af31325 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -93,6 +93,9 @@ def tearDown(self): test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) + from .subworkflows.create_test_yml import ( + test_subworkflows_custom_yml_dumper, + ) from .subworkflows.info import ( test_subworkflows_info_in_modules_repo, test_subworkflows_info_local, From 488b25e1498f72675dec264dbb15449be0e00de9 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Mon, 27 Mar 2023 14:34:21 +0200 Subject: [PATCH 120/246] Test create_test_file_dict --- tests/subworkflows/create_test_yml.py | 16 ++++++++++++++++ tests/test_subworkflows.py | 1 + 2 files changed, 17 insertions(+) diff --git a/tests/subworkflows/create_test_yml.py b/tests/subworkflows/create_test_yml.py index 1654117bb6..ff4f03c1fd 100644 --- a/tests/subworkflows/create_test_yml.py +++ b/tests/subworkflows/create_test_yml.py @@ -22,3 +22,19 @@ def test_subworkflows_custom_yml_dumper(self, out_dir): meta_builder.tests = [{"testname": "myname"}] meta_builder.print_test_yml() assert os.path.isfile(yml_output_path) + + +@with_temporary_folder +def test_subworkflows_test_file_dict(self, test_file_dir): + """Create dict of test files and create md5 sums""" + meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( + subworkflow="test/tool", + directory=self.pipeline_dir, + test_yml_output_path="./", + no_prompts=True, + ) + with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: + fh.write("this line is just for testing") + test_files = meta_builder.create_test_file_dict(test_file_dir) + assert len(test_files) == 1 + assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index b11af31325..50f674f29e 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -95,6 +95,7 @@ def tearDown(self): ) from .subworkflows.create_test_yml import ( test_subworkflows_custom_yml_dumper, + test_subworkflows_test_file_dict, ) from .subworkflows.info import ( test_subworkflows_info_in_modules_repo, From c404c8717d98e44a703575a754a298b1e09d35da Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Mon, 27 Mar 2023 15:06:17 +0200 Subject: [PATCH 121/246] Test get_md5_sums --- nf_core/subworkflows/test_yml_builder.py | 4 ++-- tests/subworkflows/create_test_yml.py | 19 +++++++++++++++++++ tests/test_subworkflows.py | 1 + 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index 39877ca241..384561bf68 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -195,7 +195,7 @@ def build_single_test(self, entry_point): ).strip() ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] - ep_test["files"] = self.get_md5_sums(entry_point, ep_test["command"]) + ep_test["files"] = self.get_md5_sums(ep_test["command"]) return ep_test @@ -272,7 +272,7 @@ def create_test_file_dict(self, results_dir, is_repeat=False): return test_files - def get_md5_sums(self, entry_point, command, results_dir=None, results_dir_repeat=None): + def get_md5_sums(self, command, results_dir=None, results_dir_repeat=None): """ Recursively go through directories and subdirectories and generate tuples of (, ) diff --git a/tests/subworkflows/create_test_yml.py b/tests/subworkflows/create_test_yml.py index ff4f03c1fd..33ad59d655 100644 --- a/tests/subworkflows/create_test_yml.py +++ b/tests/subworkflows/create_test_yml.py @@ -38,3 +38,22 @@ def test_subworkflows_test_file_dict(self, test_file_dir): test_files = meta_builder.create_test_file_dict(test_file_dir) assert len(test_files) == 1 assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" + + +@with_temporary_folder +def test_subworkflows_create_test_yml_get_md5(self, test_file_dir): + """Get md5 sums from a dummy output""" + meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( + subworkflow="test/tool", + directory=self.pipeline_dir, + test_yml_output_path="./", + no_prompts=True, + ) + with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: + fh.write("this line is just for testing") + test_files = meta_builder.get_md5_sums( + command="dummy", + results_dir=test_file_dir, + results_dir_repeat=test_file_dir, + ) + assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 50f674f29e..f888c982de 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -96,6 +96,7 @@ def tearDown(self): from .subworkflows.create_test_yml import ( test_subworkflows_custom_yml_dumper, test_subworkflows_test_file_dict, + test_subworkflows_create_test_yml_get_md5, ) from .subworkflows.info import ( test_subworkflows_info_in_modules_repo, From 4db16799abdddccf61ca058fb4bd68d88369dccb Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 28 Mar 2023 13:29:09 +0200 Subject: [PATCH 122/246] Test scrape_workflow_entry_points --- nf_core/subworkflow-template/tests/main.nf | 2 +- nf_core/subworkflows/test_yml_builder.py | 2 +- tests/subworkflows/create_test_yml.py | 16 ++++++++++++++++ tests/test_subworkflows.py | 1 + 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/nf_core/subworkflow-template/tests/main.nf b/nf_core/subworkflow-template/tests/main.nf index 2bf63da2f3..f8c9b10dcb 100644 --- a/nf_core/subworkflow-template/tests/main.nf +++ b/nf_core/subworkflow-template/tests/main.nf @@ -4,7 +4,7 @@ nextflow.enable.dsl = 2 include { {{ subworkflow_name|upper }} } from '../../../../subworkflows/{{ org }}/{{ subworkflow_dir }}/main.nf' -workflow test_{{ subworkflow_name }} { +workflow test_{{ component_name_underscore }} { {% if has_meta %} input = [ [ id:'test' ], // meta map diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py index 384561bf68..2ad50d4e25 100644 --- a/nf_core/subworkflows/test_yml_builder.py +++ b/nf_core/subworkflows/test_yml_builder.py @@ -139,7 +139,7 @@ def scrape_workflow_entry_points(self): if match: self.entry_points.append(match.group(1)) if len(self.entry_points) == 0: - raise UserWarning("No workflow entry points found in 'self.module_test_main'") + raise UserWarning(f"No workflow entry points found in '{self.subworkflow_test_main}'") def build_all_tests(self): """ diff --git a/tests/subworkflows/create_test_yml.py b/tests/subworkflows/create_test_yml.py index 33ad59d655..ac06a45e68 100644 --- a/tests/subworkflows/create_test_yml.py +++ b/tests/subworkflows/create_test_yml.py @@ -57,3 +57,19 @@ def test_subworkflows_create_test_yml_get_md5(self, test_file_dir): results_dir_repeat=test_file_dir, ) assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" + + +def test_subworkflows_create_test_yml_entry_points(self): + """Test extracting test entry points from a main.nf file""" + subworkflow = "test_subworkflow" + meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( + subworkflow=f"{subworkflow}/test", + directory=self.pipeline_dir, + test_yml_output_path="./", + no_prompts=True, + ) + meta_builder.subworkflow_test_main = os.path.join( + self.nfcore_modules, "tests", "subworkflows", "nf-core", subworkflow, "main.nf" + ) + meta_builder.scrape_workflow_entry_points() + assert meta_builder.entry_points[0] == f"test_{subworkflow}" diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index f888c982de..663d51a474 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -97,6 +97,7 @@ def tearDown(self): test_subworkflows_custom_yml_dumper, test_subworkflows_test_file_dict, test_subworkflows_create_test_yml_get_md5, + test_subworkflows_create_test_yml_entry_points, ) from .subworkflows.info import ( test_subworkflows_info_in_modules_repo, From e7bda6db3da0638e62baa5ab4d845d0bb60dac38 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 28 Mar 2023 14:16:12 +0200 Subject: [PATCH 123/246] Test check_inputs --- tests/subworkflows/create_test_yml.py | 20 ++++++++++++++++++++ tests/test_subworkflows.py | 1 + 2 files changed, 21 insertions(+) diff --git a/tests/subworkflows/create_test_yml.py b/tests/subworkflows/create_test_yml.py index ac06a45e68..a2f3bb321d 100644 --- a/tests/subworkflows/create_test_yml.py +++ b/tests/subworkflows/create_test_yml.py @@ -73,3 +73,23 @@ def test_subworkflows_create_test_yml_entry_points(self): ) meta_builder.scrape_workflow_entry_points() assert meta_builder.entry_points[0] == f"test_{subworkflow}" + + +def test_subworkflows_create_test_yml_check_inputs(self): + """Test the check_inputs() function - raise UserWarning because test.yml exists""" + cwd = os.getcwd() + os.chdir(self.nfcore_modules) + subworkflow = "test_subworkflow" + meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( + subworkflow=f"{subworkflow}", + directory=self.pipeline_dir, + test_yml_output_path="./", + no_prompts=True, + ) + meta_builder.subworkflow_test_main = os.path.join( + self.nfcore_modules, "tests", "subworkflows", "nf-core", subworkflow, "main.nf" + ) + with pytest.raises(UserWarning) as excinfo: + meta_builder.check_inputs() + os.chdir(cwd) + assert "Test YAML file already exists!" in str(excinfo.value) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 663d51a474..aa8be60863 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -98,6 +98,7 @@ def tearDown(self): test_subworkflows_test_file_dict, test_subworkflows_create_test_yml_get_md5, test_subworkflows_create_test_yml_entry_points, + test_subworkflows_create_test_yml_check_inputs, ) from .subworkflows.info import ( test_subworkflows_info_in_modules_repo, From 36aab3837db4ec020245923d6239115891b3b742 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 28 Mar 2023 14:30:39 +0200 Subject: [PATCH 124/246] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4797f020e0..21f41cef7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,10 @@ - Use `nfcore/gitpod:dev` container in the dev branch ([#2196](https://github.com/nf-core/tools/pull/2196)) - Replace requests_mock with responses in test mocks ([#2165](https://github.com/nf-core/tools/pull/2165)). +### Bug fixes, maintenance and tests + +- Add tests for `nf-core subworkflows create-test-yml` ([#2219](https://github.com/nf-core/tools/pull/2219)) + ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] ### Template From a661087f0d1a5e1040071a3b9a9ec4adb1bbf13e Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Tue, 28 Mar 2023 16:16:32 +0200 Subject: [PATCH 125/246] Fix linting --- CHANGELOG.md | 2 +- README.md | 4 ++-- tests/test_subworkflows.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21f41cef7e..fc18f9ccee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ ### Bug fixes, maintenance and tests -- Add tests for `nf-core subworkflows create-test-yml` ([#2219](https://github.com/nf-core/tools/pull/2219)) +- Add tests for `nf-core subworkflows create-test-yml` ([#2219](https://github.com/nf-core/tools/pull/2219)) ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] diff --git a/README.md b/README.md index 63bf235bc7..2fe09e80d2 100644 --- a/README.md +++ b/README.md @@ -612,7 +612,7 @@ The graphical interface is oganzised in groups and within the groups the single Now you can start to change the parameter itself. The `ID` of a new parameter should be defined in small letters without whitespaces. The description is a short free text explanation about the parameter, that appears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. Usually, they contain a small paragraph about the parameter settings or a used datasource, like databases or references. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For integers you can define a min and max value, and for strings the file extension can be specified. -The `type` field is one of the most important points in your pipeline schema, since it defines the datatype of your input and how it will be interpreted. This allows extensive testing prior to starting the pipeline. +The `type` field is one of the most important points in your pipeline schema, since it defines the datatype of your input and how it will be interpreted. This allows extensive testing prior to starting the pipeline. The basic datatypes for a pipeline schema are: @@ -621,7 +621,7 @@ The basic datatypes for a pipeline schema are: - `integer` - `boolean` -For the `string` type you have three different options in the settings (nut icon): `enumerated values`, `pattern` and `format`. The first option, `enumerated values`, allows you to specify a list of specific input values. The list has to be separated with a pipe. The `pattern` and `format` settings can depend on each other. The `format` has to be either a directory or a file path. Depending on the `format` setting selected, specifying the `pattern` setting can be the most efficient and time saving option, especially for `file paths`. The `number` and `integer` types share the same settings. Similarly to `string`, there is an `enumerated values` option with the possibility of specifying a `min` and `max` value. For the `boolean` there is no further settings and the default value is usually `false`. The `boolean` value can be switched to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. +For the `string` type you have three different options in the settings (nut icon): `enumerated values`, `pattern` and `format`. The first option, `enumerated values`, allows you to specify a list of specific input values. The list has to be separated with a pipe. The `pattern` and `format` settings can depend on each other. The `format` has to be either a directory or a file path. Depending on the `format` setting selected, specifying the `pattern` setting can be the most efficient and time saving option, especially for `file paths`. The `number` and `integer` types share the same settings. Similarly to `string`, there is an `enumerated values` option with the possibility of specifying a `min` and `max` value. For the `boolean` there is no further settings and the default value is usually `false`. The `boolean` value can be switched to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. After filling the schema, click on the `Finished` button in the top right corner, this will automatically update your `nextflow_schema.json`. If this is not working, the schema can be copied from the graphical interface and pasted in your `nextflow_schema.json` file. diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index aa8be60863..19a090f7f0 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -94,11 +94,11 @@ def tearDown(self): test_subworkflows_create_succeed, ) from .subworkflows.create_test_yml import ( + test_subworkflows_create_test_yml_check_inputs, + test_subworkflows_create_test_yml_entry_points, + test_subworkflows_create_test_yml_get_md5, test_subworkflows_custom_yml_dumper, test_subworkflows_test_file_dict, - test_subworkflows_create_test_yml_get_md5, - test_subworkflows_create_test_yml_entry_points, - test_subworkflows_create_test_yml_check_inputs, ) from .subworkflows.info import ( test_subworkflows_info_in_modules_repo, From 41837df4b7ebb45e83bd6dd005405fe7dd36a49a Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 29 Mar 2023 10:19:59 +0200 Subject: [PATCH 126/246] =?UTF-8?q?Switch=20from=20`os.path`=C2=A0to=20`Pa?= =?UTF-8?q?th`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- tests/subworkflows/create_test_yml.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/subworkflows/create_test_yml.py b/tests/subworkflows/create_test_yml.py index a2f3bb321d..ed5715832f 100644 --- a/tests/subworkflows/create_test_yml.py +++ b/tests/subworkflows/create_test_yml.py @@ -1,6 +1,6 @@ import os from unittest import mock - +from pathlib import Path import pytest import nf_core.subworkflows @@ -11,7 +11,7 @@ @with_temporary_folder def test_subworkflows_custom_yml_dumper(self, out_dir): """Try to create a yml file with the custom yml dumper""" - yml_output_path = os.path.join(out_dir, "test.yml") + yml_output_path = Path(out_dir, "test.yml") meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( subworkflow="test/tool", directory=self.pipeline_dir, @@ -21,7 +21,7 @@ def test_subworkflows_custom_yml_dumper(self, out_dir): meta_builder.test_yml_output_path = yml_output_path meta_builder.tests = [{"testname": "myname"}] meta_builder.print_test_yml() - assert os.path.isfile(yml_output_path) + assert Path(yml_output_path).is_file() @with_temporary_folder @@ -33,7 +33,7 @@ def test_subworkflows_test_file_dict(self, test_file_dir): test_yml_output_path="./", no_prompts=True, ) - with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: + with open(Path(test_file_dir, "test_file.txt"), "w") as fh: fh.write("this line is just for testing") test_files = meta_builder.create_test_file_dict(test_file_dir) assert len(test_files) == 1 @@ -68,7 +68,7 @@ def test_subworkflows_create_test_yml_entry_points(self): test_yml_output_path="./", no_prompts=True, ) - meta_builder.subworkflow_test_main = os.path.join( + meta_builder.subworkflow_test_main = Path( self.nfcore_modules, "tests", "subworkflows", "nf-core", subworkflow, "main.nf" ) meta_builder.scrape_workflow_entry_points() @@ -86,7 +86,7 @@ def test_subworkflows_create_test_yml_check_inputs(self): test_yml_output_path="./", no_prompts=True, ) - meta_builder.subworkflow_test_main = os.path.join( + meta_builder.subworkflow_test_main = Path( self.nfcore_modules, "tests", "subworkflows", "nf-core", subworkflow, "main.nf" ) with pytest.raises(UserWarning) as excinfo: From ef4738b26d3d79516cfa6eab823bdd59f076d40a Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 29 Mar 2023 10:23:06 +0200 Subject: [PATCH 127/246] Switch from `os.path` to `Path` for modules too --- tests/modules/create_test_yml.py | 12 ++++++------ tests/subworkflows/create_test_yml.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/modules/create_test_yml.py b/tests/modules/create_test_yml.py index d444ff841a..950737062b 100644 --- a/tests/modules/create_test_yml.py +++ b/tests/modules/create_test_yml.py @@ -11,19 +11,19 @@ @with_temporary_folder def test_modules_custom_yml_dumper(self, out_dir): """Try to create a yml file with the custom yml dumper""" - yml_output_path = os.path.join(out_dir, "test.yml") + yml_output_path = Path(out_dir, "test.yml") meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", self.pipeline_dir, False, "./", False, True) meta_builder.test_yml_output_path = yml_output_path meta_builder.tests = [{"testname": "myname"}] meta_builder.print_test_yml() - assert os.path.isfile(yml_output_path) + assert Path(yml_output_path).is_file() @with_temporary_folder def test_modules_test_file_dict(self, test_file_dir): """Create dict of test files and create md5 sums""" meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", self.pipeline_dir, False, "./", False, True) - with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: + with open(Path(test_file_dir, "test_file.txt"), "w") as fh: fh.write("this line is just for testing") test_files = meta_builder.create_test_file_dict(test_file_dir) assert len(test_files) == 1 @@ -34,7 +34,7 @@ def test_modules_test_file_dict(self, test_file_dir): def test_modules_create_test_yml_get_md5(self, test_file_dir): """Get md5 sums from a dummy output""" meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", self.pipeline_dir, False, "./", False, True) - with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: + with open(Path(test_file_dir, "test_file.txt"), "w") as fh: fh.write("this line is just for testing") test_files = meta_builder.get_md5_sums(command="dummy", results_dir=test_file_dir, results_dir_repeat=test_file_dir) assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" @@ -43,7 +43,7 @@ def test_modules_create_test_yml_get_md5(self, test_file_dir): def test_modules_create_test_yml_entry_points(self): """Test extracting test entry points from a main.nf file""" meta_builder = nf_core.modules.ModulesTestYmlBuilder("bpipe/test", self.pipeline_dir, False, "./", False, True) - meta_builder.module_test_main = os.path.join( + meta_builder.module_test_main = Path( self.nfcore_modules, "tests", "modules", "nf-core", "bpipe", "test", "main.nf" ) meta_builder.scrape_workflow_entry_points() @@ -55,7 +55,7 @@ def test_modules_create_test_yml_check_inputs(self): cwd = os.getcwd() os.chdir(self.nfcore_modules) meta_builder = nf_core.modules.ModulesTestYmlBuilder("bpipe/test", ".", False, "./", False, True) - meta_builder.module_test_main = os.path.join(self.nfcore_modules, "tests", "modules", "bpipe", "test", "main.nf") + meta_builder.module_test_main = Path(self.nfcore_modules, "tests", "modules", "bpipe", "test", "main.nf") with pytest.raises(UserWarning) as excinfo: meta_builder.check_inputs() os.chdir(cwd) diff --git a/tests/subworkflows/create_test_yml.py b/tests/subworkflows/create_test_yml.py index ed5715832f..a195f74d60 100644 --- a/tests/subworkflows/create_test_yml.py +++ b/tests/subworkflows/create_test_yml.py @@ -49,7 +49,7 @@ def test_subworkflows_create_test_yml_get_md5(self, test_file_dir): test_yml_output_path="./", no_prompts=True, ) - with open(os.path.join(test_file_dir, "test_file.txt"), "w") as fh: + with open(Path(test_file_dir, "test_file.txt"), "w") as fh: fh.write("this line is just for testing") test_files = meta_builder.get_md5_sums( command="dummy", From 8d81588af485759eda1549e62fc55b1e43090027 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 29 Mar 2023 10:37:06 +0200 Subject: [PATCH 128/246] Lint code (isort, black) --- tests/modules/create_test_yml.py | 4 +--- tests/subworkflows/create_test_yml.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/modules/create_test_yml.py b/tests/modules/create_test_yml.py index 950737062b..243378af78 100644 --- a/tests/modules/create_test_yml.py +++ b/tests/modules/create_test_yml.py @@ -43,9 +43,7 @@ def test_modules_create_test_yml_get_md5(self, test_file_dir): def test_modules_create_test_yml_entry_points(self): """Test extracting test entry points from a main.nf file""" meta_builder = nf_core.modules.ModulesTestYmlBuilder("bpipe/test", self.pipeline_dir, False, "./", False, True) - meta_builder.module_test_main = Path( - self.nfcore_modules, "tests", "modules", "nf-core", "bpipe", "test", "main.nf" - ) + meta_builder.module_test_main = Path(self.nfcore_modules, "tests", "modules", "nf-core", "bpipe", "test", "main.nf") meta_builder.scrape_workflow_entry_points() assert meta_builder.entry_points[0] == "test_bpipe_test" diff --git a/tests/subworkflows/create_test_yml.py b/tests/subworkflows/create_test_yml.py index a195f74d60..40384b420f 100644 --- a/tests/subworkflows/create_test_yml.py +++ b/tests/subworkflows/create_test_yml.py @@ -1,6 +1,7 @@ import os -from unittest import mock from pathlib import Path +from unittest import mock + import pytest import nf_core.subworkflows From a9db16da857c2d1132ae68885fb6a06dfb0b0390 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Wed, 29 Mar 2023 11:16:24 +0100 Subject: [PATCH 129/246] Remove unused "log" args Two methods WorkflowMain::help and WorkflowMain::paramsSummaryLog use a "log" argument that is not used in the method body. This commit removes those arguments and their references. --- nf_core/pipeline-template/lib/WorkflowMain.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy index 05db418b2d..6b5302be66 100755 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ b/nf_core/pipeline-template/lib/WorkflowMain.groovy @@ -21,7 +21,7 @@ class WorkflowMain { // // Generate help string // - public static String help(workflow, params, log) { + public static String help(workflow, params) { {% if igenomes -%} def command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" {% else -%} @@ -38,7 +38,7 @@ class WorkflowMain { // // Generate parameter summary log string // - public static String paramsSummaryLog(workflow, params, log) { + public static String paramsSummaryLog(workflow, params) { def summary_log = '' summary_log += NfcoreTemplate.logo(workflow, params.monochrome_logs) summary_log += NfcoreSchema.paramsSummaryLog(workflow, params) @@ -53,7 +53,7 @@ class WorkflowMain { public static void initialise(workflow, params, log) { // Print help to screen if required if (params.help) { - log.info help(workflow, params, log) + log.info help(workflow, params) System.exit(0) } @@ -65,7 +65,7 @@ class WorkflowMain { } // Print parameter summary log to screen - log.info paramsSummaryLog(workflow, params, log) + log.info paramsSummaryLog(workflow, params) // Validate workflow parameters via the JSON schema if (params.validate_params) { From 62c87314c4f28eb1ddc5a155593ca6f1442078d4 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 29 Mar 2023 11:53:00 +0100 Subject: [PATCH 130/246] Extra error handling --- nf_core/modules/patch.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 8b8068bafb..4dd4ab12e3 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -174,7 +174,6 @@ def remove(self, module): patch_filename = f"{module.replace('/', '-')}.diff" module_relpath = Path("modules", module_dir, module) patch_relpath = Path(module_relpath, patch_filename) - module_current_dir = Path(self.dir, module_relpath) patch_path = Path(self.dir, patch_relpath) if patch_path.exists(): @@ -189,9 +188,12 @@ def remove(self, module): temp_module_dir = self.modules_json.try_apply_patch_reverse( module, self.modules_repo.repo_path, patch_relpath, module_relpath ) - for file in Path(temp_module_dir).glob("*"): - file.rename(module_relpath.joinpath(file.name)) - os.rmdir(temp_module_dir) + try: + for file in Path(temp_module_dir).glob("*"): + file.rename(module_relpath.joinpath(file.name)) + os.rmdir(temp_module_dir) + except Exception as err: + raise UserWarning(f"There was a problem reverting the patched file: {err}") # Remove patch file patch_path.unlink() From 800e1d2003d3df8fdbcc014da60058e9c9c4cc5a Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 29 Mar 2023 12:35:22 +0100 Subject: [PATCH 131/246] Check if the reverted module code matches the known sha --- nf_core/modules/patch.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 4dd4ab12e3..8c3f0cf382 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -195,8 +195,14 @@ def remove(self, module): except Exception as err: raise UserWarning(f"There was a problem reverting the patched file: {err}") - # Remove patch file + log.info(f"Patch for {module} reverted!") + # Remove patch file if we could revert the patch patch_path.unlink() - # Write changes to modules.json + # Write changes to module.json self.modules_json.remove_patch_entry(module, self.modules_repo.remote_url, module_dir) - log.info(f"Patch for {module} reverted!") + + if not all(self.modules_repo.module_files_identical(module, module_relpath, module_version).values()): + log.error( + f"Module files do not appear to match the remote for the commit sha in the 'module.json': {module_version}\n" + f"Recommend reinstalling with 'nf-core modules install --force --sha {module_version} {module}' " + ) From 20f82c08d7f7a529f33cd0af36da3e4dd7621393 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Wed, 29 Mar 2023 12:04:10 +0200 Subject: [PATCH 132/246] Fix changelog [skip ci] --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc18f9ccee..bb12675044 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Fixing problem when a module included in a subworkflow had a name change from TOOL to TOOL/SUBTOOL ([#2177](https://github.com/nf-core/tools/pull/2177)) - Fix `nf-core subworkflows test` not running subworkflow tests ([#2181](https://github.com/nf-core/tools/pull/2181)) +- Add tests for `nf-core subworkflows create-test-yml` ([#2219](https://github.com/nf-core/tools/pull/2219)) ### General @@ -29,10 +30,6 @@ - Use `nfcore/gitpod:dev` container in the dev branch ([#2196](https://github.com/nf-core/tools/pull/2196)) - Replace requests_mock with responses in test mocks ([#2165](https://github.com/nf-core/tools/pull/2165)). -### Bug fixes, maintenance and tests - -- Add tests for `nf-core subworkflows create-test-yml` ([#2219](https://github.com/nf-core/tools/pull/2219)) - ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] ### Template From 46fcedf73ca429bc855174fe54e4a8a3faf8b12d Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 29 Mar 2023 14:44:25 +0100 Subject: [PATCH 133/246] Lint all labels and check for conflicting or non-standard --- nf_core/modules/lint/main_nf.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index d44fe90f1e..6a75667a75 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -236,25 +236,30 @@ def check_process_section(self, lines, fix_version, progress_bar): # Check that process labels are correct correct_process_labels = ["process_single", "process_low", "process_medium", "process_high", "process_long"] - process_label = [l for l in lines if l.lstrip().startswith("label")] - if len(process_label) > 0: - try: - process_label = re.search("process_[A-Za-z]+", process_label[0]).group(0) - except AttributeError: - process_label = re.search("'([A-Za-z_-]+)'", process_label[0]).group(0) - finally: - if not process_label in correct_process_labels: + all_labels = [l for l in lines if l.lstrip().startswith("label ")] + bad_labels = [] + good_labels = [] + if len(all_labels) > 0: + for label in all_labels: + label = re.match("^label\s+([a-zA-Z0-9_-]+)", label).group(1) + if label not in correct_process_labels: + bad_labels.append(label) + else: + good_labels.append(label) + if len(good_labels) > 1: self.warned.append( ( "process_standard_label", - f"Process label ({process_label}) is not among standard labels: `{'`,`'.join(correct_process_labels)}`", + f"Conflicting process labels found: `{'`,`'.join(good_labels)}`", self.main_nf, ) ) - else: + elif len(good_labels) == 1: self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) + else: + self.warned.append(("process_standard_label", "Standard process label not found", self.main_nf)) else: - self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf)) + self.warned.append(("process_standard_label", "Process label not specified", self.main_nf)) for i, l in enumerate(lines): url = None if _container_type(l) == "bioconda": From 790a9b86de92de4703bc65a33ce9ee9f97b53b2f Mon Sep 17 00:00:00 2001 From: ash Date: Wed, 29 Mar 2023 10:47:34 +0100 Subject: [PATCH 134/246] Don't require an argument for `nf-core schema lint` --- README.md | 4 ++-- nf_core/__main__.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2fe09e80d2..8235bf4386 100644 --- a/README.md +++ b/README.md @@ -634,13 +634,13 @@ It's important to change the default value of a parameter in the `nextflow.confi The pipeline schema is linted as part of the main pipeline `nf-core lint` command, however sometimes it can be useful to quickly check the syntax of the JSONSchema without running a full lint run. -Usage is `nf-core schema lint `, eg: +Usage is `nf-core schema lint ` (defaulting to `nextflow_schema.json`), eg: -![`nf-core schema lint nextflow_schema.json`](docs/images/nf-core-schema-lint.svg) +![`nf-core schema lint`](docs/images/nf-core-schema-lint.svg) ## Bumping a pipeline version number diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 2cf3805902..a801df4266 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1398,7 +1398,7 @@ def build(dir, no_prompts, web_only, url): # nf-core schema lint @schema.command() -@click.argument("schema_path", type=click.Path(exists=True), required=True, metavar="") +@click.argument("schema_path", type=click.Path(exists=True), default="nextflow_schema.json", metavar="") def lint(schema_path): """ Check that a given pipeline schema is valid. @@ -1408,6 +1408,8 @@ def lint(schema_path): This function runs as part of the nf-core lint command, this is a convenience command that does just the schema linting nice and quickly. + + If no schema path is provided, "nextflow_schema.json" will be used (if it exists). """ schema_obj = nf_core.schema.PipelineSchema() try: From d53259ea6257adb731e14821f08682d5dfd9967d Mon Sep 17 00:00:00 2001 From: ash Date: Wed, 29 Mar 2023 12:26:00 +0100 Subject: [PATCH 135/246] Add basic tests for `nf-core schema lint` --- tests/test_cli.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2bd8af5c59..58c4525a76 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -350,3 +350,20 @@ def test_lint_log_user_warning(self, mock_lint, mock_is_pipeline): assert result.exit_code == 1 assert error_txt in captured_logs.output[-1] assert captured_logs.records[-1].levelname == "ERROR" + + @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") + def test_schema_lint(self, mock_get_schema_path): + """Test nf-core schema lint defaults to nextflow_schema.json""" + cmd = ["schema", "lint"] + result = self.invoke_cli(cmd) + assert mock_get_schema_path.called_with("nextflow_schema.json") + assert "nextflow_schema.json" in result.output + + @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") + def test_schema_lint_filename(self, mock_get_schema_path): + """Test nf-core schema lint accepts a filename""" + cmd = ["schema", "lint", "some_other_filename"] + result = self.invoke_cli(cmd) + assert mock_get_schema_path.called_with("some_other_filename") + assert "some_other_filename" in result.output + assert "nextflow_schema.json" not in result.output From b4e22d363bcd9ddbcc62fd495e6c5a94c2bad0eb Mon Sep 17 00:00:00 2001 From: ash Date: Wed, 29 Mar 2023 14:52:43 +0100 Subject: [PATCH 136/246] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df9a14d03d..40d62f1345 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ### Linting - Read module lint configuration from `.nf-core.yml`, not `.nf-core-lint.yml` ([#2221](https://github.com/nf-core/tools/pull/2221)) +- `nf-core schema lint` now defaults to linting `nextflow_schema.json` if no filename is provided ([#2225](https://github.com/nf-core/tools/pull/2225)) ### Modules From 203380750f229cfc96c6201a197d913fca3b6b1f Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Wed, 29 Mar 2023 15:49:43 -0400 Subject: [PATCH 137/246] Add check for multiple remotes with the same org --- nf_core/components/install.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 850cc9d60f..fc88f36ce2 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -58,6 +58,9 @@ def install(self, component, silent=False): if not silent: modules_json.check_up_to_date() + # Verify that the remote repo's org_path does not match the org_path of any alternate repo among the installed modules + self.check_alternate_remotes(modules_json) + # Verify SHA if not self.modules_repo.verify_sha(self.prompt, self.sha): return False @@ -264,3 +267,24 @@ def clean_modules_json(self, component, modules_repo, modules_json): self.component_type, component, repo_to_remove, modules_repo.repo_path ) return component_values["installed_by"] + + def check_alternate_remotes(self, modules_json): + """ + Check whether there are previously installed components with the same org_path but different remote urls + Log warning if multiple remotes exist. + + Return: + True: if problematic components are found + False: if problematic components are not found + """ + alternate_remotes = False + for repo_url, repo_content in modules_json.modules_json["repos"].items(): # ex: https, [] + for dir in repo_content[self.component_type].keys(): # ex: nf-core + if dir == self.org and repo_url != self.modules_repo.remote_url: + alternate_remotes = True + if alternate_remotes: + warn_msg = f"Multiple module remotes are used with the same org_path '{self.org}': {', '.join(alternate_remotes)}. This may result in reinstalled modules from the wrong remote." + log.warning(warn_msg) + return True + else: + return False From ff84a09385287a00ee6881e9685de33a47f3241a Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Wed, 29 Mar 2023 17:48:32 -0400 Subject: [PATCH 138/246] fix linting with black --- nf_core/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a801df4266..15bacde92f 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1398,7 +1398,9 @@ def build(dir, no_prompts, web_only, url): # nf-core schema lint @schema.command() -@click.argument("schema_path", type=click.Path(exists=True), default="nextflow_schema.json", metavar="") +@click.argument( + "schema_path", type=click.Path(exists=True), default="nextflow_schema.json", metavar="" +) def lint(schema_path): """ Check that a given pipeline schema is valid. From cf39ce123554b2efdb592da813a0970bf01ed31c Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Wed, 29 Mar 2023 18:27:40 -0400 Subject: [PATCH 139/246] bugfix --- nf_core/components/install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index fc88f36ce2..5b1557447b 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -278,6 +278,7 @@ def check_alternate_remotes(self, modules_json): False: if problematic components are not found """ alternate_remotes = False + modules_json.load() for repo_url, repo_content in modules_json.modules_json["repos"].items(): # ex: https, [] for dir in repo_content[self.component_type].keys(): # ex: nf-core if dir == self.org and repo_url != self.modules_repo.remote_url: From 2be86d67f7e227d836299d02b4fa4fa0fa82fb2b Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Wed, 29 Mar 2023 18:42:48 -0400 Subject: [PATCH 140/246] updated `CHANGELOG.md` with new PR --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40d62f1345..605e41c9d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ - Add the Nextflow version to Gitpod container matching the minimal Nextflow version for nf-core (according to `nextflow.config`) ([#2196](https://github.com/nf-core/tools/pull/2196)) - Use `nfcore/gitpod:dev` container in the dev branch ([#2196](https://github.com/nf-core/tools/pull/2196)) - Replace requests_mock with responses in test mocks ([#2165](https://github.com/nf-core/tools/pull/2165)). +- Add warning when installing a module from an `org_path` that exists in multiple remotes in `modules.json` ([#2228](https://github.com/nf-core/tools/pull/2228)). ## [v2.7.2 - Mercury Eagle Patch](https://github.com/nf-core/tools/releases/tag/2.7.2) - [2022-12-19] From 96aea751c24ea50a674700f6c5cbe4b79397df4c Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Wed, 29 Mar 2023 21:11:52 -0400 Subject: [PATCH 141/246] bugfix --- nf_core/components/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 5b1557447b..429c4ec00c 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -279,8 +279,8 @@ def check_alternate_remotes(self, modules_json): """ alternate_remotes = False modules_json.load() - for repo_url, repo_content in modules_json.modules_json["repos"].items(): # ex: https, [] - for dir in repo_content[self.component_type].keys(): # ex: nf-core + for repo_url, repo_content in modules_json.modules_json.get("repos",dict()).items(): + for dir in repo_content.get(self.component_type,dict()).keys(): if dir == self.org and repo_url != self.modules_repo.remote_url: alternate_remotes = True if alternate_remotes: From b2455b410f2ff68f866483402e46c271a3795992 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Wed, 29 Mar 2023 21:52:46 -0400 Subject: [PATCH 142/246] fix linting with black --- nf_core/components/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 429c4ec00c..c5dcb7d8f5 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -279,8 +279,8 @@ def check_alternate_remotes(self, modules_json): """ alternate_remotes = False modules_json.load() - for repo_url, repo_content in modules_json.modules_json.get("repos",dict()).items(): - for dir in repo_content.get(self.component_type,dict()).keys(): + for repo_url, repo_content in modules_json.modules_json.get("repos", dict()).items(): + for dir in repo_content.get(self.component_type, dict()).keys(): if dir == self.org and repo_url != self.modules_repo.remote_url: alternate_remotes = True if alternate_remotes: From 958a889265049625f1a0fd18c9e22a279bc90b26 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 30 Mar 2023 07:46:00 +0000 Subject: [PATCH 143/246] [automated] Fix code linting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 63bf235bc7..2fe09e80d2 100644 --- a/README.md +++ b/README.md @@ -612,7 +612,7 @@ The graphical interface is oganzised in groups and within the groups the single Now you can start to change the parameter itself. The `ID` of a new parameter should be defined in small letters without whitespaces. The description is a short free text explanation about the parameter, that appears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. Usually, they contain a small paragraph about the parameter settings or a used datasource, like databases or references. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For integers you can define a min and max value, and for strings the file extension can be specified. -The `type` field is one of the most important points in your pipeline schema, since it defines the datatype of your input and how it will be interpreted. This allows extensive testing prior to starting the pipeline. +The `type` field is one of the most important points in your pipeline schema, since it defines the datatype of your input and how it will be interpreted. This allows extensive testing prior to starting the pipeline. The basic datatypes for a pipeline schema are: @@ -621,7 +621,7 @@ The basic datatypes for a pipeline schema are: - `integer` - `boolean` -For the `string` type you have three different options in the settings (nut icon): `enumerated values`, `pattern` and `format`. The first option, `enumerated values`, allows you to specify a list of specific input values. The list has to be separated with a pipe. The `pattern` and `format` settings can depend on each other. The `format` has to be either a directory or a file path. Depending on the `format` setting selected, specifying the `pattern` setting can be the most efficient and time saving option, especially for `file paths`. The `number` and `integer` types share the same settings. Similarly to `string`, there is an `enumerated values` option with the possibility of specifying a `min` and `max` value. For the `boolean` there is no further settings and the default value is usually `false`. The `boolean` value can be switched to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. +For the `string` type you have three different options in the settings (nut icon): `enumerated values`, `pattern` and `format`. The first option, `enumerated values`, allows you to specify a list of specific input values. The list has to be separated with a pipe. The `pattern` and `format` settings can depend on each other. The `format` has to be either a directory or a file path. Depending on the `format` setting selected, specifying the `pattern` setting can be the most efficient and time saving option, especially for `file paths`. The `number` and `integer` types share the same settings. Similarly to `string`, there is an `enumerated values` option with the possibility of specifying a `min` and `max` value. For the `boolean` there is no further settings and the default value is usually `false`. The `boolean` value can be switched to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. After filling the schema, click on the `Finished` button in the top right corner, this will automatically update your `nextflow_schema.json`. If this is not working, the schema can be copied from the graphical interface and pasted in your `nextflow_schema.json` file. From 729d2ee7d152936e18054bd76d12099353d48c49 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 30 Mar 2023 12:45:36 -0400 Subject: [PATCH 144/246] Change logged warning to logged error with clearer message --- nf_core/components/install.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index c5dcb7d8f5..8c85f5a52b 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -59,7 +59,9 @@ def install(self, component, silent=False): modules_json.check_up_to_date() # Verify that the remote repo's org_path does not match the org_path of any alternate repo among the installed modules - self.check_alternate_remotes(modules_json) + if self.check_alternate_remotes(modules_json): + err_msg = f"You are trying to install {self.component_type} from different repositories with the same organization name '{self.org}' (set in the `.nf-core.yml` file in the `org_path` field).\nThis is not supported, and will likely cause problems. org_path should be set to the github account/organization name." + log.err(err_msg) # Verify SHA if not self.modules_repo.verify_sha(self.prompt, self.sha): @@ -271,7 +273,7 @@ def clean_modules_json(self, component, modules_repo, modules_json): def check_alternate_remotes(self, modules_json): """ Check whether there are previously installed components with the same org_path but different remote urls - Log warning if multiple remotes exist. + Log error if multiple remotes exist. Return: True: if problematic components are found @@ -284,8 +286,6 @@ def check_alternate_remotes(self, modules_json): if dir == self.org and repo_url != self.modules_repo.remote_url: alternate_remotes = True if alternate_remotes: - warn_msg = f"Multiple module remotes are used with the same org_path '{self.org}': {', '.join(alternate_remotes)}. This may result in reinstalled modules from the wrong remote." - log.warning(warn_msg) return True else: return False From f9ea217544cf6942fa39d8b15b702b1c061e0f7f Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 30 Mar 2023 12:57:17 -0400 Subject: [PATCH 145/246] Simplified logic for returning True/False in check --- nf_core/components/install.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 8c85f5a52b..3d38712af8 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -279,13 +279,9 @@ def check_alternate_remotes(self, modules_json): True: if problematic components are found False: if problematic components are not found """ - alternate_remotes = False modules_json.load() for repo_url, repo_content in modules_json.modules_json.get("repos", dict()).items(): for dir in repo_content.get(self.component_type, dict()).keys(): if dir == self.org and repo_url != self.modules_repo.remote_url: - alternate_remotes = True - if alternate_remotes: - return True - else: - return False + return True + return False From 1b3cb23b8d17e5676cc7d61a238660856e00e470 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 31 Mar 2023 11:12:00 -0400 Subject: [PATCH 146/246] Update nf_core/components/install.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/components/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 3d38712af8..c37103b694 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -60,7 +60,7 @@ def install(self, component, silent=False): # Verify that the remote repo's org_path does not match the org_path of any alternate repo among the installed modules if self.check_alternate_remotes(modules_json): - err_msg = f"You are trying to install {self.component_type} from different repositories with the same organization name '{self.org}' (set in the `.nf-core.yml` file in the `org_path` field).\nThis is not supported, and will likely cause problems. org_path should be set to the github account/organization name." + err_msg = f"You are trying to install {self.component_type} from different repositories with the same organization name '{self.modules_repo.repo_path}' (set in the `.nf-core.yml` file in the `org_path` field).\nThis is not supported, and will likely cause problems. org_path should be set to the github account/organization name." log.err(err_msg) # Verify SHA From bf043bba52beaa10f5ce260f6da84dc23b357dad Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 31 Mar 2023 11:12:08 -0400 Subject: [PATCH 147/246] Update nf_core/components/install.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/components/install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index c37103b694..a492b59642 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -62,6 +62,7 @@ def install(self, component, silent=False): if self.check_alternate_remotes(modules_json): err_msg = f"You are trying to install {self.component_type} from different repositories with the same organization name '{self.modules_repo.repo_path}' (set in the `.nf-core.yml` file in the `org_path` field).\nThis is not supported, and will likely cause problems. org_path should be set to the github account/organization name." log.err(err_msg) + return False # Verify SHA if not self.modules_repo.verify_sha(self.prompt, self.sha): From 473ca496b5b1bfbfc3cfa0e3d40af07c552f9fc2 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Fri, 31 Mar 2023 11:12:18 -0400 Subject: [PATCH 148/246] Update nf_core/components/install.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/components/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index a492b59642..ad43f2ade7 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -283,6 +283,6 @@ def check_alternate_remotes(self, modules_json): modules_json.load() for repo_url, repo_content in modules_json.modules_json.get("repos", dict()).items(): for dir in repo_content.get(self.component_type, dict()).keys(): - if dir == self.org and repo_url != self.modules_repo.remote_url: + if dir == self.modules_repo.repo_path and repo_url != self.modules_repo.remote_url: return True return False From a2e2ab39f14a0197b42b5851ff81d424ea2b1cb2 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Mon, 3 Apr 2023 14:36:49 +0200 Subject: [PATCH 149/246] Declutter README --- nf_core/pipeline-template/README.md | 79 ++++++++++++++++++----------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 0845f6aca0..9719a8e952 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -25,57 +25,76 @@ ## Introduction - + -The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool to run tasks across multiple compute infrastructures in a very portable manner. It uses Docker/Singularity containers making installation trivial and results highly reproducible. The [Nextflow DSL2](https://www.nextflow.io/docs/latest/dsl2.html) implementation of this pipeline uses one container per process which makes it much easier to maintain and update software dependencies. Where possible, these processes have been submitted to and installed from [nf-core/modules](https://github.com/nf-core/modules) in order to make them available to all nf-core pipelines, and to everyone within the Nextflow community! - - - -On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. -{%- if branded -%} -The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/{{ short_name }}/results).{% endif %} - -## Pipeline summary +**{{ name }}** is a bioinformatics pipeline for {{ description }}. + 1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) 2. Present QC for raw reads ([`MultiQC`](http://multiqc.info/)) -## Quick Start +## Usage + +{% if branded -%} + +> **Note** +> If you are new to nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how +> to set-up nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) +> with `-profile test` before running the workflow on actual data. + +{% endif -%} -1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.1`) + -2. Install any of [`Docker`](https://docs.docker.com/engine/installation/), [`Singularity`](https://www.sylabs.io/guides/3.0/user-guide/) (you can follow [this tutorial](https://singularity-tutorial.github.io/01-installation/)), [`Podman`](https://podman.io/), [`Shifter`](https://nersc.gitlab.io/development/shifter/how-to-use/) or [`Charliecloud`](https://hpc.github.io/charliecloud/) for full pipeline reproducibility _(you can use [`Conda`](https://conda.io/miniconda.html) both to install Nextflow itself and also to manage software within pipelines. Please only use it within pipelines as a last resort; see [docs](https://nf-co.re/usage/configuration#basic-configuration-profiles))_. +First, you need to prepare a samplesheet with your input data that looks as follows: -3. Download the pipeline and test it on a minimal dataset with a single command: +**samplesheet.csv**: - ```bash - nextflow run {{ name }} -profile test,YOURPROFILE --outdir - ``` +```csv +sample,fastq_1,fastq_2 +CONTROL_REP1,AEG588A1_S1_L002_R1_001.fastq.gz,AEG588A1_S1_L002_R2_001.fastq.gz +``` - Note that some form of configuration will be needed so that Nextflow knows how to fetch the required software. This is usually done in the form of a config profile (`YOURPROFILE` in the example command above). You can chain multiple config profiles in a comma-separated string. +Each row represents a fastq file (single-end) or a pair of fastq files (paired end). - > - The pipeline comes with config profiles called `docker`, `singularity`, `podman`, `shifter`, `charliecloud` and `conda` which instruct the pipeline to use the named tool for software management. For example, `-profile test,docker`. - > - Please check [nf-core/configs](https://github.com/nf-core/configs#documentation) to see if a custom config file to run nf-core pipelines already exists for your Institute. If so, you can simply use `-profile ` in your command. This will enable either `docker` or `singularity` and set the appropriate execution settings for your local compute environment. - > - If you are using `singularity`, please use the [`nf-core download`](https://nf-co.re/tools/#downloading-pipelines-for-offline-use) command to download images first, before running the pipeline. Setting the [`NXF_SINGULARITY_CACHEDIR` or `singularity.cacheDir`](https://www.nextflow.io/docs/latest/singularity.html?#singularity-docker-hub) Nextflow options enables you to store and re-use the images from a central location for future pipeline runs. - > - If you are using `conda`, it is highly recommended to use the [`NXF_CONDA_CACHEDIR` or `conda.cacheDir`](https://www.nextflow.io/docs/latest/conda.html) settings to store the environments in a central location for future pipeline runs. +> **Warning** +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those +> provided by the `-c` Nextflow option can be used to provide any configuration **except for parameters**; +> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). -4. Start running your own analysis! +Now, you can run the pipeline using: - + - ```bash - nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile - ``` +```bash +nextflow run {{ name }} \ + --input samplesheet.csv \ + -profile +``` {% if branded -%} -## Documentation +For more details, please refer to the [usage documentation](https://nf-co.re/{{ short_name }}/usage) and the [parameter documentation](https://nf-co.re/{{ short_name }}/parameters). + +{% endif -%} + +{% if branded -%} + +## Pipeline output + + -The {{ name }} pipeline comes with documentation about the pipeline [usage](https://nf-co.re/{{ short_name }}/usage), [parameters](https://nf-co.re/{{ short_name }}/parameters) and [output](https://nf-co.re/{{ short_name }}/output). +For more details, please refer to the [output documentation](https://nf-co.re/{{ short_name }}/output). {% endif -%} From 8c714e78b9e8f31a758675af4e7f5d00f0eb9719 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Mon, 3 Apr 2023 14:41:15 +0200 Subject: [PATCH 150/246] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce74b77b8f..118856b2f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) - Fixed minor Jinja2 templating bug that caused the PR template to miss a newline - Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action` +- Simplify pipeline README ([#2186](https://github.com/nf-core/tools/issues/2186)) ### Linting From ccc919a0263ae637d9ee60ff411f44776a3fe86b Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 3 Apr 2023 14:55:42 +0100 Subject: [PATCH 151/246] Add docker registry as configurable parameter - Adds parameter `--registry` which specifies which base registry to use - Defaults to quay.io/biocontainers to match existing modules - When it is changed, any containers specified by name (e.g. multiqc:latest) will be pulled from the registry defined in `--registry`. - This is aimed at using `--registry public.ecr.aws/biocontainers` --- CHANGELOG.md | 1 + nf_core/pipeline-template/nextflow.config | 2 ++ .../pipeline-template/nextflow_schema.json | 23 ++++++++++++++++--- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce74b77b8f..72c9a3a92a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) - Fixed minor Jinja2 templating bug that caused the PR template to miss a newline - Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action` +- Added parameter `--registry` to pipeline template for a configurable default container registry. ### Linting diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index cc591a54e5..9e906c4b5d 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -41,6 +41,7 @@ params { validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' + registry = 'quay.io/biocontainers' {% if nf_core_configs %} // Config options @@ -101,6 +102,7 @@ profiles { } docker { docker.enabled = true + docker.registry = params.registry docker.userEmulation = true conda.enabled = false singularity.enabled = false diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 2743562d6c..6193efc65d 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -10,7 +10,10 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": ["input", "outdir"], + "required": [ + "input", + "outdir" + ], "properties": { "input": { "type": "string", @@ -39,6 +42,13 @@ "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" + }, + "registry": { + "type": "string", + "default": "quay.io/biocontainers", + "description": "Default docker container registry", + "help_text": "The default docker container registry that will be used if a simple docker image name is used. Set this to a different registry if an mirror or private registry is required, e.g. public.ecr.aws/biocontainers/. By default, this is quay.io/biocontainers.", + "fa_icon": "fas fa-cog" } } }, @@ -188,7 +198,14 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "enum": [ + "symlink", + "rellink", + "link", + "copy", + "copyNoFollow", + "move" + ], "hidden": true }, "email_on_fail": { @@ -284,4 +301,4 @@ "$ref": "#/definitions/generic_options" } ] -} +} \ No newline at end of file From e7ac058b89f8f36fb45b54ec16db3f1206fc3fb5 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 3 Apr 2023 15:08:04 +0100 Subject: [PATCH 152/246] swap params.registry -> params.container_registry --- CHANGELOG.md | 2 +- nf_core/pipeline-template/nextflow.config | 4 ++-- nf_core/pipeline-template/nextflow_schema.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72c9a3a92a..71bb614374 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) - Fixed minor Jinja2 templating bug that caused the PR template to miss a newline - Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action` -- Added parameter `--registry` to pipeline template for a configurable default container registry. +- Added parameter `--container_registry` to pipeline template for a configurable default container registry. ### Linting diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 9e906c4b5d..cd2fc05789 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -41,7 +41,7 @@ params { validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' - registry = 'quay.io/biocontainers' + params.container_registry = 'quay.io/biocontainers' {% if nf_core_configs %} // Config options @@ -102,7 +102,7 @@ profiles { } docker { docker.enabled = true - docker.registry = params.registry + docker.registry = params.container_registry docker.userEmulation = true conda.enabled = false singularity.enabled = false diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 6193efc65d..f07bd4ca45 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -43,7 +43,7 @@ "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" }, - "registry": { + "container_registry": { "type": "string", "default": "quay.io/biocontainers", "description": "Default docker container registry", From f114ec653d7c18e11c0f163a4e229c02287ad437 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 3 Apr 2023 15:24:10 +0100 Subject: [PATCH 153/246] prettier --- nf_core/pipeline-template/nextflow_schema.json | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index f07bd4ca45..dbc4d776d4 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -10,10 +10,7 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": [ - "input", - "outdir" - ], + "required": ["input", "outdir"], "properties": { "input": { "type": "string", @@ -198,14 +195,7 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": [ - "symlink", - "rellink", - "link", - "copy", - "copyNoFollow", - "move" - ], + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], "hidden": true }, "email_on_fail": { @@ -301,4 +291,4 @@ "$ref": "#/definitions/generic_options" } ] -} \ No newline at end of file +} From 9934081ae666d35c5479fb5b76c2195ddb001976 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Mon, 3 Apr 2023 12:16:08 -0400 Subject: [PATCH 154/246] change log.err to log.error --- nf_core/components/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index ad43f2ade7..3e440424d6 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -61,7 +61,7 @@ def install(self, component, silent=False): # Verify that the remote repo's org_path does not match the org_path of any alternate repo among the installed modules if self.check_alternate_remotes(modules_json): err_msg = f"You are trying to install {self.component_type} from different repositories with the same organization name '{self.modules_repo.repo_path}' (set in the `.nf-core.yml` file in the `org_path` field).\nThis is not supported, and will likely cause problems. org_path should be set to the github account/organization name." - log.err(err_msg) + log.error(err_msg) return False # Verify SHA From 7849dcccd64f789315fef61baece44b7c078da29 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 4 Apr 2023 12:13:25 +0100 Subject: [PATCH 155/246] Add checks for duplicated labels and warn if non-standard labels found --- nf_core/modules/lint/main_nf.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 6a75667a75..fbcc68202c 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -258,6 +258,19 @@ def check_process_section(self, lines, fix_version, progress_bar): self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) else: self.warned.append(("process_standard_label", "Standard process label not found", self.main_nf)) + if len(bad_labels) > 0: + self.warned.append( + ("process_standard_label", f"Non-standard labels found: `{'`,`'.join(good_labels)}`", self.main_nf) + ) + if len(all_labels) > len(set(all_labels)): + self.warned.append( + ( + "process_standard_label", + f"Duplicate labels found: `{'`,`'.join(sorted(all_labels))}`", + self.main_nf, + ) + ) + else: self.warned.append(("process_standard_label", "Process label not specified", self.main_nf)) for i, l in enumerate(lines): From 2915f2ebd4353b4be5f67e4a1c1b0a98c55234d9 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 4 Apr 2023 12:15:34 +0100 Subject: [PATCH 156/246] Fix indentation so that warning logic is outside loop and not repeated for every label --- nf_core/modules/lint/main_nf.py | 44 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index fbcc68202c..b980b892af 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -246,30 +246,30 @@ def check_process_section(self, lines, fix_version, progress_bar): bad_labels.append(label) else: good_labels.append(label) - if len(good_labels) > 1: - self.warned.append( - ( - "process_standard_label", - f"Conflicting process labels found: `{'`,`'.join(good_labels)}`", - self.main_nf, - ) - ) - elif len(good_labels) == 1: - self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) - else: - self.warned.append(("process_standard_label", "Standard process label not found", self.main_nf)) - if len(bad_labels) > 0: - self.warned.append( - ("process_standard_label", f"Non-standard labels found: `{'`,`'.join(good_labels)}`", self.main_nf) + if len(good_labels) > 1: + self.warned.append( + ( + "process_standard_label", + f"Conflicting process labels found: `{'`,`'.join(good_labels)}`", + self.main_nf, ) - if len(all_labels) > len(set(all_labels)): - self.warned.append( - ( - "process_standard_label", - f"Duplicate labels found: `{'`,`'.join(sorted(all_labels))}`", - self.main_nf, - ) + ) + elif len(good_labels) == 1: + self.passed.append(("process_standard_label", "Correct process label", self.main_nf)) + else: + self.warned.append(("process_standard_label", "Standard process label not found", self.main_nf)) + if len(bad_labels) > 0: + self.warned.append( + ("process_standard_label", f"Non-standard labels found: `{'`,`'.join(good_labels)}`", self.main_nf) + ) + if len(all_labels) > len(set(all_labels)): + self.warned.append( + ( + "process_standard_label", + f"Duplicate labels found: `{'`,`'.join(sorted(all_labels))}`", + self.main_nf, ) + ) else: self.warned.append(("process_standard_label", "Process label not specified", self.main_nf)) From c57dbf3d90cb64e841b9860cb1e2527763047569 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 4 Apr 2023 12:23:44 +0100 Subject: [PATCH 157/246] Add handling and warning for labels that do not match the expected regex --- nf_core/modules/lint/main_nf.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index b980b892af..22fb6459d8 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -241,7 +241,17 @@ def check_process_section(self, lines, fix_version, progress_bar): good_labels = [] if len(all_labels) > 0: for label in all_labels: - label = re.match("^label\s+([a-zA-Z0-9_-]+)", label).group(1) + try: + label = re.match("^label\s+([a-zA-Z0-9_-]+)", label).group(1) + except AttributeError: + self.warned.append( + ( + "process_standard_label", + f"Specified label appears to contain non-alphanumerics: {label}", + self.main_nf, + ) + ) + continue if label not in correct_process_labels: bad_labels.append(label) else: From b8fcbd2fe69d7a3f9864a4a1bd394c3433c83f93 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 4 Apr 2023 13:25:22 +0100 Subject: [PATCH 158/246] Add a test for patch removal --- tests/modules/patch.py | 29 +++++++++++++++++++++++++++++ tests/test_modules.py | 1 + 2 files changed, 30 insertions(+) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 09b892e2c8..360bb99120 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -322,3 +322,32 @@ def test_create_patch_update_fail(self): with open(module_path / patch_fn, "r") as fh: new_patch_contents = fh.read() assert patch_contents == new_patch_contents + + +def test_remove_patch(self): + """Test creating a patch when there is no change to the module""" + setup_patch(self.pipeline_dir, True) + + # Try creating a patch file + patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) + patch_obj.patch(BISMARK_ALIGN) + + module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) + + # Check that a patch file with the correct name has been created + patch_fn = f"{'-'.join(BISMARK_ALIGN.split('/'))}.diff" + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", patch_fn} + + # Check the 'modules.json' contains a patch file for the module + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) == Path( + "modules", REPO_NAME, BISMARK_ALIGN, patch_fn + ) + + patch_obj.remove(BISMARK_ALIGN) + # Check that the diff file has been removed + assert set(os.listdir(module_path)) == {"main.nf", "meta.yml"} + + # Check that the 'modules.json' entry has been removed + modules_json_obj = nf_core.modules.modules_json.ModulesJson(self.pipeline_dir) + assert modules_json_obj.get_patch_fn(BISMARK_ALIGN, REPO_URL, REPO_NAME) is None diff --git a/tests/test_modules.py b/tests/test_modules.py index 21c003112e..df2ab901ac 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -203,6 +203,7 @@ def test_modulesrepo_class(self): test_create_patch_try_apply_successful, test_create_patch_update_fail, test_create_patch_update_success, + test_remove_patch, ) from .modules.remove import ( test_modules_remove_multiqc_from_gitlab, From d81b049000630fe39e7e77fd829e89926886b6f7 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Tue, 4 Apr 2023 15:55:57 +0200 Subject: [PATCH 159/246] Update nf_core/pipeline-template/README.md Co-authored-by: James A. Fellows Yates --- nf_core/pipeline-template/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 9719a8e952..93ed109e75 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -35,7 +35,7 @@ **{{ name }}** is a bioinformatics pipeline for {{ description }}. + workflows use the "tube map" design for that. See https://nf-co.re/docs/contributing/design_guidelines#examples for examples. --> 1. Read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) From 23c4a2f79fb532ed2c10eb3a02b175e743c53247 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 4 Apr 2023 15:43:26 +0100 Subject: [PATCH 160/246] Fix issue with using relpaths instead of paths with the pipeline_dir. Mock confirmation during tests --- nf_core/modules/patch.py | 7 ++++--- tests/modules/patch.py | 5 ++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/patch.py b/nf_core/modules/patch.py index 8c3f0cf382..4890345052 100644 --- a/nf_core/modules/patch.py +++ b/nf_core/modules/patch.py @@ -175,6 +175,7 @@ def remove(self, module): module_relpath = Path("modules", module_dir, module) patch_relpath = Path(module_relpath, patch_filename) patch_path = Path(self.dir, patch_relpath) + module_path = Path(self.dir, module_relpath) if patch_path.exists(): remove = questionary.confirm( @@ -186,11 +187,11 @@ def remove(self, module): # Try to apply the patch in reverse and move resulting files to module dir temp_module_dir = self.modules_json.try_apply_patch_reverse( - module, self.modules_repo.repo_path, patch_relpath, module_relpath + module, self.modules_repo.repo_path, patch_relpath, module_path ) try: for file in Path(temp_module_dir).glob("*"): - file.rename(module_relpath.joinpath(file.name)) + file.rename(module_path.joinpath(file.name)) os.rmdir(temp_module_dir) except Exception as err: raise UserWarning(f"There was a problem reverting the patched file: {err}") @@ -201,7 +202,7 @@ def remove(self, module): # Write changes to module.json self.modules_json.remove_patch_entry(module, self.modules_repo.remote_url, module_dir) - if not all(self.modules_repo.module_files_identical(module, module_relpath, module_version).values()): + if not all(self.modules_repo.module_files_identical(module, module_path, module_version).values()): log.error( f"Module files do not appear to match the remote for the commit sha in the 'module.json': {module_version}\n" f"Recommend reinstalling with 'nf-core modules install --force --sha {module_version} {module}' " diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 360bb99120..72a4bf1788 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -1,6 +1,7 @@ import os import tempfile from pathlib import Path +from unittest import mock import pytest @@ -344,7 +345,9 @@ def test_remove_patch(self): "modules", REPO_NAME, BISMARK_ALIGN, patch_fn ) - patch_obj.remove(BISMARK_ALIGN) + with mock.patch.object(nf_core.create.questionary, "confirm") as mock_questionary: + mock_questionary.unsafe_ask.return_value = True + patch_obj.remove(BISMARK_ALIGN) # Check that the diff file has been removed assert set(os.listdir(module_path)) == {"main.nf", "meta.yml"} From f6d65721b3c8328a3af56e394a65db78e6d7db9e Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Wed, 5 Apr 2023 08:48:41 +0200 Subject: [PATCH 161/246] move warning --- nf_core/pipeline-template/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 93ed109e75..ddf5495dc7 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -67,11 +67,6 @@ CONTROL_REP1,AEG588A1_S1_L002_R1_001.fastq.gz,AEG588A1_S1_L002_R2_001.fastq.gz Each row represents a fastq file (single-end) or a pair of fastq files (paired end). -> **Warning** -> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those -> provided by the `-c` Nextflow option can be used to provide any configuration **except for parameters**; -> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). - Now, you can run the pipeline using: @@ -82,6 +77,11 @@ nextflow run {{ name }} \ -profile ``` +> **Warning** +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those +> provided by the `-c` Nextflow option can be used to provide any configuration **except for parameters**; +> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). + {% if branded -%} For more details, please refer to the [usage documentation](https://nf-co.re/{{ short_name }}/usage) and the [parameter documentation](https://nf-co.re/{{ short_name }}/parameters). From 53b99b3546af6993cdf3e0309be430fba8d68d1a Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Wed, 5 Apr 2023 08:53:40 +0200 Subject: [PATCH 162/246] update output documentation statement --- nf_core/pipeline-template/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index ddf5495dc7..19e2121b4e 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -92,9 +92,9 @@ For more details, please refer to the [usage documentation](https://nf-co.re/{{ ## Pipeline output - - -For more details, please refer to the [output documentation](https://nf-co.re/{{ short_name }}/output). +[results](https://nf-co.re/{{ short_name }}/results) section of the documentation. +For more details about the output files and reports, please refer to the +[output documentation](https://nf-co.re/{{ short_name }}/output). {% endif -%} From dcc50e556ab3bcefb7157645657c614eecfc3d7b Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Wed, 5 Apr 2023 09:18:29 +0200 Subject: [PATCH 163/246] Remove linting test for nextflow version in quick start section --- nf_core/lint/readme.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py index daf92f7932..5f6dd87030 100644 --- a/nf_core/lint/readme.py +++ b/nf_core/lint/readme.py @@ -62,24 +62,4 @@ def readme(self): else: warned.append("README did not have a Nextflow minimum version badge.") - # Check that the minimum version mentioned in the quick start section is consistent - # Looking for: "1. Install [`Nextflow`](https://www.nextflow.io/docs/latest/getstarted.html#installation) (`>=22.10.1`)" - nf_version_re = r"1\.\s*Install\s*\[`Nextflow`\]\(https://www.nextflow.io/docs/latest/getstarted.html#installation\)\s*\(`>=(\d*\.\d*\.\d*)`\)" - match = re.search(nf_version_re, content) - if match: - nf_quickstart_version = match.group(1) - try: - if nf_quickstart_version != self.minNextflowVersion: - raise AssertionError() - except (AssertionError, KeyError): - failed.append( - f"README Nextflow minimium version in Quick Start section does not match config. README: `{nf_quickstart_version}`, Config `{self.minNextflowVersion}`" - ) - else: - passed.append( - f"README Nextflow minimum version in Quick Start section matched config. README: `{nf_quickstart_version}`, Config: `{self.minNextflowVersion}`" - ) - else: - warned.append("README did not have a Nextflow minimum version mentioned in Quick Start section.") - return {"passed": passed, "warned": warned, "failed": failed} From f3eee2a746b75bf9d27254d82beffc5d51fc8149 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 5 Apr 2023 10:57:02 +0200 Subject: [PATCH 164/246] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce74b77b8f..6dd88ce28b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) - Fixed minor Jinja2 templating bug that caused the PR template to miss a newline - Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action` +- Remove `.cff` files from `.editorconfig` [(#2145)[https://github.com/nf-core/tools/pull/2145]] ### Linting From 1f30b401b801d43ac025a42a150ee20f5accce38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Wed, 5 Apr 2023 11:01:06 +0200 Subject: [PATCH 165/246] Update nf_core/pipeline-template/nextflow.config --- nf_core/pipeline-template/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index fe8c6587ab..42643ac9fe 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -84,6 +84,7 @@ profiles { debug { dumpHashes = true process.beforeScript = 'echo $HOSTNAME' + cleanup = false } conda { conda.enabled = true From 653c2f4bf17a80d7c207203e99a276fcbb014f2f Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Wed, 5 Apr 2023 17:08:53 +0100 Subject: [PATCH 166/246] Remove container_registry from params Changes: - Sets docker.registry directly in the Docker scope - Removes param.container_registry which might have caused problems in various environments. --- CHANGELOG.md | 2 +- nf_core/pipeline-template/nextflow.config | 3 +-- nf_core/pipeline-template/nextflow_schema.json | 7 ------- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71bb614374..ef1321eaf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) - Fixed minor Jinja2 templating bug that caused the PR template to miss a newline - Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action` -- Added parameter `--container_registry` to pipeline template for a configurable default container registry. +- Added config `docker.registry` to pipeline template for a configurable default container registry when using Docker containers. Defaults to `quay.io`. ### Linting diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index cd2fc05789..a67f79a6d0 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -41,7 +41,6 @@ params { validate_params = true show_hidden_params = false schema_ignore_params = 'genomes' - params.container_registry = 'quay.io/biocontainers' {% if nf_core_configs %} // Config options @@ -102,7 +101,7 @@ profiles { } docker { docker.enabled = true - docker.registry = params.container_registry + docker.registry = 'quay.io' docker.userEmulation = true conda.enabled = false singularity.enabled = false diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index dbc4d776d4..2743562d6c 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -39,13 +39,6 @@ "type": "string", "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.", "fa_icon": "fas fa-file-signature" - }, - "container_registry": { - "type": "string", - "default": "quay.io/biocontainers", - "description": "Default docker container registry", - "help_text": "The default docker container registry that will be used if a simple docker image name is used. Set this to a different registry if an mirror or private registry is required, e.g. public.ecr.aws/biocontainers/. By default, this is quay.io/biocontainers.", - "fa_icon": "fas fa-cog" } } }, From a963a8554342e7559eef0553b1cc4243ccdad6c3 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Wed, 5 Apr 2023 17:33:58 +0100 Subject: [PATCH 167/246] CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef1321eaf7..85d07a6fef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - Consistent syntax for branch checks in PRs ([#2202](https://github.com/nf-core/tools/issues/2202)) - Fixed minor Jinja2 templating bug that caused the PR template to miss a newline - Updated AWS tests to use newly moved `seqeralabs/action-tower-launch` instead of `nf-core/tower-action` -- Added config `docker.registry` to pipeline template for a configurable default container registry when using Docker containers. Defaults to `quay.io`. +- Added config `docker.registry` to pipeline template for a configurable default container registry when using Docker containers. Defaults to `quay.io` ([#2133](https://github.com/nf-core/tools/pull/2133)) ### Linting From fc991988f75ddb7927e76c0a3d56f4f62d6c625f Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Wed, 5 Apr 2023 20:02:56 -0400 Subject: [PATCH 168/246] Changed functionality of check so that conflict is not limited to component type --- nf_core/components/install.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 3e440424d6..f7a5fe6680 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -282,7 +282,8 @@ def check_alternate_remotes(self, modules_json): """ modules_json.load() for repo_url, repo_content in modules_json.modules_json.get("repos", dict()).items(): - for dir in repo_content.get(self.component_type, dict()).keys(): - if dir == self.modules_repo.repo_path and repo_url != self.modules_repo.remote_url: - return True + for component_type in repo_content: + for dir in repo_content.get(component_type, dict()).keys(): + if dir == self.modules_repo.repo_path and repo_url != self.modules_repo.remote_url: + return True return False From ccd506fd02c68cf0646b1fadd017e9bfdae17e26 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Wed, 5 Apr 2023 20:03:54 -0400 Subject: [PATCH 169/246] fixed pytests and added tests for alternate remotes when installing modules or subworkflows --- tests/modules/install.py | 9 ++++++++ tests/modules/lint.py | 8 +++++-- tests/modules/patch.py | 41 +++++++++++++++++++++++++++++------ tests/modules/remove.py | 6 ++++- tests/modules/update.py | 11 +++++++++- tests/subworkflows/install.py | 8 +++++++ tests/subworkflows/list.py | 2 ++ tests/test_modules.py | 6 +++-- tests/test_subworkflows.py | 6 +++-- tests/utils.py | 19 ++++++++++++++++ 10 files changed, 101 insertions(+), 15 deletions(-) diff --git a/tests/modules/install.py b/tests/modules/install.py index d01459f142..985f916998 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -12,6 +12,7 @@ with_temporary_folder, ) +from ..utils import remove_template_modules def test_modules_install_nopipeline(self): """Test installing a module - no pipeline given""" @@ -49,6 +50,7 @@ def test_modules_install_trimgalore_twice(self): def test_modules_install_from_gitlab(self): """Test installing a module from GitLab""" + remove_template_modules(self) assert self.mods_install_gitlab.install("fastqc") is True @@ -61,6 +63,7 @@ def test_modules_install_different_branch_fail(self): def test_modules_install_different_branch_succeed(self): """Test installing a module from a different branch""" + remove_template_modules(self) install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) # The fastp module does exists in the branch-test branch assert install_obj.install("fastp") is True @@ -83,3 +86,9 @@ def test_modules_install_tracking(self): assert mod_json["repos"]["https://github.com/nf-core/modules.git"]["modules"]["nf-core"]["trimgalore"][ "installed_by" ] == ["modules"] + +def test_modules_install_alternate_remote(self): + """Test installing a module from a different remote with the same organization path""" + install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) + # The fastp module does exists in the branch-test branch + assert install_obj.install("fastp") is False diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 18c0dc4dab..21e9fdd16e 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -6,6 +6,7 @@ import nf_core.modules from ..utils import GITLAB_URL, set_wd +from ..utils import remove_template_modules from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf @@ -62,6 +63,7 @@ def test_modules_lint_no_gitlab(self): def test_modules_lint_gitlab_modules(self): """Lint modules from a different remote""" + remove_template_modules(self) self.mods_install_gitlab.install("fastqc") self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) @@ -70,10 +72,11 @@ def test_modules_lint_gitlab_modules(self): assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - +#should we remove this? We want to test multiple remotes but not for linting! def test_modules_lint_multiple_remotes(self): """Lint modules from a different remote""" - self.mods_install.install("fastqc") + remove_template_modules(self) + #self.mods_install.install("fastqc") self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) module_lint.lint(print_results=False, all_modules=True) @@ -86,6 +89,7 @@ def test_modules_lint_patched_modules(self): """ Test creating a patch file and applying it to a new version of the the files """ + remove_template_modules(self) setup_patch(self.pipeline_dir, True) # Create a patch file diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 09b892e2c8..370dd88d82 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -9,6 +9,8 @@ from ..utils import GITLAB_URL +from ..utils import remove_template_modules + """ Test the 'nf-core modules patch' command @@ -26,7 +28,7 @@ REPO_URL = "https://gitlab.com/nf-core/modules-test.git" -def setup_patch(pipeline_dir, modify_module): +def setup_patch(pipeline_dir, modify_module, pipeline_name): install_obj = nf_core.modules.ModuleInstall( pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=ORG_SHA ) @@ -39,6 +41,13 @@ def setup_patch(pipeline_dir, modify_module): module_path = Path(pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) modify_main_nf(module_path / "main.nf") +def modify_workflow_nf(path): + with open(path, "r") as fh: + lines = fh.readlines() + with open(path, "w") as fh: + for line in lines: + if not line.startswith("include {"): + fh.write(line) def modify_main_nf(path): """Modify a file to test patch creation""" @@ -60,7 +69,10 @@ def modify_main_nf(path): def test_create_patch_no_change(self): """Test creating a patch when there is no change to the module""" - setup_patch(self.pipeline_dir, False) + # Remove modules that may cause org_path conflict + remove_template_modules(self) + + setup_patch(self.pipeline_dir, False, self.pipeline_name) # Try creating a patch file patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) @@ -79,7 +91,10 @@ def test_create_patch_no_change(self): def test_create_patch_change(self): """Test creating a patch when there is a change to the module""" - setup_patch(self.pipeline_dir, True) + # Remove modules that may cause org_path conflict + remove_template_modules(self) + + setup_patch(self.pipeline_dir, True, self.pipeline_name) # Try creating a patch file patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) @@ -112,7 +127,10 @@ def test_create_patch_try_apply_successful(self): """ Test creating a patch file and applying it to a new version of the the files """ - setup_patch(self.pipeline_dir, True) + # Remove modules that may cause org_path conflict + remove_template_modules(self) + + setup_patch(self.pipeline_dir, True, self.pipeline_name) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) module_path = Path(self.pipeline_dir, module_relpath) @@ -178,7 +196,10 @@ def test_create_patch_try_apply_failed(self): """ Test creating a patch file and applying it to a new version of the the files """ - setup_patch(self.pipeline_dir, True) + # Remove modules that may cause org_path conflict + remove_template_modules(self) + + setup_patch(self.pipeline_dir, True, self.pipeline_name) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) module_path = Path(self.pipeline_dir, module_relpath) @@ -216,7 +237,10 @@ def test_create_patch_update_success(self): Should have the same effect as 'test_create_patch_try_apply_successful' but uses higher level api """ - setup_patch(self.pipeline_dir, True) + # Remove modules that may cause org_path conflict + remove_template_modules(self) + + setup_patch(self.pipeline_dir, True, self.pipeline_name) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) # Try creating a patch file @@ -277,7 +301,10 @@ def test_create_patch_update_fail(self): """ Test creating a patch file and updating a module when there is a diff conflict """ - setup_patch(self.pipeline_dir, True) + # Remove modules that may cause org_path conflict + remove_template_modules(self) + + setup_patch(self.pipeline_dir, True, self.pipeline_name) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) # Try creating a patch file diff --git a/tests/modules/remove.py b/tests/modules/remove.py index b343a02da7..b2af1f10bd 100644 --- a/tests/modules/remove.py +++ b/tests/modules/remove.py @@ -1,5 +1,6 @@ import os +from ..utils import remove_template_modules def test_modules_remove_trimgalore(self): """Test removing TrimGalore! module after installing it""" @@ -13,9 +14,12 @@ def test_modules_remove_trimgalore_uninstalled(self): """Test removing TrimGalore! module without installing it""" assert self.mods_remove.remove("trimgalore") is False - +# Should this check be removed? def test_modules_remove_multiqc_from_gitlab(self): """Test removing multiqc module after installing it from an alternative source""" + # Remove modules that may cause org_path conflict + remove_template_modules(self) + self.mods_install_gitlab.install("multiqc") module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core", "multiqc") assert self.mods_remove_gitlab.remove("multiqc", force=True) diff --git a/tests/modules/update.py b/tests/modules/update.py index fcfd92fc39..ea94488b5d 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -26,6 +26,7 @@ OLD_TRIMGALORE_SHA, ) +from ..utils import remove_template_modules def test_install_and_update(self): """Installs a module in the pipeline and updates it (no change)""" @@ -41,9 +42,9 @@ def test_install_and_update(self): assert update_obj.update("trimgalore") is True assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True - def test_install_at_hash_and_update(self): """Installs an old version of a module in the pipeline and updates it""" + remove_template_modules(self) assert self.mods_install_old.install("trimgalore") update_obj = ModuleUpdate( self.pipeline_dir, show_diff=False, update_deps=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH @@ -69,6 +70,7 @@ def test_install_at_hash_and_update(self): def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a module in the pipeline and updates it""" + remove_template_modules(self) self.mods_install_old.install("trimgalore") patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") update_obj = ModuleUpdate( @@ -109,6 +111,7 @@ def test_update_all(self): def test_update_with_config_fixed_version(self): """Try updating when there are entries in the .nf-core.yml""" + remove_template_modules(self) # Install trimgalore at the latest version assert self.mods_install_trimgalore.install("trimgalore") @@ -134,6 +137,7 @@ def test_update_with_config_fixed_version(self): def test_update_with_config_dont_update(self): """Try updating when module is to be ignored""" + remove_template_modules(self) # Install an old version of trimgalore self.mods_install_old.install("trimgalore") @@ -164,6 +168,7 @@ def test_update_with_config_dont_update(self): def test_update_with_config_fix_all(self): """Fix the version of all nf-core modules""" + remove_template_modules(self) self.mods_install_trimgalore.install("trimgalore") # Fix the version of all nf-core modules in the .nf-core.yml to an old version @@ -187,6 +192,7 @@ def test_update_with_config_fix_all(self): def test_update_with_config_no_updates(self): """Don't update any nf-core modules""" + remove_template_modules(self) self.mods_install_old.install("trimgalore") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() @@ -220,6 +226,7 @@ def test_update_with_config_no_updates(self): def test_update_different_branch_single_module(self): """Try updating a module in a specific branch""" + remove_template_modules(self) install_obj = ModuleInstall( self.pipeline_dir, prompt=False, @@ -246,6 +253,7 @@ def test_update_different_branch_single_module(self): def test_update_different_branch_mixed_modules_main(self): """Try updating all modules where MultiQC is installed from main branch""" + remove_template_modules(self) # Install fastp assert self.mods_install_gitlab_old.install("fastp") @@ -272,6 +280,7 @@ def test_update_different_branch_mixed_modules_main(self): def test_update_different_branch_mix_modules_branch_test(self): """Try updating all modules where MultiQC is installed from branch-test branch""" + remove_template_modules(self) # Install multiqc from the branch-test branch assert self.mods_install_gitlab_old.install( "multiqc" diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 6c04c9ad22..bf41694e1a 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -13,6 +13,8 @@ with_temporary_folder, ) +from ..utils import remove_template_modules + def test_subworkflow_install_nopipeline(self): """Test installing a subworkflow - no pipeline given""" @@ -62,6 +64,7 @@ def test_subworkflows_install_bam_sort_stats_samtools_twice(self): def test_subworkflows_install_from_gitlab(self): """Test installing a subworkflow from GitLab""" + remove_template_modules(self) assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is True # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) @@ -140,3 +143,8 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self): "installed_by" ] ) == sorted(["subworkflows", "bam_sort_stats_samtools"]) + +def test_subworkflows_install_alternate_remote(self): + """Test installing a subworkflow from a different remote with the same organization path""" + self.subworkflow_install.install("bam_sort_stats_samtools") + assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is False diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index c65999d42c..2d44112859 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -4,6 +4,7 @@ from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL +from ..utils import remove_template_modules def test_subworkflows_list_remote(self): """Test listing available subworkflows""" @@ -40,6 +41,7 @@ def test_subworkflows_install_and_list_subworkflows(self): def test_subworkflows_install_gitlab_and_list_subworkflows(self): """Test listing locally installed subworkflows""" + remove_template_modules(self) self.subworkflow_install_gitlab.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() diff --git a/tests/test_modules.py b/tests/test_modules.py index 21c003112e..e3fdd810ea 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -59,9 +59,10 @@ def setUp(self): # Set up the schema root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") - self.pipeline_dir = os.path.join(self.tmp_dir, "mypipeline") + self.pipeline_name = "mypipeline" + self.pipeline_dir = os.path.join(self.tmp_dir, self.pipeline_name) nf_core.create.PipelineCreate( - "mypipeline", "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True + self.pipeline_name, "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True ).init_pipeline() # Set up install objects self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) @@ -159,6 +160,7 @@ def test_modulesrepo_class(self): test_modules_install_tracking, test_modules_install_trimgalore, test_modules_install_trimgalore_twice, + test_modules_install_alternate_remote, ) from .modules.lint import ( test_modules_lint_empty, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 19a090f7f0..336e736da1 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -48,9 +48,10 @@ def setUp(self): # Set up the pipeline structure root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") - self.pipeline_dir = os.path.join(self.tmp_dir, "mypipeline") + self.pipeline_name = "mypipeline" + self.pipeline_dir = os.path.join(self.tmp_dir, self.pipeline_name) nf_core.create.PipelineCreate( - "mypipeline", "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True + self.pipeline_name, "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True ).init_pipeline() # Set up the nf-core/modules repo dummy @@ -117,6 +118,7 @@ def tearDown(self): test_subworkflows_install_tracking, test_subworkflows_install_tracking_added_already_installed, test_subworkflows_install_tracking_added_super_subworkflow, + test_subworkflows_install_alternate_remote, ) from .subworkflows.list import ( test_subworkflows_install_and_list_subworkflows, diff --git a/tests/utils.py b/tests/utils.py index 0dd60dd051..eafe9bdaac 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,6 +10,8 @@ import responses +import nf_core.modules + OLD_TRIMGALORE_SHA = "06348dffce2a732fc9e656bdc5c64c3e02d302cb" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" @@ -104,3 +106,20 @@ def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module, version): ], } rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) + +def remove_template_modules(self): + # Remove modules that may cause org_path conflict + workflow_path = Path(self.pipeline_dir, "workflows", self.pipeline_name + ".nf") + with open(workflow_path, "r") as fh: + lines = fh.readlines() + with open(workflow_path, "w") as fh: + for line in lines: + if not line.startswith("include {"): + fh.write(line) + + remove_obj = nf_core.modules.ModuleRemove( + self.pipeline_dir, no_pull=False + ) + + for i in ['multiqc','fastqc','custom/dumpsoftwareversions']: + remove_obj.remove(i) From 4f130bdd60c0d890f404b6a1f942cc255235298c Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Wed, 5 Apr 2023 20:08:32 -0400 Subject: [PATCH 170/246] fix black and isort linting errors --- tests/modules/install.py | 3 ++- tests/modules/lint.py | 8 ++++---- tests/modules/patch.py | 6 +++--- tests/modules/remove.py | 2 ++ tests/modules/update.py | 3 ++- tests/subworkflows/install.py | 4 ++-- tests/subworkflows/list.py | 3 +-- tests/test_modules.py | 2 +- tests/test_subworkflows.py | 2 +- tests/utils.py | 7 +++---- 10 files changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/modules/install.py b/tests/modules/install.py index 985f916998..fa588daf00 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -9,10 +9,10 @@ GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, GITLAB_URL, + remove_template_modules, with_temporary_folder, ) -from ..utils import remove_template_modules def test_modules_install_nopipeline(self): """Test installing a module - no pipeline given""" @@ -87,6 +87,7 @@ def test_modules_install_tracking(self): "installed_by" ] == ["modules"] + def test_modules_install_alternate_remote(self): """Test installing a module from a different remote with the same organization path""" install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 21e9fdd16e..9780c6a3aa 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -5,8 +5,7 @@ import nf_core.modules -from ..utils import GITLAB_URL, set_wd -from ..utils import remove_template_modules +from ..utils import GITLAB_URL, remove_template_modules, set_wd from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf @@ -72,11 +71,12 @@ def test_modules_lint_gitlab_modules(self): assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 -#should we remove this? We want to test multiple remotes but not for linting! + +# should we remove this? We want to test multiple remotes but not for linting! def test_modules_lint_multiple_remotes(self): """Lint modules from a different remote""" remove_template_modules(self) - #self.mods_install.install("fastqc") + # self.mods_install.install("fastqc") self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) module_lint.lint(print_results=False, all_modules=True) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 370dd88d82..26ca5805a5 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -7,9 +7,7 @@ import nf_core.components.components_command import nf_core.modules -from ..utils import GITLAB_URL - -from ..utils import remove_template_modules +from ..utils import GITLAB_URL, remove_template_modules """ Test the 'nf-core modules patch' command @@ -41,6 +39,7 @@ def setup_patch(pipeline_dir, modify_module, pipeline_name): module_path = Path(pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) modify_main_nf(module_path / "main.nf") + def modify_workflow_nf(path): with open(path, "r") as fh: lines = fh.readlines() @@ -49,6 +48,7 @@ def modify_workflow_nf(path): if not line.startswith("include {"): fh.write(line) + def modify_main_nf(path): """Modify a file to test patch creation""" with open(path, "r") as fh: diff --git a/tests/modules/remove.py b/tests/modules/remove.py index b2af1f10bd..267c757e56 100644 --- a/tests/modules/remove.py +++ b/tests/modules/remove.py @@ -2,6 +2,7 @@ from ..utils import remove_template_modules + def test_modules_remove_trimgalore(self): """Test removing TrimGalore! module after installing it""" self.mods_install.install("trimgalore") @@ -14,6 +15,7 @@ def test_modules_remove_trimgalore_uninstalled(self): """Test removing TrimGalore! module without installing it""" assert self.mods_remove.remove("trimgalore") is False + # Should this check be removed? def test_modules_remove_multiqc_from_gitlab(self): """Test removing multiqc module after installing it from an alternative source""" diff --git a/tests/modules/update.py b/tests/modules/update.py index ea94488b5d..e28f058760 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -24,9 +24,9 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, + remove_template_modules, ) -from ..utils import remove_template_modules def test_install_and_update(self): """Installs a module in the pipeline and updates it (no change)""" @@ -42,6 +42,7 @@ def test_install_and_update(self): assert update_obj.update("trimgalore") is True assert cmp_module(trimgalore_tmpdir, trimgalore_path) is True + def test_install_at_hash_and_update(self): """Installs an old version of a module in the pipeline and updates it""" remove_template_modules(self) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index bf41694e1a..1c451d8964 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -10,11 +10,10 @@ GITLAB_REPO, GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, + remove_template_modules, with_temporary_folder, ) -from ..utils import remove_template_modules - def test_subworkflow_install_nopipeline(self): """Test installing a subworkflow - no pipeline given""" @@ -144,6 +143,7 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self): ] ) == sorted(["subworkflows", "bam_sort_stats_samtools"]) + def test_subworkflows_install_alternate_remote(self): """Test installing a subworkflow from a different remote with the same organization path""" self.subworkflow_install.install("bam_sort_stats_samtools") diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index 2d44112859..cde6808bb8 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -2,9 +2,8 @@ import nf_core.subworkflows -from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL +from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, remove_template_modules -from ..utils import remove_template_modules def test_subworkflows_list_remote(self): """Test listing available subworkflows""" diff --git a/tests/test_modules.py b/tests/test_modules.py index e3fdd810ea..057e77737c 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -151,6 +151,7 @@ def test_modulesrepo_class(self): test_modules_info_remote_gitlab, ) from .modules.install import ( + test_modules_install_alternate_remote, test_modules_install_different_branch_fail, test_modules_install_different_branch_succeed, test_modules_install_emptypipeline, @@ -160,7 +161,6 @@ def test_modulesrepo_class(self): test_modules_install_tracking, test_modules_install_trimgalore, test_modules_install_trimgalore_twice, - test_modules_install_alternate_remote, ) from .modules.lint import ( test_modules_lint_empty, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 336e736da1..27373d2478 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -109,6 +109,7 @@ def tearDown(self): ) from .subworkflows.install import ( test_subworkflow_install_nopipeline, + test_subworkflows_install_alternate_remote, test_subworkflows_install_bam_sort_stats_samtools, test_subworkflows_install_bam_sort_stats_samtools_twice, test_subworkflows_install_different_branch_fail, @@ -118,7 +119,6 @@ def tearDown(self): test_subworkflows_install_tracking, test_subworkflows_install_tracking_added_already_installed, test_subworkflows_install_tracking_added_super_subworkflow, - test_subworkflows_install_alternate_remote, ) from .subworkflows.list import ( test_subworkflows_install_and_list_subworkflows, diff --git a/tests/utils.py b/tests/utils.py index eafe9bdaac..f33af2a7af 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -107,6 +107,7 @@ def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module, version): } rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) + def remove_template_modules(self): # Remove modules that may cause org_path conflict workflow_path = Path(self.pipeline_dir, "workflows", self.pipeline_name + ".nf") @@ -117,9 +118,7 @@ def remove_template_modules(self): if not line.startswith("include {"): fh.write(line) - remove_obj = nf_core.modules.ModuleRemove( - self.pipeline_dir, no_pull=False - ) + remove_obj = nf_core.modules.ModuleRemove(self.pipeline_dir, no_pull=False) - for i in ['multiqc','fastqc','custom/dumpsoftwareversions']: + for i in ["multiqc", "fastqc", "custom/dumpsoftwareversions"]: remove_obj.remove(i) From 2de51673c711f53ecca78107551b58e7943f6735 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Thu, 6 Apr 2023 10:37:53 +0200 Subject: [PATCH 171/246] Update nf_core/pipeline-template/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/pipeline-template/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 19e2121b4e..d32d73a3f9 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -92,7 +92,7 @@ For more details, please refer to the [usage documentation](https://nf-co.re/{{ ## Pipeline output -[results](https://nf-co.re/{{ short_name }}/results) section of the documentation. +To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/{{ short_name }}/results) section of the documentation. For more details about the output files and reports, please refer to the [output documentation](https://nf-co.re/{{ short_name }}/output). From 14c8a26bc3ae0e3a765bd7eae27b2aad45cd658c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 6 Apr 2023 16:17:16 +0200 Subject: [PATCH 172/246] fix test_modules_lint_new_modules --- nf_core/module-template/modules/meta.yml | 14 ++++++++------ tests/modules/lint.py | 2 +- tests/test_modules.py | 11 +++++++++++ tests/utils.py | 4 ++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 5def7dd301..d834e1f866 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -7,6 +7,8 @@ name: "{{ component_name_underscore }}" description: write your description here keywords: - sort + - example + - genomics tools: - "{{ component }}": {% if not_empty_template -%} @@ -34,9 +36,9 @@ input: ## TODO nf-core: Delete / customise this example input {%- endif %} - {{ 'bam:' if not_empty_template else "input:" }} - type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + type: file + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} {% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as output @@ -57,9 +59,9 @@ output: ## TODO nf-core: Delete / customise this example output {%- endif %} - {{ 'bam:' if not_empty_template else "output:" }} - type: file - description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} - pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} + type: file + description: {{ 'Sorted BAM/CRAM/SAM file' if not_empty_template else "" }} + pattern: {{ '"*.{bam,cram,sam}"' if not_empty_template else "" }} authors: - "{{ author }}" diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 18c0dc4dab..a4181a358b 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -43,7 +43,7 @@ def test_modules_lint_empty(self): def test_modules_lint_new_modules(self): - """lint all modules in nf-core/modules repo clone""" + """lint a new module""" module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=True, all_modules=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" diff --git a/tests/test_modules.py b/tests/test_modules.py index 21c003112e..77769ad11d 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -45,6 +45,17 @@ def create_modules_repo_dummy(tmp_dir): with requests_cache.disabled(): module_create.create() + # Remove doi from meta.yml which makes lint fail + meta_yml = os.path.join(root_dir, "modules", "nf-core", "bpipe", "test", "meta.yml") + with open(meta_yml, "r") as fh: + lines = fh.readlines() + for line_index in range(len(lines)): + if "doi" in lines[line_index]: + to_pop = line_index + lines.pop(to_pop) + with open(meta_yml, "w") as fh: + fh.writelines(lines) + return root_dir diff --git a/tests/utils.py b/tests/utils.py index 0dd60dd051..ba21eea382 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -76,8 +76,8 @@ def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): anaconda_mock = { "latest_version": version.split("--")[0], "summary": "", - "doc_url": "", - "dev_url": "", + "doc_url": "http://test", + "dev_url": "http://test", "files": [{"version": version.split("--")[0]}], "license": "", } From 70ce71bf80f9f5dffc53e2c0af9ce70574fde9a7 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 6 Apr 2023 12:15:10 -0400 Subject: [PATCH 173/246] fix MakeTestWorkflow CI tests --- .github/workflows/create-lint-wf.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index d09bdfd822..aa9af45b64 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -107,15 +107,24 @@ jobs: - name: nf-core modules install run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ - - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ - name: nf-core modules list remote run: nf-core --log-file log.txt modules list remote + - name: nf-core modules remove + run: | + nf-core --log-file log.txt modules remove fastqc + nf-core --log-file log.txt modules remove multiqc + nf-core --log-file log.txt modules remove custom/dumpsoftwareversions + + - name: nf-core modules install gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ + + - name: nf-core modules list local gitlab + run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ + - name: nf-core modules list remote gitlab run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote From 3593c3c72b681778b4824e19abb5194e2f986eca Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 6 Apr 2023 12:47:54 -0400 Subject: [PATCH 174/246] fix MakeTestWorkflow CI tests again --- .github/workflows/create-lint-wf.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index aa9af45b64..3fb520189d 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -115,9 +115,9 @@ jobs: - name: nf-core modules remove run: | - nf-core --log-file log.txt modules remove fastqc - nf-core --log-file log.txt modules remove multiqc - nf-core --log-file log.txt modules remove custom/dumpsoftwareversions + nf-core --log-file log.txt modules remove fastqc --dir nf-core-testpipeline/ + nf-core --log-file log.txt modules remove multiqc --dir nf-core-testpipeline/ + nf-core --log-file log.txt modules remove custom/dumpsoftwareversions --dir nf-core-testpipeline/ - name: nf-core modules install gitlab run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ From 195ee31d02027879769082307548e8b20e3c3e66 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 6 Apr 2023 12:48:09 -0400 Subject: [PATCH 175/246] cleanup --- tests/modules/lint.py | 2 -- tests/modules/remove.py | 1 - 2 files changed, 3 deletions(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 9780c6a3aa..cc38e0ac34 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -72,11 +72,9 @@ def test_modules_lint_gitlab_modules(self): assert len(module_lint.warned) >= 0 -# should we remove this? We want to test multiple remotes but not for linting! def test_modules_lint_multiple_remotes(self): """Lint modules from a different remote""" remove_template_modules(self) - # self.mods_install.install("fastqc") self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) module_lint.lint(print_results=False, all_modules=True) diff --git a/tests/modules/remove.py b/tests/modules/remove.py index 267c757e56..9a8953a7ca 100644 --- a/tests/modules/remove.py +++ b/tests/modules/remove.py @@ -16,7 +16,6 @@ def test_modules_remove_trimgalore_uninstalled(self): assert self.mods_remove.remove("trimgalore") is False -# Should this check be removed? def test_modules_remove_multiqc_from_gitlab(self): """Test removing multiqc module after installing it from an alternative source""" # Remove modules that may cause org_path conflict From 3f44849d8fa79b1ee9edd4959a7ba764b702d662 Mon Sep 17 00:00:00 2001 From: Anne Marie Noronha Date: Thu, 6 Apr 2023 13:42:11 -0400 Subject: [PATCH 176/246] fix MakeTestWorkflow CI tests again --- .github/workflows/create-lint-wf.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 3fb520189d..a5b69650bf 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -115,6 +115,7 @@ jobs: - name: nf-core modules remove run: | + find nf-core-testpipeline/workflows/ -type f -exec sed -i '/^include /d' {} \; nf-core --log-file log.txt modules remove fastqc --dir nf-core-testpipeline/ nf-core --log-file log.txt modules remove multiqc --dir nf-core-testpipeline/ nf-core --log-file log.txt modules remove custom/dumpsoftwareversions --dir nf-core-testpipeline/ From fb1f0bcf97f2753ae13cba6807d1fe452a7e6573 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 18 Apr 2023 13:00:35 +0200 Subject: [PATCH 177/246] remove remove_template_modules --- .github/workflows/create-lint-wf.yml | 16 +++------------- tests/modules/install.py | 3 --- tests/modules/patch.py | 16 +--------------- tests/modules/remove.py | 5 ----- tests/modules/update.py | 10 ---------- tests/subworkflows/install.py | 2 -- tests/subworkflows/list.py | 3 +-- tests/utils.py | 16 ---------------- 8 files changed, 5 insertions(+), 66 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index a5b69650bf..d09bdfd822 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -107,25 +107,15 @@ jobs: - name: nf-core modules install run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force + - name: nf-core modules install gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ + - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ - name: nf-core modules list remote run: nf-core --log-file log.txt modules list remote - - name: nf-core modules remove - run: | - find nf-core-testpipeline/workflows/ -type f -exec sed -i '/^include /d' {} \; - nf-core --log-file log.txt modules remove fastqc --dir nf-core-testpipeline/ - nf-core --log-file log.txt modules remove multiqc --dir nf-core-testpipeline/ - nf-core --log-file log.txt modules remove custom/dumpsoftwareversions --dir nf-core-testpipeline/ - - - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ - - - name: nf-core modules list local gitlab - run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ - - name: nf-core modules list remote gitlab run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote diff --git a/tests/modules/install.py b/tests/modules/install.py index fa588daf00..01f58b6fd2 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -9,7 +9,6 @@ GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, GITLAB_URL, - remove_template_modules, with_temporary_folder, ) @@ -50,7 +49,6 @@ def test_modules_install_trimgalore_twice(self): def test_modules_install_from_gitlab(self): """Test installing a module from GitLab""" - remove_template_modules(self) assert self.mods_install_gitlab.install("fastqc") is True @@ -63,7 +61,6 @@ def test_modules_install_different_branch_fail(self): def test_modules_install_different_branch_succeed(self): """Test installing a module from a different branch""" - remove_template_modules(self) install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) # The fastp module does exists in the branch-test branch assert install_obj.install("fastp") is True diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 26ca5805a5..aba87e1857 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -7,7 +7,7 @@ import nf_core.components.components_command import nf_core.modules -from ..utils import GITLAB_URL, remove_template_modules +from ..utils import GITLAB_URL """ Test the 'nf-core modules patch' command @@ -69,9 +69,6 @@ def modify_main_nf(path): def test_create_patch_no_change(self): """Test creating a patch when there is no change to the module""" - # Remove modules that may cause org_path conflict - remove_template_modules(self) - setup_patch(self.pipeline_dir, False, self.pipeline_name) # Try creating a patch file @@ -91,9 +88,6 @@ def test_create_patch_no_change(self): def test_create_patch_change(self): """Test creating a patch when there is a change to the module""" - # Remove modules that may cause org_path conflict - remove_template_modules(self) - setup_patch(self.pipeline_dir, True, self.pipeline_name) # Try creating a patch file @@ -127,8 +121,6 @@ def test_create_patch_try_apply_successful(self): """ Test creating a patch file and applying it to a new version of the the files """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) setup_patch(self.pipeline_dir, True, self.pipeline_name) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) @@ -196,8 +188,6 @@ def test_create_patch_try_apply_failed(self): """ Test creating a patch file and applying it to a new version of the the files """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) setup_patch(self.pipeline_dir, True, self.pipeline_name) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) @@ -237,8 +227,6 @@ def test_create_patch_update_success(self): Should have the same effect as 'test_create_patch_try_apply_successful' but uses higher level api """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) setup_patch(self.pipeline_dir, True, self.pipeline_name) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) @@ -301,8 +289,6 @@ def test_create_patch_update_fail(self): """ Test creating a patch file and updating a module when there is a diff conflict """ - # Remove modules that may cause org_path conflict - remove_template_modules(self) setup_patch(self.pipeline_dir, True, self.pipeline_name) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) diff --git a/tests/modules/remove.py b/tests/modules/remove.py index 9a8953a7ca..b343a02da7 100644 --- a/tests/modules/remove.py +++ b/tests/modules/remove.py @@ -1,7 +1,5 @@ import os -from ..utils import remove_template_modules - def test_modules_remove_trimgalore(self): """Test removing TrimGalore! module after installing it""" @@ -18,9 +16,6 @@ def test_modules_remove_trimgalore_uninstalled(self): def test_modules_remove_multiqc_from_gitlab(self): """Test removing multiqc module after installing it from an alternative source""" - # Remove modules that may cause org_path conflict - remove_template_modules(self) - self.mods_install_gitlab.install("multiqc") module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core", "multiqc") assert self.mods_remove_gitlab.remove("multiqc", force=True) diff --git a/tests/modules/update.py b/tests/modules/update.py index e28f058760..fcfd92fc39 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -24,7 +24,6 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, - remove_template_modules, ) @@ -45,7 +44,6 @@ def test_install_and_update(self): def test_install_at_hash_and_update(self): """Installs an old version of a module in the pipeline and updates it""" - remove_template_modules(self) assert self.mods_install_old.install("trimgalore") update_obj = ModuleUpdate( self.pipeline_dir, show_diff=False, update_deps=True, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH @@ -71,7 +69,6 @@ def test_install_at_hash_and_update(self): def test_install_at_hash_and_update_and_save_diff_to_file(self): """Installs an old version of a module in the pipeline and updates it""" - remove_template_modules(self) self.mods_install_old.install("trimgalore") patch_path = os.path.join(self.pipeline_dir, "trimgalore.patch") update_obj = ModuleUpdate( @@ -112,7 +109,6 @@ def test_update_all(self): def test_update_with_config_fixed_version(self): """Try updating when there are entries in the .nf-core.yml""" - remove_template_modules(self) # Install trimgalore at the latest version assert self.mods_install_trimgalore.install("trimgalore") @@ -138,7 +134,6 @@ def test_update_with_config_fixed_version(self): def test_update_with_config_dont_update(self): """Try updating when module is to be ignored""" - remove_template_modules(self) # Install an old version of trimgalore self.mods_install_old.install("trimgalore") @@ -169,7 +164,6 @@ def test_update_with_config_dont_update(self): def test_update_with_config_fix_all(self): """Fix the version of all nf-core modules""" - remove_template_modules(self) self.mods_install_trimgalore.install("trimgalore") # Fix the version of all nf-core modules in the .nf-core.yml to an old version @@ -193,7 +187,6 @@ def test_update_with_config_fix_all(self): def test_update_with_config_no_updates(self): """Don't update any nf-core modules""" - remove_template_modules(self) self.mods_install_old.install("trimgalore") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() @@ -227,7 +220,6 @@ def test_update_with_config_no_updates(self): def test_update_different_branch_single_module(self): """Try updating a module in a specific branch""" - remove_template_modules(self) install_obj = ModuleInstall( self.pipeline_dir, prompt=False, @@ -254,7 +246,6 @@ def test_update_different_branch_single_module(self): def test_update_different_branch_mixed_modules_main(self): """Try updating all modules where MultiQC is installed from main branch""" - remove_template_modules(self) # Install fastp assert self.mods_install_gitlab_old.install("fastp") @@ -281,7 +272,6 @@ def test_update_different_branch_mixed_modules_main(self): def test_update_different_branch_mix_modules_branch_test(self): """Try updating all modules where MultiQC is installed from branch-test branch""" - remove_template_modules(self) # Install multiqc from the branch-test branch assert self.mods_install_gitlab_old.install( "multiqc" diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 1c451d8964..c7fe5e18e2 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -10,7 +10,6 @@ GITLAB_REPO, GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, - remove_template_modules, with_temporary_folder, ) @@ -63,7 +62,6 @@ def test_subworkflows_install_bam_sort_stats_samtools_twice(self): def test_subworkflows_install_from_gitlab(self): """Test installing a subworkflow from GitLab""" - remove_template_modules(self) assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is True # Verify that the branch entry was added correctly modules_json = ModulesJson(self.pipeline_dir) diff --git a/tests/subworkflows/list.py b/tests/subworkflows/list.py index cde6808bb8..c65999d42c 100644 --- a/tests/subworkflows/list.py +++ b/tests/subworkflows/list.py @@ -2,7 +2,7 @@ import nf_core.subworkflows -from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, remove_template_modules +from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL def test_subworkflows_list_remote(self): @@ -40,7 +40,6 @@ def test_subworkflows_install_and_list_subworkflows(self): def test_subworkflows_install_gitlab_and_list_subworkflows(self): """Test listing locally installed subworkflows""" - remove_template_modules(self) self.subworkflow_install_gitlab.install("bam_sort_stats_samtools") subworkflows_list = nf_core.subworkflows.SubworkflowList(self.pipeline_dir, remote=False) listed_subworkflows = subworkflows_list.list_components() diff --git a/tests/utils.py b/tests/utils.py index f33af2a7af..3fd265dd0c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -106,19 +106,3 @@ def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module, version): ], } rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) - - -def remove_template_modules(self): - # Remove modules that may cause org_path conflict - workflow_path = Path(self.pipeline_dir, "workflows", self.pipeline_name + ".nf") - with open(workflow_path, "r") as fh: - lines = fh.readlines() - with open(workflow_path, "w") as fh: - for line in lines: - if not line.startswith("include {"): - fh.write(line) - - remove_obj = nf_core.modules.ModuleRemove(self.pipeline_dir, no_pull=False) - - for i in ["multiqc", "fastqc", "custom/dumpsoftwareversions"]: - remove_obj.remove(i) From 1ddaad8e3e5e76bad126776c04f5445e73b55d33 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 18 Apr 2023 14:52:25 +0200 Subject: [PATCH 178/246] remove the steps which were removing pre-installed modules from gitlab nf-core --- tests/modules/install.py | 3 ++- tests/modules/lint.py | 5 +---- tests/subworkflows/install.py | 2 +- tests/test_subworkflows.py | 14 +++++++++++++- tests/utils.py | 4 +++- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/modules/install.py b/tests/modules/install.py index 01f58b6fd2..7f7f06de39 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -6,6 +6,7 @@ from nf_core.modules.modules_json import ModulesJson from ..utils import ( + GITLAB_BRANCH_ORG_PATH_BRANCH, GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, GITLAB_URL, @@ -87,6 +88,6 @@ def test_modules_install_tracking(self): def test_modules_install_alternate_remote(self): """Test installing a module from a different remote with the same organization path""" - install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH) + install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_ORG_PATH_BRANCH) # The fastp module does exists in the branch-test branch assert install_obj.install("fastp") is False diff --git a/tests/modules/lint.py b/tests/modules/lint.py index cc38e0ac34..9bab9eddeb 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -5,7 +5,7 @@ import nf_core.modules -from ..utils import GITLAB_URL, remove_template_modules, set_wd +from ..utils import GITLAB_URL, set_wd from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf @@ -62,7 +62,6 @@ def test_modules_lint_no_gitlab(self): def test_modules_lint_gitlab_modules(self): """Lint modules from a different remote""" - remove_template_modules(self) self.mods_install_gitlab.install("fastqc") self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) @@ -74,7 +73,6 @@ def test_modules_lint_gitlab_modules(self): def test_modules_lint_multiple_remotes(self): """Lint modules from a different remote""" - remove_template_modules(self) self.mods_install_gitlab.install("multiqc") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) module_lint.lint(print_results=False, all_modules=True) @@ -87,7 +85,6 @@ def test_modules_lint_patched_modules(self): """ Test creating a patch file and applying it to a new version of the the files """ - remove_template_modules(self) setup_patch(self.pipeline_dir, True) # Create a patch file diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index c7fe5e18e2..653de93a09 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -145,4 +145,4 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self): def test_subworkflows_install_alternate_remote(self): """Test installing a subworkflow from a different remote with the same organization path""" self.subworkflow_install.install("bam_sort_stats_samtools") - assert self.subworkflow_install_gitlab.install("bam_stats_samtools") is False + assert self.subworkflow_install_gitlab_same_org_path.install("bam_stats_samtools") is False diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 27373d2478..1c290cb882 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -12,7 +12,12 @@ import nf_core.modules import nf_core.subworkflows -from .utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL, OLD_SUBWORKFLOWS_SHA +from .utils import ( + GITLAB_SUBWORKFLOWS_BRANCH, + GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, + GITLAB_URL, + OLD_SUBWORKFLOWS_SHA, +) def create_modules_repo_dummy(tmp_dir): @@ -62,6 +67,13 @@ def setUp(self): self.subworkflow_install_gitlab = nf_core.subworkflows.SubworkflowInstall( self.pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH ) + self.subworkflow_install_gitlab_same_org_path = nf_core.subworkflows.SubworkflowInstall( + self.pipeline_dir, + prompt=False, + force=False, + remote_url=GITLAB_URL, + branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, + ) self.subworkflow_install_old = nf_core.subworkflows.SubworkflowInstall( self.pipeline_dir, prompt=False, diff --git a/tests/utils.py b/tests/utils.py index 3fd265dd0c..a49de974fc 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -15,12 +15,14 @@ OLD_TRIMGALORE_SHA = "06348dffce2a732fc9e656bdc5c64c3e02d302cb" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" -GITLAB_REPO = "nf-core" +GITLAB_REPO = "nf-core-test" GITLAB_DEFAULT_BRANCH = "main" GITLAB_SUBWORKFLOWS_BRANCH = "subworkflows" +GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH = "subworkflows-org-path" OLD_SUBWORKFLOWS_SHA = "f3c078809a2513f1c95de14f6633fe1f03572fdb" # Branch test stuff GITLAB_BRANCH_TEST_BRANCH = "branch-tester" +GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" GITLAB_BRANCH_TEST_OLD_SHA = "bce3f17980b8d1beae5e917cfd3c65c0c69e04b5" GITLAB_BRANCH_TEST_NEW_SHA = "2f5f180f6e705bb81d6e7742dc2f24bf4a0c721e" From 29c0defbb3058dc87fb33e4d6eabd1f387073546 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 09:12:13 +0200 Subject: [PATCH 179/246] try to fix some tests --- .github/workflows/create-lint-wf.yml | 2 +- tests/modules/install.py | 14 +++++++++++--- tests/modules/update.py | 2 +- tests/test_modules.py | 6 +++--- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index d09bdfd822..391a8cb9c0 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -108,7 +108,7 @@ jobs: run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --force --dir nf-core-testpipeline/ + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --dir nf-core-testpipeline/ - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ diff --git a/tests/modules/install.py b/tests/modules/install.py index 7f7f06de39..0a072159d3 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -86,8 +86,16 @@ def test_modules_install_tracking(self): ] == ["modules"] -def test_modules_install_alternate_remote(self): +@pytest.fixture +def test_modules_install_alternate_remote(self, caplog): """Test installing a module from a different remote with the same organization path""" install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_ORG_PATH_BRANCH) - # The fastp module does exists in the branch-test branch - assert install_obj.install("fastp") is False + # Install fastqc from GitLab which is also installed from GitHub with the same org_path + install_obj.install("fastqc") + assert ( + caplog.msg + == "You are trying to install modules from different repositories with the same organization name 'nf-core'" + ) + # with pytest.raises(UserWarning) as excinfo: + # install_obj.install("fastqc")) + # assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) diff --git a/tests/modules/update.py b/tests/modules/update.py index fcfd92fc39..399e9cc12c 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -187,7 +187,7 @@ def test_update_with_config_fix_all(self): def test_update_with_config_no_updates(self): """Don't update any nf-core modules""" - self.mods_install_old.install("trimgalore") + assert self.mods_install_old.install("trimgalore") old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json() # Fix the version of all nf-core modules in the .nf-core.yml to an old version diff --git a/tests/test_modules.py b/tests/test_modules.py index 057e77737c..74795c8080 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -78,21 +78,21 @@ def setUp(self): self.mods_install_trimgalore = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, - force=True, + force=False, remote_url=GITLAB_URL, branch=OLD_TRIMGALORE_BRANCH, ) self.mods_install_gitlab = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, - force=True, + force=False, remote_url=GITLAB_URL, branch=GITLAB_DEFAULT_BRANCH, ) self.mods_install_gitlab_old = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, - force=True, + force=False, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_TEST_BRANCH, sha=GITLAB_BRANCH_TEST_OLD_SHA, From 3ddfb83471555f29e18ade596218af01a2b831d2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 09:27:54 +0200 Subject: [PATCH 180/246] fix test_modules_install_alternate_remote and remove test_subworkflows_install_alternate_remote --- tests/modules/install.py | 14 ++++---------- tests/subworkflows/install.py | 6 ------ 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/tests/modules/install.py b/tests/modules/install.py index 0a072159d3..deca31204e 100644 --- a/tests/modules/install.py +++ b/tests/modules/install.py @@ -86,16 +86,10 @@ def test_modules_install_tracking(self): ] == ["modules"] -@pytest.fixture -def test_modules_install_alternate_remote(self, caplog): +def test_modules_install_alternate_remote(self): """Test installing a module from a different remote with the same organization path""" install_obj = ModuleInstall(self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_BRANCH_ORG_PATH_BRANCH) # Install fastqc from GitLab which is also installed from GitHub with the same org_path - install_obj.install("fastqc") - assert ( - caplog.msg - == "You are trying to install modules from different repositories with the same organization name 'nf-core'" - ) - # with pytest.raises(UserWarning) as excinfo: - # install_obj.install("fastqc")) - # assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) + with pytest.raises(Exception) as excinfo: + install_obj.install("fastqc") + assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 653de93a09..6c04c9ad22 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -140,9 +140,3 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self): "installed_by" ] ) == sorted(["subworkflows", "bam_sort_stats_samtools"]) - - -def test_subworkflows_install_alternate_remote(self): - """Test installing a subworkflow from a different remote with the same organization path""" - self.subworkflow_install.install("bam_sort_stats_samtools") - assert self.subworkflow_install_gitlab_same_org_path.install("bam_stats_samtools") is False From 359fa4ff463b6606360d1ecc57cef36714fdda5c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 09:51:27 +0200 Subject: [PATCH 181/246] remove unused modifications --- tests/modules/patch.py | 23 +++++++---------------- tests/test_modules.py | 1 - 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index aba87e1857..989aee8b8b 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -26,7 +26,7 @@ REPO_URL = "https://gitlab.com/nf-core/modules-test.git" -def setup_patch(pipeline_dir, modify_module, pipeline_name): +def setup_patch(pipeline_dir, modify_module): install_obj = nf_core.modules.ModuleInstall( pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=ORG_SHA ) @@ -40,15 +40,6 @@ def setup_patch(pipeline_dir, modify_module, pipeline_name): modify_main_nf(module_path / "main.nf") -def modify_workflow_nf(path): - with open(path, "r") as fh: - lines = fh.readlines() - with open(path, "w") as fh: - for line in lines: - if not line.startswith("include {"): - fh.write(line) - - def modify_main_nf(path): """Modify a file to test patch creation""" with open(path, "r") as fh: @@ -69,7 +60,7 @@ def modify_main_nf(path): def test_create_patch_no_change(self): """Test creating a patch when there is no change to the module""" - setup_patch(self.pipeline_dir, False, self.pipeline_name) + setup_patch(self.pipeline_dir, False) # Try creating a patch file patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) @@ -88,7 +79,7 @@ def test_create_patch_no_change(self): def test_create_patch_change(self): """Test creating a patch when there is a change to the module""" - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) # Try creating a patch file patch_obj = nf_core.modules.ModulePatch(self.pipeline_dir, GITLAB_URL, PATCH_BRANCH) @@ -122,7 +113,7 @@ def test_create_patch_try_apply_successful(self): Test creating a patch file and applying it to a new version of the the files """ - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) module_path = Path(self.pipeline_dir, module_relpath) @@ -189,7 +180,7 @@ def test_create_patch_try_apply_failed(self): Test creating a patch file and applying it to a new version of the the files """ - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_relpath = Path("modules", REPO_NAME, BISMARK_ALIGN) module_path = Path(self.pipeline_dir, module_relpath) @@ -228,7 +219,7 @@ def test_create_patch_update_success(self): but uses higher level api """ - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) # Try creating a patch file @@ -290,7 +281,7 @@ def test_create_patch_update_fail(self): Test creating a patch file and updating a module when there is a diff conflict """ - setup_patch(self.pipeline_dir, True, self.pipeline_name) + setup_patch(self.pipeline_dir, True) module_path = Path(self.pipeline_dir, "modules", REPO_NAME, BISMARK_ALIGN) # Try creating a patch file diff --git a/tests/test_modules.py b/tests/test_modules.py index 74795c8080..1c24f9be7e 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -66,7 +66,6 @@ def setUp(self): ).init_pipeline() # Set up install objects self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) - self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True) self.mods_install_old = nf_core.modules.ModuleInstall( self.pipeline_dir, prompt=False, From 66092a8122e52c69190a84f6d7cda88ac13d2ee6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 12:58:57 +0200 Subject: [PATCH 182/246] update sha for tests using gitlab --- tests/modules/patch.py | 10 +++++----- tests/utils.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 989aee8b8b..49c1a9255b 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -16,12 +16,12 @@ testing if the update commands works correctly with patch files """ -ORG_SHA = "775fcd090fb776a0be695044f8ab1af8896c8452" -CORRECT_SHA = "335cd32405568ca3b6d4c05ab1e8a98c21e18a4d" -SUCCEED_SHA = "f1566140c752e9c68fffc189fbe8cb9ee942b3ca" -FAIL_SHA = "1fc8b0f953d915d66ee40d28bc337ff0998d05bd" +ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" +CORRECT_SHA = "63fd3cdb1be733041db74c15542a7b5b8f4095ed" +SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" +FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align" -REPO_NAME = "nf-core" +REPO_NAME = "nf-core-test" PATCH_BRANCH = "patch-tester" REPO_URL = "https://gitlab.com/nf-core/modules-test.git" diff --git a/tests/utils.py b/tests/utils.py index a49de974fc..ea6eac7315 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,7 +12,7 @@ import nf_core.modules -OLD_TRIMGALORE_SHA = "06348dffce2a732fc9e656bdc5c64c3e02d302cb" +OLD_TRIMGALORE_SHA = "9b7a3bdefeaad5d42324aa7dd50f87bea1b04386" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" GITLAB_REPO = "nf-core-test" @@ -23,8 +23,8 @@ # Branch test stuff GITLAB_BRANCH_TEST_BRANCH = "branch-tester" GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" -GITLAB_BRANCH_TEST_OLD_SHA = "bce3f17980b8d1beae5e917cfd3c65c0c69e04b5" -GITLAB_BRANCH_TEST_NEW_SHA = "2f5f180f6e705bb81d6e7742dc2f24bf4a0c721e" +GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca" +GITLAB_BRANCH_TEST_NEW_SHA = "f5a3da1aea979ba7fa49cc51d2b7b6cd3f25239e" def with_temporary_folder(func): From 82ac98b18bd0c6224dd37d8e4dd30d70ec430b0e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 13:00:46 +0200 Subject: [PATCH 183/246] fix path for test_modules_remove_multiqc_from_gitlab --- tests/modules/remove.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/remove.py b/tests/modules/remove.py index b343a02da7..e412fd35a8 100644 --- a/tests/modules/remove.py +++ b/tests/modules/remove.py @@ -17,6 +17,6 @@ def test_modules_remove_trimgalore_uninstalled(self): def test_modules_remove_multiqc_from_gitlab(self): """Test removing multiqc module after installing it from an alternative source""" self.mods_install_gitlab.install("multiqc") - module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core", "multiqc") + module_path = os.path.join(self.mods_install_gitlab.dir, "modules", "nf-core-test", "multiqc") assert self.mods_remove_gitlab.remove("multiqc", force=True) assert os.path.exists(module_path) is False From 4573a754c8e61e57883f2683f5825437f0f284ad Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 13:18:03 +0200 Subject: [PATCH 184/246] add modified test_subworkflows_install_alternate_remote --- tests/subworkflows/install.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 6c04c9ad22..dfe71686fb 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -9,6 +9,7 @@ GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, GITLAB_SUBWORKFLOWS_BRANCH, + GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, GITLAB_URL, with_temporary_folder, ) @@ -140,3 +141,14 @@ def test_subworkflows_install_tracking_added_super_subworkflow(self): "installed_by" ] ) == sorted(["subworkflows", "bam_sort_stats_samtools"]) + + +def test_subworkflows_install_alternate_remote(self): + """Test installing a module from a different remote with the same organization path""" + install_obj = SubworkflowInstall( + self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH + ) + # Install a subworkflow from GitLab which is also installed from GitHub with the same org_path + with pytest.raises(Exception) as excinfo: + install_obj.install("fastqc") + assert "Could not find a 'main.nf' or 'nextflow.config' file" in str(excinfo.value) From d51f82c6acfeb94ec1bc959b1a6ef50d4f0095e8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 15:06:05 +0200 Subject: [PATCH 185/246] fix create-lint-wf.yml GHA --- .github/workflows/create-lint-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 391a8cb9c0..b936d9a74a 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -108,7 +108,7 @@ jobs: run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch main install fastqc --dir nf-core-testpipeline/ + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastq --dir nf-core-testpipeline/ - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ From ecfa1019d297e8e90ee2a278bfe988c3b3f96998 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 15:15:03 +0200 Subject: [PATCH 186/246] fixing a typo --- .github/workflows/create-lint-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index b936d9a74a..f76187d6af 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -108,7 +108,7 @@ jobs: run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastq --dir nf-core-testpipeline/ + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastp --dir nf-core-testpipeline/ - name: nf-core modules list local run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ From bf22b785bc1fa263f4184000383196664ba44ba6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 15:54:22 +0200 Subject: [PATCH 187/246] refactor stale bot --- .../stale.yml => .github/workflows/stale-tagged.yml | 8 ++++---- .../pipeline-template/.github/workflows/stale-tagged.yml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) rename nf_core/pipeline-template/.github/workflows/stale.yml => .github/workflows/stale-tagged.yml (50%) rename .github/workflows/stale.yml => nf_core/pipeline-template/.github/workflows/stale-tagged.yml (54%) diff --git a/nf_core/pipeline-template/.github/workflows/stale.yml b/.github/workflows/stale-tagged.yml similarity index 50% rename from nf_core/pipeline-template/.github/workflows/stale.yml rename to .github/workflows/stale-tagged.yml index 8e121b39fa..7b54546e09 100644 --- a/nf_core/pipeline-template/.github/workflows/stale.yml +++ b/.github/workflows/stale-tagged.yml @@ -1,4 +1,4 @@ -name: "Close stale issues and stale PRs" +name: "Close user-tagged issues and PRs" on: schedule: - cron: "30 1 * * 7" # Once a week @@ -12,9 +12,9 @@ jobs: steps: - uses: actions/stale@v7 with: - stale-issue-message: "This issue is stale because it has been inactive for more than 30 days. More information is required. Remove stale label or comment or this will be closed in 20 days." - stale-pr-message: "This PR is stale because it has been open more than 45 days with no activity. Remove stale label or comment if it is still useful. In any case a PR will be automatically closed." - close-issue-message: "This issue was closed because it has been stalled for 20 days with no activity." + stale-issue-message: "This issue is stale because a user tagged it for stale. It has been inactive for more than 30 days. More information is required. Remove stale label or comment or this will be closed in 20 days." + stale-pr-message: "This PR is stale because a user tagged it for stale. It has been open more than 45 days with no activity. Remove stale label or comment if it is still useful. In any case a PR will be automatically closed." + close-issue-message: "This issue was closed because it has been staled for 20 days with no activity." days-before-stale: 30 days-before-close: 20 days-before-pr-close: -1 diff --git a/.github/workflows/stale.yml b/nf_core/pipeline-template/.github/workflows/stale-tagged.yml similarity index 54% rename from .github/workflows/stale.yml rename to nf_core/pipeline-template/.github/workflows/stale-tagged.yml index 8e121b39fa..906655a300 100644 --- a/.github/workflows/stale.yml +++ b/nf_core/pipeline-template/.github/workflows/stale-tagged.yml @@ -1,4 +1,4 @@ -name: "Close stale issues and stale PRs" +name: "Close user-tagged issues and PRs" on: schedule: - cron: "30 1 * * 7" # Once a week @@ -12,8 +12,8 @@ jobs: steps: - uses: actions/stale@v7 with: - stale-issue-message: "This issue is stale because it has been inactive for more than 30 days. More information is required. Remove stale label or comment or this will be closed in 20 days." - stale-pr-message: "This PR is stale because it has been open more than 45 days with no activity. Remove stale label or comment if it is still useful. In any case a PR will be automatically closed." + stale-issue-message: "This issue is stale because a user tagged it for stale. It has been inactive for more than 30 days. More information is required. Remove stale label or comment or this will be closed in 20 days." + stale-pr-message: "This PR is stale because a user tagged it for stale. It has been open more than 45 days with no activity. Remove stale label or comment if it is still useful. In any case a PR will be automatically closed." close-issue-message: "This issue was closed because it has been stalled for 20 days with no activity." days-before-stale: 30 days-before-close: 20 From 68019595938e7da13a2db68d41a29d336c5fe237 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 19 Apr 2023 16:30:17 +0200 Subject: [PATCH 188/246] hopefully this last sha update will fix all tests --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index ea6eac7315..86c07fd469 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -24,7 +24,7 @@ GITLAB_BRANCH_TEST_BRANCH = "branch-tester" GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca" -GITLAB_BRANCH_TEST_NEW_SHA = "f5a3da1aea979ba7fa49cc51d2b7b6cd3f25239e" +GITLAB_BRANCH_TEST_NEW_SHA = "7d73e21f30041297ea44367f2b4fd4e045c0b991" def with_temporary_folder(func): From 56a71edb5b89842f37790efc1bac2fb88766c93a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 20 Apr 2023 09:17:36 +0200 Subject: [PATCH 189/246] add review suggestions from @anoronh4 --- nf_core/modules/modules_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 431ad1d657..b9b64c56d2 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -427,8 +427,8 @@ def parse_dirs(self, dirs, missing_installation, component_type): git_url = None for repo in missing_installation: if component_type in missing_installation[repo]: - for dir_name in missing_installation[repo][component_type]: - if component in missing_installation[repo][component_type][dir_name]: + if install_dir in missing_installation[repo][component_type]: + if component in missing_installation[repo][component_type][install_dir]: component_in_file = True git_url = repo break From e25a81d523a0dbcacc5662af1c314bad55d98227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 20 Apr 2023 09:40:53 +0200 Subject: [PATCH 190/246] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- .../pipeline-template/.github/workflows/stale-tagged.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/stale-tagged.yml b/nf_core/pipeline-template/.github/workflows/stale-tagged.yml index 906655a300..1498286338 100644 --- a/nf_core/pipeline-template/.github/workflows/stale-tagged.yml +++ b/nf_core/pipeline-template/.github/workflows/stale-tagged.yml @@ -12,9 +12,10 @@ jobs: steps: - uses: actions/stale@v7 with: - stale-issue-message: "This issue is stale because a user tagged it for stale. It has been inactive for more than 30 days. More information is required. Remove stale label or comment or this will be closed in 20 days." - stale-pr-message: "This PR is stale because a user tagged it for stale. It has been open more than 45 days with no activity. Remove stale label or comment if it is still useful. In any case a PR will be automatically closed." - close-issue-message: "This issue was closed because it has been stalled for 20 days with no activity." + stale-issue-message: "This issue has been tagged as stale by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." + stale-issue-message: "This issue has been tagged as stale by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." + stale-pr-message: "This PR has been tagged as stale by an nf-core contributor. Remove stale label or add a comment if it is still useful. Otherwise, this PR will be automatically closed." + close-issue-message: "This issue was closed because it has been tagged as stale by an nf-core contributor and then stalled for 20 days with no activity." days-before-stale: 30 days-before-close: 20 days-before-pr-close: -1 From ee424335a45977cbc9f242a99c8d26f0dbf4952d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 20 Apr 2023 10:42:48 +0200 Subject: [PATCH 191/246] fix linting and typo --- nf_core/pipeline-template/.github/workflows/stale-tagged.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/stale-tagged.yml b/nf_core/pipeline-template/.github/workflows/stale-tagged.yml index 1498286338..5826d2eae0 100644 --- a/nf_core/pipeline-template/.github/workflows/stale-tagged.yml +++ b/nf_core/pipeline-template/.github/workflows/stale-tagged.yml @@ -13,9 +13,8 @@ jobs: - uses: actions/stale@v7 with: stale-issue-message: "This issue has been tagged as stale by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." - stale-issue-message: "This issue has been tagged as stale by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as stale by an nf-core contributor. Remove stale label or add a comment if it is still useful. Otherwise, this PR will be automatically closed." - close-issue-message: "This issue was closed because it has been tagged as stale by an nf-core contributor and then stalled for 20 days with no activity." + close-issue-message: "This issue was closed because it has been tagged as stale by an nf-core contributor and then staled for 20 days with no activity." days-before-stale: 30 days-before-close: 20 days-before-pr-close: -1 From 0e53823ab261b31ea363a0372ac39c3e2f9a1563 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 20 Apr 2023 11:27:06 +0200 Subject: [PATCH 192/246] fix jinja error --- .github/workflows/stale-tagged.yml | 2 +- nf_core/pipeline-template/.github/workflows/stale-tagged.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/stale-tagged.yml b/.github/workflows/stale-tagged.yml index 7b54546e09..1aa23930b3 100644 --- a/.github/workflows/stale-tagged.yml +++ b/.github/workflows/stale-tagged.yml @@ -1,7 +1,7 @@ name: "Close user-tagged issues and PRs" on: schedule: - - cron: "30 1 * * 7" # Once a week + - cron: "0 0 * * 0" # Once a week jobs: stale: diff --git a/nf_core/pipeline-template/.github/workflows/stale-tagged.yml b/nf_core/pipeline-template/.github/workflows/stale-tagged.yml index 5826d2eae0..292b44cf9c 100644 --- a/nf_core/pipeline-template/.github/workflows/stale-tagged.yml +++ b/nf_core/pipeline-template/.github/workflows/stale-tagged.yml @@ -1,7 +1,7 @@ name: "Close user-tagged issues and PRs" on: schedule: - - cron: "30 1 * * 7" # Once a week + - cron: "0 0 * * 0" # Once a week jobs: stale: @@ -21,4 +21,4 @@ jobs: any-of-labels: "awaiting-changes,awaiting-feedback" exempt-issue-labels: "WIP" exempt-pr-labels: "WIP" - repo-token: ${{ secrets.GITHUB_TOKEN }} + repo-token: "{% raw %}${{ secrets.GITHUB_TOKEN }}{% endraw %}" From 6c84332f5d736b4bd6aec90beb4bee0cb7ac37e0 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Thu, 20 Apr 2023 12:42:10 +0200 Subject: [PATCH 193/246] Apply suggestions from code review --- nf_core/pipeline-template/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index d32d73a3f9..4eccd953c6 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -43,14 +43,12 @@ ## Usage -{% if branded -%} > **Note** > If you are new to nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how > to set-up nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) > with `-profile test` before running the workflow on actual data. -{% endif -%} ```bash - nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile + nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile ``` {% if branded -%} diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 47270b7432..73e1132541 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -126,7 +126,7 @@ To further assist in reproducbility, you can use share and re-use [parameter fil Use this parameter to choose a configuration profile. Profiles can give configuration presets for different compute environments. -Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Conda) - see below. +Several generic profiles are bundled with the pipeline which instruct the pipeline to use software packaged using different methods (Docker, Singularity, Podman, Shifter, Charliecloud, Apptainer, Conda) - see below. > We highly recommend the use of Docker or Singularity containers for full pipeline reproducibility, however when this is not possible, Conda is also supported. @@ -153,8 +153,10 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Shifter](https://nersc.gitlab.io/development/shifter/how-to-use/) - `charliecloud` - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) +- `apptainer` + - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) - `conda` - - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter or Charliecloud. + - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. ### `-resume` diff --git a/nf_core/pipeline-template/lib/NfcoreSchema.groovy b/nf_core/pipeline-template/lib/NfcoreSchema.groovy index 4d29681431..9b34804d6d 100755 --- a/nf_core/pipeline-template/lib/NfcoreSchema.groovy +++ b/nf_core/pipeline-template/lib/NfcoreSchema.groovy @@ -84,6 +84,7 @@ class NfcoreSchema { 'stub-run', 'test', 'w', + 'with-apptainer', 'with-charliecloud', 'with-conda', 'with-dag', From 0bfd799eaaa7a4c5e6abdc0a3f0cc8e208d72430 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Fri, 21 Apr 2023 13:03:23 +0200 Subject: [PATCH 206/246] Update nf_core/pipeline-template/README.md Co-authored-by: Phil Ewels --- nf_core/pipeline-template/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index ecb843aae0..950a45415c 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -71,8 +71,9 @@ Now, you can run the pipeline using: ```bash nextflow run {{ name }} \ + -profile \ --input samplesheet.csv \ - -profile + --outdir ``` > **Warning:** From 8855813290f940a1f6ba1c5ebf7dc11eec4abebe Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Fri, 21 Apr 2023 13:03:54 +0200 Subject: [PATCH 207/246] Update nf_core/pipeline-template/README.md Co-authored-by: Phil Ewels --- nf_core/pipeline-template/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 950a45415c..d7d445a88b 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -27,7 +27,7 @@ From 1c2356d448e0405b856cc597835c8aa9177d03f5 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 21 Apr 2023 12:43:01 +0100 Subject: [PATCH 208/246] Add podman.registry config option --- nf_core/pipeline-template/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index b406b37378..94a591e8f4 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -127,6 +127,7 @@ profiles { } podman { podman.enabled = true + podman.registry = 'quay.io' conda.enabled = false docker.enabled = false singularity.enabled = false From 029bde300ebace568b05236e74911d45620fff0f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 21 Apr 2023 17:39:11 +0200 Subject: [PATCH 209/246] fix invalid dir type in meta.yml and update sha --- tests/modules/patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 49c1a9255b..b2e881969f 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -17,7 +17,7 @@ """ ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "63fd3cdb1be733041db74c15542a7b5b8f4095ed" +CORRECT_SHA = "65ba2ee3bf08c6bb31d880467df72ccdbc8d2b40" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align" From d822093473c72ea13a99387bda757352e56e125d Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Sun, 23 Apr 2023 19:12:26 +0200 Subject: [PATCH 210/246] Update TODO comment for short pipeline description --- nf_core/pipeline-template/README.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index d7d445a88b..7aeb9d5503 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -25,15 +25,14 @@ ## Introduction - -**{{ name }}** is a bioinformatics pipeline for {{ description }}. - @@ -43,13 +42,11 @@ ## Usage - > **Note** > If you are new to nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how > to set-up nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) > with `-profile test` before running the workflow on actual data. - @@ -85,7 +82,6 @@ nextflow run {{ name }} \ For more details, please refer to the [usage documentation](https://nf-co.re/{{ short_name }}/usage) and the [parameter documentation](https://nf-co.re/{{ short_name }}/parameters). - ## Pipeline output To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/{{ short_name }}/results) section of the documentation. From 549a6fef02c5f9b0a032d37bea7cae0279846e1d Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Sun, 23 Apr 2023 19:15:15 +0200 Subject: [PATCH 211/246] Put samplesheet description in comment --- nf_core/pipeline-template/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 7aeb9d5503..b678b40b39 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -27,8 +27,8 @@ **{{ name }}** is a bioinformatics pipeline that ... - @@ -48,12 +48,11 @@ > with `-profile test` before running the workflow on actual data. + Explain what rows and columns represent. For instance (please edit as appropriate): First, you need to prepare a samplesheet with your input data that looks as follows: -**samplesheet.csv**: +`samplesheet.csv`: ```csv sample,fastq_1,fastq_2 @@ -62,6 +61,8 @@ CONTROL_REP1,AEG588A1_S1_L002_R1_001.fastq.gz,AEG588A1_S1_L002_R2_001.fastq.gz Each row represents a fastq file (single-end) or a pair of fastq files (paired end). +--> + Now, you can run the pipeline using: From 832ef1e832dd8815a0cf2d487afa6d877d6aec01 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 24 Apr 2023 10:20:39 +0200 Subject: [PATCH 212/246] add log info --- nf_core/refgenie.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/refgenie.py b/nf_core/refgenie.py index 75b4752a62..177dfc78b7 100644 --- a/nf_core/refgenie.py +++ b/nf_core/refgenie.py @@ -59,6 +59,7 @@ def _print_nf_config(rgc): else: # Translate an alias name to the alias used in the pipeline if asset in alias_translations.keys(): + log.info(f"Translating refgenie asset alias {asset} to {alias_translations[asset]}.") asset = alias_translations[asset] genomes_str += f' {asset.ljust(20, " ")} = "{pth}"\n' genomes_str += " }\n" From 46a3c3845c572cd7fefc9fbe9f03f53f424477e4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 24 Apr 2023 10:43:36 +0200 Subject: [PATCH 213/246] obtain refgenie path also from refgenconf object --- nf_core/refgenie.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nf_core/refgenie.py b/nf_core/refgenie.py index 177dfc78b7..b666844699 100644 --- a/nf_core/refgenie.py +++ b/nf_core/refgenie.py @@ -47,7 +47,7 @@ def _print_nf_config(rgc): """ abg = rgc.list_assets_by_genome() genomes_str = "" - alias_translations = _get_alias_translation_file() + alias_translations = _get_alias_translation_file(rgc) for genome, asset_list in abg.items(): genomes_str += f" '{genome}' {{\n" for asset in asset_list: @@ -107,7 +107,7 @@ def _update_nextflow_home_config(refgenie_genomes_config_file, nxf_home): log.info(f"Created new nextflow config file: {nxf_home_config}") -def _get_alias_translation_file(): +def _get_alias_translation_file(rgc): """ Read a file containing alias translations. @@ -124,10 +124,12 @@ def _get_alias_translation_file(): if "REFGENIE" in os.environ: refgenie_genomes_config_path = os.environ.get("REFGENIE") + refgenie_genomes_config_directory = Path(refgenie_genomes_config_path).parents[0] + elif "genome_folder" in rgc: + refgenie_genomes_config_directory = Path(rgc["genome_folder"]) else: return translations - refgenie_genomes_config_directory = Path(refgenie_genomes_config_path).parents[0] try: with open(refgenie_genomes_config_directory / "alias_translations.yaml") as yaml_file: translations = yaml.load(yaml_file, Loader=yaml.Loader) From 00dcc4e9fb38a186e72e8c17ca1c3aee1c48ee10 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 25 Apr 2023 12:56:00 +0200 Subject: [PATCH 214/246] add schema hint for yaml language server to template --- README.md | 4 ++-- nf_core/module-template/modules/meta.yml | 2 +- nf_core/modules/lint/meta_yml.py | 2 +- .../modules/nf-core/custom/dumpsoftwareversions/meta.yml | 5 ++++- nf_core/subworkflow-template/subworkflows/meta.yml | 1 + 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 63bf235bc7..2fe09e80d2 100644 --- a/README.md +++ b/README.md @@ -612,7 +612,7 @@ The graphical interface is oganzised in groups and within the groups the single Now you can start to change the parameter itself. The `ID` of a new parameter should be defined in small letters without whitespaces. The description is a short free text explanation about the parameter, that appears if you run your pipeline with the `--help` flag. By clicking on the dictionary icon you can add a longer explanation for the parameter page of your pipeline. Usually, they contain a small paragraph about the parameter settings or a used datasource, like databases or references. If you want to specify some conditions for your parameter, like the file extension, you can use the nut icon to open the settings. This menu depends on the `type` you assigned to your parameter. For integers you can define a min and max value, and for strings the file extension can be specified. -The `type` field is one of the most important points in your pipeline schema, since it defines the datatype of your input and how it will be interpreted. This allows extensive testing prior to starting the pipeline. +The `type` field is one of the most important points in your pipeline schema, since it defines the datatype of your input and how it will be interpreted. This allows extensive testing prior to starting the pipeline. The basic datatypes for a pipeline schema are: @@ -621,7 +621,7 @@ The basic datatypes for a pipeline schema are: - `integer` - `boolean` -For the `string` type you have three different options in the settings (nut icon): `enumerated values`, `pattern` and `format`. The first option, `enumerated values`, allows you to specify a list of specific input values. The list has to be separated with a pipe. The `pattern` and `format` settings can depend on each other. The `format` has to be either a directory or a file path. Depending on the `format` setting selected, specifying the `pattern` setting can be the most efficient and time saving option, especially for `file paths`. The `number` and `integer` types share the same settings. Similarly to `string`, there is an `enumerated values` option with the possibility of specifying a `min` and `max` value. For the `boolean` there is no further settings and the default value is usually `false`. The `boolean` value can be switched to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. +For the `string` type you have three different options in the settings (nut icon): `enumerated values`, `pattern` and `format`. The first option, `enumerated values`, allows you to specify a list of specific input values. The list has to be separated with a pipe. The `pattern` and `format` settings can depend on each other. The `format` has to be either a directory or a file path. Depending on the `format` setting selected, specifying the `pattern` setting can be the most efficient and time saving option, especially for `file paths`. The `number` and `integer` types share the same settings. Similarly to `string`, there is an `enumerated values` option with the possibility of specifying a `min` and `max` value. For the `boolean` there is no further settings and the default value is usually `false`. The `boolean` value can be switched to `true` by adding the flag to the command. This parameter type is often used to skip specific sections of a pipeline. After filling the schema, click on the `Finished` button in the top right corner, this will automatically update your `nextflow_schema.json`. If this is not working, the schema can be copied from the graphical interface and pasted in your `nextflow_schema.json` file. diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index 5def7dd301..ae0130ea3a 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,5 +1,5 @@ --- -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/yaml-schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: "{{ component_name_underscore }}" {% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 9e78670bcc..47627cae7a 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -13,7 +13,7 @@ def meta_yml(module_lint_object, module): The lint test checks that the module has a ``meta.yml`` file and that it follows the - JSON schema defined in the ``yaml-schema.json`` + JSON schema defined in the ``modules/yaml-schema.json`` file in the nf-core/modules repository. In addition it checks that the module name diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml index 60b546a012..c1bccd5f76 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,8 +1,11 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: - custom - - version + - software + - versions + tools: - custom: description: Custom module used to dump software versions within the nf-core pipeline template diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/subworkflows/meta.yml index 3db57b6fb1..4c5b454ddf 100644 --- a/nf_core/subworkflow-template/subworkflows/meta.yml +++ b/nf_core/subworkflow-template/subworkflows/meta.yml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json name: "{{ subworkflow_name }}" ## TODO nf-core: Add a description of the subworkflow and list keywords description: Sort SAM/BAM/CRAM file From 84cafecd57ef4efcc146bb133b233dfe4544ccd8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 25 Apr 2023 13:18:37 +0200 Subject: [PATCH 215/246] fix sha --- tests/modules/patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/patch.py b/tests/modules/patch.py index b2e881969f..49c1a9255b 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -17,7 +17,7 @@ """ ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "65ba2ee3bf08c6bb31d880467df72ccdbc8d2b40" +CORRECT_SHA = "63fd3cdb1be733041db74c15542a7b5b8f4095ed" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align" From 53177bc21c798406c37e068c73a540ac3bdb759a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 25 Apr 2023 13:20:36 +0200 Subject: [PATCH 216/246] Pipeline readme: minor tweaks --- nf_core/pipeline-template/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index b678b40b39..1bb2cba4f1 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -28,7 +28,7 @@ **{{ name }}** is a bioinformatics pipeline that ... @@ -43,14 +43,14 @@ ## Usage > **Note** -> If you are new to nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how -> to set-up nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how +> to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) > with `-profile test` before running the workflow on actual data. - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO     Testing pipeline: . - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 181 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   0 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Testing pipeline: . + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ +││ +│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ +│doi (after the first release).│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭─[!] 4 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│custom/dumpsoftwareversions│modules/nf-core/custom/…│New version available│ +│multiqc│modules/nf-core/multiqc │New version available│ +│samplesheet_check│modules/local/sampleshe…│Specified label appears │ +│││to contain │ +│││non-alphanumerics: label │ +│││'process_single'│ +│samplesheet_check│modules/local/sampleshe…│Standard process label │ +│││not found│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─[✗] 1 Module Test Failed───────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│custom/dumpsoftwareversions│modules/nf-core/custom/…│Local copy of module does│ +│││not match remote│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY │ +├───────────────────────┤ +│[✔] 179 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   5 Test Warnings│ +│[✗]   1 Test Failed│ +╰───────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index e8346c16ca..419fe124a7 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ smrnaseq             │    44 │          2.1.0 │ 2 months ago │           - │ -                   │ -│ rnaseq               │   542 │            3.9 │ 3 months ago │           - │ -                   │ -│ rnafusion            │    85 │          2.1.0 │ 5 months ago │           - │ -                   │ -│ dualrnaseq           │     8 │          1.0.0 │  2 years ago │           - │ -                   │ -│ circrna              │    23 │            dev │            - │           - │ -                   │ -│ lncpipe              │    23 │            dev │            - │           - │ -                   │ -│ scflow               │    15 │            dev │            - │           - │ -                   │ -│ spatialtranscriptom… │    12 │            dev │            - │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃Pipeline Name       ┃Stars┃Latest Release┃   Released┃Last Pulled┃Have latest release?┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ smrnaseq             │    49 │          2.2.0 │   yesterday │           - │ -                    │ +│ rnafusion            │    95 │          2.3.4 │   yesterday │           - │ -                    │ +│ rnaseq               │   604 │         3.11.2 │  2 days ago │           - │ -                    │ +│ dualrnaseq           │    12 │          1.0.0 │ 2 years ago │           - │ -                    │ +│ circrna              │    27 │            dev │           - │           - │ -                    │ +│ lncpipe              │    25 │            dev │           - │           - │ -                    │ +│ scflow               │    19 │            dev │           - │           - │ -                    │ +│ spatialtranscriptom… │    19 │            dev │           - │           - │ -                    │ +└──────────────────────┴───────┴────────────────┴─────────────┴─────────────┴──────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 57c4f1dba1..4fb4401571 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-1326709712-matrix { + .terminal-3643603892-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1326709712-title { + .terminal-3643603892-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1326709712-r1 { fill: #c5c8c6 } -.terminal-1326709712-r2 { fill: #98a84b } -.terminal-1326709712-r3 { fill: #9a9b99 } -.terminal-1326709712-r4 { fill: #608ab1 } -.terminal-1326709712-r5 { fill: #d0b344 } -.terminal-1326709712-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1326709712-r7 { fill: #868887 } -.terminal-1326709712-r8 { fill: #868887;font-style: italic; } + .terminal-3643603892-r1 { fill: #c5c8c6 } +.terminal-3643603892-r2 { fill: #98a84b } +.terminal-3643603892-r3 { fill: #9a9b99 } +.terminal-3643603892-r4 { fill: #608ab1 } +.terminal-3643603892-r5 { fill: #d0b344 } +.terminal-3643603892-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3643603892-r7 { fill: #868887 } +.terminal-3643603892-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   542 │            3.9 │ 3 months ago │           - │ -                   │ -│ sarek                │   211 │          3.1.1 │  4 weeks ago │           - │ -                   │ -│ chipseq              │   133 │          2.0.0 │ 3 months ago │           - │ -                   │ -│ atacseq              │   124 │            2.0 │  3 weeks ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq              │   604 │         3.11.2 │    2 days ago │           - │ -                   │ +│ sarek               │   235 │          3.1.2 │  4 months ago │           - │ -                   │ +│ chipseq             │   144 │          2.0.0 │  7 months ago │           - │ -                   │ +│ atacseq             │   134 │            2.0 │  5 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index cf4bc6f125..55a5cfd68c 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-4187913691-matrix { + .terminal-831160234-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4187913691-title { + .terminal-831160234-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4187913691-r1 { fill: #c5c8c6 } -.terminal-4187913691-r2 { fill: #98a84b } -.terminal-4187913691-r3 { fill: #9a9b99 } -.terminal-4187913691-r4 { fill: #608ab1 } -.terminal-4187913691-r5 { fill: #d0b344 } -.terminal-4187913691-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-4187913691-r7 { fill: #868887 } -.terminal-4187913691-r8 { fill: #868887;font-style: italic; } + .terminal-831160234-r1 { fill: #c5c8c6 } +.terminal-831160234-r2 { fill: #98a84b } +.terminal-831160234-r3 { fill: #9a9b99 } +.terminal-831160234-r4 { fill: #608ab1 } +.terminal-831160234-r5 { fill: #d0b344 } +.terminal-831160234-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-831160234-r7 { fill: #868887 } +.terminal-831160234-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ methylseq            │    95 │          2.3.0 │   2 days ago │           - │ -                   │ -│ ampliseq             │   102 │          2.4.1 │  2 weeks ago │           - │ -                   │ -│ airrflow             │    23 │          2.4.0 │  2 weeks ago │           - │ -                   │ -│ mhcquant             │    21 │          2.4.0 │  2 weeks ago │           - │ -                   │ -│ atacseq              │   124 │            2.0 │  3 weeks ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ funcscan            │    29 │          1.1.0 │   9 hours ago │           - │ -                   │ +│ smrnaseq            │    49 │          2.2.0 │     yesterday │           - │ -                   │ +│ rnafusion           │    95 │          2.3.4 │     yesterday │           - │ -                   │ +│ rnaseq              │   604 │         3.11.2 │    2 days ago │           - │ -                   │ +│ demultiplex         │    25 │          1.2.0 │    2 days ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 227de6bb26..c562f52dc1 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-2041051160-matrix { + .terminal-930935328-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2041051160-title { + .terminal-930935328-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2041051160-r1 { fill: #c5c8c6 } -.terminal-2041051160-r2 { fill: #98a84b } -.terminal-2041051160-r3 { fill: #9a9b99 } -.terminal-2041051160-r4 { fill: #608ab1 } -.terminal-2041051160-r5 { fill: #d0b344 } -.terminal-2041051160-r6 { fill: #98a84b;font-weight: bold } -.terminal-2041051160-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-930935328-r1 { fill: #c5c8c6 } +.terminal-930935328-r2 { fill: #98a84b } +.terminal-930935328-r3 { fill: #9a9b99 } +.terminal-930935328-r4 { fill: #608ab1 } +.terminal-930935328-r5 { fill: #d0b344 } +.terminal-930935328-r6 { fill: #d0b344;font-weight: bold } +.terminal-930935328-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module updated│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module updated:  0.11.9 --> 0.12.1                    │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 57e29b961f..0f1a19330d 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,84 +19,84 @@ font-weight: 700; } - .terminal-474522378-matrix { + .terminal-2971378258-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-474522378-title { + .terminal-2971378258-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-474522378-r1 { fill: #c5c8c6 } -.terminal-474522378-r2 { fill: #98a84b } -.terminal-474522378-r3 { fill: #9a9b99 } -.terminal-474522378-r4 { fill: #608ab1 } -.terminal-474522378-r5 { fill: #d0b344 } -.terminal-474522378-r6 { fill: #ff2c7a } -.terminal-474522378-r7 { fill: #98729f } + .terminal-2971378258-r1 { fill: #c5c8c6 } +.terminal-2971378258-r2 { fill: #98a84b } +.terminal-2971378258-r3 { fill: #9a9b99 } +.terminal-2971378258-r4 { fill: #608ab1 } +.terminal-2971378258-r5 { fill: #d0b344 } +.terminal-2971378258-r6 { fill: #ff2c7a } +.terminal-2971378258-r7 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108,28 +108,28 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc_single_end' -INFO     Running 'fastqc' test with command:                                                         -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc_single_end -c  -./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmp17isquh4 -work-dir /tmp/tmpseaglcoa + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc_paired_end' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc_paired_end -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmpm5j883iy -work-dir /tmp/tmp8cxlkocy diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 6b3ce27a41..3a97b353d1 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -19,74 +19,74 @@ font-weight: 700; } - .terminal-3073846549-matrix { + .terminal-2173119663-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3073846549-title { + .terminal-2173119663-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3073846549-r1 { fill: #c5c8c6 } -.terminal-3073846549-r2 { fill: #98a84b } -.terminal-3073846549-r3 { fill: #9a9b99 } -.terminal-3073846549-r4 { fill: #608ab1 } -.terminal-3073846549-r5 { fill: #d0b344 } -.terminal-3073846549-r6 { fill: #68a0b3;font-weight: bold } + .terminal-2173119663-r1 { fill: #c5c8c6 } +.terminal-2173119663-r2 { fill: #98a84b } +.terminal-2173119663-r3 { fill: #9a9b99 } +.terminal-2173119663-r4 { fill: #608ab1 } +.terminal-2173119663-r5 { fill: #d0b344 } +.terminal-2173119663-r6 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -98,25 +98,25 @@ - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.11.9' + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index 320672f91a..5bd142d9aa 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,163 +19,163 @@ font-weight: 700; } - .terminal-3462197848-matrix { + .terminal-957411833-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3462197848-title { + .terminal-957411833-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3462197848-r1 { fill: #c5c8c6 } -.terminal-3462197848-r2 { fill: #98a84b } -.terminal-3462197848-r3 { fill: #9a9b99 } -.terminal-3462197848-r4 { fill: #608ab1 } -.terminal-3462197848-r5 { fill: #d0b344 } -.terminal-3462197848-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3462197848-r7 { fill: #98a84b;font-weight: bold } -.terminal-3462197848-r8 { fill: #868887 } -.terminal-3462197848-r9 { fill: #d08442 } -.terminal-3462197848-r10 { fill: #868887;font-style: italic; } -.terminal-3462197848-r11 { fill: #98729f } + .terminal-957411833-r1 { fill: #c5c8c6 } +.terminal-957411833-r2 { fill: #98a84b } +.terminal-957411833-r3 { fill: #9a9b99 } +.terminal-957411833-r4 { fill: #608ab1 } +.terminal-957411833-r5 { fill: #d0b344 } +.terminal-957411833-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-957411833-r7 { fill: #98a84b;font-weight: bold } +.terminal-957411833-r8 { fill: #868887 } +.terminal-957411833-r9 { fill: #d08442 } +.terminal-957411833-r10 { fill: #868887;font-style: italic; } +.terminal-957411833-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -187,53 +187,53 @@ - + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (files)│List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index 6fb029df0a..f2869ed343 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-3957689443-matrix { + .terminal-2146405380-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3957689443-title { + .terminal-2146405380-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3957689443-r1 { fill: #c5c8c6 } -.terminal-3957689443-r2 { fill: #98a84b } -.terminal-3957689443-r3 { fill: #9a9b99 } -.terminal-3957689443-r4 { fill: #608ab1 } -.terminal-3957689443-r5 { fill: #d0b344 } + .terminal-2146405380-r1 { fill: #c5c8c6 } +.terminal-2146405380-r2 { fill: #98a84b } +.terminal-2146405380-r3 { fill: #9a9b99 } +.terminal-2146405380-r4 { fill: #608ab1 } +.terminal-2146405380-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO     Installing 'abacas' -INFO     Use the following statement to include this module:                                         - - include { ABACAS } from '../modules/nf-core/abacas/main'                                            - + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Installing 'abacas' +INFO     Use the following statement to include this module:                                         + + include { ABACAS } from '../modules/nf-core/abacas/main'                                            + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index b39f7aa4fd..625d164311 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' - -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  23 Tests Passed │ -│[!]   0 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' + +╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│multiqc│modules/nf-core/multiqc…│Specified label appears │ +│││to contain │ +│││non-alphanumerics: label │ +│││'process_single'│ +│multiqc│modules/nf-core/multiqc…│Standard process label │ +│││not found│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─[✗] 1 Module Test Failed───────────────────────────────────────────────────────────────────────╮ +│                                           ╷                          ╷                           │ +│Module name                              │File path               │Test message             │ +│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ +│multiqc│modules/nf-core/multiqc…│The meta.yml of the │ +│││module multiqc is not │ +│││valid: 'dir' is not one │ +│││of ['map', 'file', │ +│││'directory', 'string', │ +│││'integer', 'float'].│ +│││Check the entry for │ +│││output.│ +│                                           ╵                          ╵                           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  21 Tests Passed │ +│[!]   2 Test Warnings│ +│[✗]   1 Test Failed  │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index fab2cecf7e..f96f3bbf0f 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-2617511112-matrix { + .terminal-161221737-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2617511112-title { + .terminal-161221737-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2617511112-r1 { fill: #c5c8c6 } -.terminal-2617511112-r2 { fill: #98a84b } -.terminal-2617511112-r3 { fill: #9a9b99 } -.terminal-2617511112-r4 { fill: #608ab1 } -.terminal-2617511112-r5 { fill: #d0b344 } -.terminal-2617511112-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2617511112-r7 { fill: #868887;font-style: italic; } + .terminal-161221737-r1 { fill: #c5c8c6 } +.terminal-161221737-r2 { fill: #98a84b } +.terminal-161221737-r3 { fill: #9a9b99 } +.terminal-161221737-r4 { fill: #608ab1 } +.terminal-161221737-r5 { fill: #d0b344 } +.terminal-161221737-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-161221737-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                                   - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftwar… │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ -│                     │                    │                     │ syntax for all     │            │ -│                     │                    │                     │ modules (#2654)    │            │ -│ fastqc              │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ -│                     │                    │                     │ syntax for all     │            │ -│                     │                    │                     │ modules (#2654)    │            │ -│ multiqc             │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ -│                     │                    │                     │ syntax for all     │            │ -│                     │                    │                     │ modules (#2654)    │            │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                                   + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftwar… │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ +│                     │                    │                     │ syntax for all     │            │ +│                     │                    │                     │ modules (#2654)    │            │ +│ fastqc              │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ +│                     │                    │                     │ syntax for all     │            │ +│                     │                    │                     │ modules (#2654)    │            │ +│ multiqc             │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ +│                     │                    │                     │ syntax for all     │            │ +│                     │                    │                     │ modules (#2654)    │            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index cd2d1df6e5..4faf10450f 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-3444989756-matrix { + .terminal-3737934755-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3444989756-title { + .terminal-3737934755-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3444989756-r1 { fill: #c5c8c6 } -.terminal-3444989756-r2 { fill: #98a84b } -.terminal-3444989756-r3 { fill: #9a9b99 } -.terminal-3444989756-r4 { fill: #608ab1 } -.terminal-3444989756-r5 { fill: #d0b344 } -.terminal-3444989756-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-3444989756-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-3444989756-r8 { fill: #868887;font-style: italic; } + .terminal-3737934755-r1 { fill: #c5c8c6 } +.terminal-3737934755-r2 { fill: #98a84b } +.terminal-3737934755-r3 { fill: #9a9b99 } +.terminal-3737934755-r4 { fill: #608ab1 } +.terminal-3737934755-r5 { fill: #d0b344 } +.terminal-3737934755-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-3737934755-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3737934755-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO     Modules available from https://github.com/nf-core/modules.git(master):                     - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                                          ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                                │ -│ abricate/run                                          │ -│ abricate/summary                                      │ -│ adapterremoval                                        │ -│ adapterremovalfixprefix                               │ -│ agat/convertspgff2gtf                                 │ -│ agat/convertspgxf2gxf                                 │ -│ agat/spstatistics                                     │ -│ agat/sqstatbasic                                      │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):                     + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                                          ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                                │ +│ abricate/run                                          │ +│ abricate/summary                                      │ +│ adapterremoval                                        │ +│ adapterremovalfixprefix                               │ +│ admixture                                             │ +│ affy/justrma                                          │ +│ agat/convertspgff2gtf                                 │ +│ agat/convertspgxf2gxf                                 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-mulled.svg b/docs/images/nf-core-modules-mulled.svg deleted file mode 100644 index 2ac4b28f23..0000000000 --- a/docs/images/nf-core-modules-mulled.svg +++ /dev/null @@ -1,116 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $ nf-core modules mulled pysam==0.16.0.1 biopython==1.78 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - -INFO     Found docker image on quay.io! ✨                                                           -INFO     Mulled container hash:                                                                      -mulled-v2-3a59640f3fe1ed11819984087d31d68600200c3f:185a25ca79923df85b58f42deb48f5ac4481e91f-0 - - - - diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index 729bb1d519..b0d8a66a95 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-2827097699-matrix { + .terminal-277091844-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2827097699-title { + .terminal-277091844-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2827097699-r1 { fill: #c5c8c6 } -.terminal-2827097699-r2 { fill: #98a84b } -.terminal-2827097699-r3 { fill: #9a9b99 } -.terminal-2827097699-r4 { fill: #608ab1 } -.terminal-2827097699-r5 { fill: #d0b344 } -.terminal-2827097699-r6 { fill: #cc555a;font-weight: bold } + .terminal-277091844-r1 { fill: #c5c8c6 } +.terminal-277091844-r2 { fill: #98a84b } +.terminal-277091844-r3 { fill: #9a9b99 } +.terminal-277091844-r4 { fill: #608ab1 } +.terminal-277091844-r5 { fill: #d0b344 } +.terminal-277091844-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute                           + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +ERROR    Module 'modules/nf-core/fastqc' is unchanged. No patch to compute                           diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index ce72c00e13..fb1d3591ff 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-2450992562-matrix { + .terminal-3673042259-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2450992562-title { + .terminal-3673042259-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2450992562-r1 { fill: #c5c8c6 } -.terminal-2450992562-r2 { fill: #98a84b } -.terminal-2450992562-r3 { fill: #9a9b99 } -.terminal-2450992562-r4 { fill: #608ab1 } -.terminal-2450992562-r5 { fill: #d0b344 } + .terminal-3673042259-r1 { fill: #c5c8c6 } +.terminal-3673042259-r2 { fill: #98a84b } +.terminal-3673042259-r3 { fill: #9a9b99 } +.terminal-3673042259-r4 { fill: #608ab1 } +.terminal-3673042259-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO     Removed files for 'abacas' and it's dependencies 'abacas'.                                  + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Removed files for 'abacas' and it's dependencies 'abacas'.                                  diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index c11314a7d1..83f6535612 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,67 +19,67 @@ font-weight: 700; } - .terminal-424201008-matrix { + .terminal-110742225-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-424201008-title { + .terminal-110742225-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-424201008-r1 { fill: #c5c8c6 } -.terminal-424201008-r2 { fill: #98a84b } -.terminal-424201008-r3 { fill: #9a9b99 } -.terminal-424201008-r4 { fill: #608ab1 } -.terminal-424201008-r5 { fill: #d0b344 } + .terminal-110742225-r1 { fill: #c5c8c6 } +.terminal-110742225-r2 { fill: #98a84b } +.terminal-110742225-r3 { fill: #9a9b99 } +.terminal-110742225-r4 { fill: #608ab1 } +.terminal-110742225-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -91,23 +91,23 @@ - + - - $ nf-core modules test samtools/view --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -────────────────────────────────────────── samtools/view ─────────────────────────────────────────── -INFO     Running pytest for module 'samtools/view' + + $ nf-core modules test samtools/view --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +────────────────────────────────────────── samtools/view ─────────────────────────────────────────── +INFO     Running pytest for module 'samtools/view' diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 161c6b2a56..4f24ce1baf 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-555766662-matrix { + .terminal-3169929543-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-555766662-title { + .terminal-3169929543-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-555766662-r1 { fill: #c5c8c6 } -.terminal-555766662-r2 { fill: #98a84b } -.terminal-555766662-r3 { fill: #9a9b99 } -.terminal-555766662-r4 { fill: #608ab1 } -.terminal-555766662-r5 { fill: #d0b344 } + .terminal-3169929543-r1 { fill: #c5c8c6 } +.terminal-3169929543-r2 { fill: #98a84b } +.terminal-3169929543-r3 { fill: #9a9b99 } +.terminal-3169929543-r4 { fill: #608ab1 } +.terminal-3169929543-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                              -INFO    'modules/nf-core/custom/dumpsoftwareversions' is already up to date                         -INFO    'modules/nf-core/fastqc' is already up to date                                              -INFO    'modules/nf-core/multiqc' is already up to date                                             -INFO     Updates complete ✨                                                                         + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                              +INFO     Updating 'nf-core/custom/dumpsoftwareversions' +INFO    'modules/nf-core/fastqc' is already up to date                                              +INFO     Updating 'nf-core/multiqc' +INFO     Updates complete ✨                                                                         diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index ac95ebba84..5fa9746d5a 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-1833749233-matrix { + .terminal-3395078802-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1833749233-title { + .terminal-3395078802-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1833749233-r1 { fill: #c5c8c6 } -.terminal-1833749233-r2 { fill: #98a84b } -.terminal-1833749233-r3 { fill: #9a9b99 } -.terminal-1833749233-r4 { fill: #608ab1 } -.terminal-1833749233-r5 { fill: #d0b344 } -.terminal-1833749233-r6 { fill: #98a84b;font-weight: bold } -.terminal-1833749233-r7 { fill: #868887;font-weight: bold } -.terminal-1833749233-r8 { fill: #868887 } -.terminal-1833749233-r9 { fill: #4e707b;font-weight: bold } -.terminal-1833749233-r10 { fill: #68a0b3;font-weight: bold } + .terminal-3395078802-r1 { fill: #c5c8c6 } +.terminal-3395078802-r2 { fill: #98a84b } +.terminal-3395078802-r3 { fill: #9a9b99 } +.terminal-3395078802-r4 { fill: #608ab1 } +.terminal-3395078802-r5 { fill: #d0b344 } +.terminal-3395078802-r6 { fill: #98a84b;font-weight: bold } +.terminal-3395078802-r7 { fill: #868887;font-weight: bold } +.terminal-3395078802-r8 { fill: #868887 } +.terminal-3395078802-r9 { fill: #4e707b;font-weight: bold } +.terminal-3395078802-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 30 params) -INFO     Writing schema with 31 params: './nextflow_schema.json' + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 30 params) +INFO     Writing schema with 31 params: './nextflow_schema.json' diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index 79113e70ec..a5b8a94fd3 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-1041388458-matrix { + .terminal-1690716924-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1041388458-title { + .terminal-1690716924-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1041388458-r1 { fill: #c5c8c6 } -.terminal-1041388458-r2 { fill: #98a84b } -.terminal-1041388458-r3 { fill: #9a9b99 } -.terminal-1041388458-r4 { fill: #608ab1 } -.terminal-1041388458-r5 { fill: #d0b344 } -.terminal-1041388458-r6 { fill: #98a84b;font-weight: bold } -.terminal-1041388458-r7 { fill: #868887;font-weight: bold } -.terminal-1041388458-r8 { fill: #868887 } -.terminal-1041388458-r9 { fill: #4e707b;font-weight: bold } + .terminal-1690716924-r1 { fill: #c5c8c6 } +.terminal-1690716924-r2 { fill: #98a84b } +.terminal-1690716924-r3 { fill: #9a9b99 } +.terminal-1690716924-r4 { fill: #608ab1 } +.terminal-1690716924-r5 { fill: #d0b344 } +.terminal-1690716924-r6 { fill: #98a84b;font-weight: bold } +.terminal-1690716924-r7 { fill: #868887;font-weight: bold } +.terminal-1690716924-r8 { fill: #868887 } +.terminal-1690716924-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint nextflow_schema.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 31 params) + + $ nf-core schema lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 31 params) diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 3133ec66b8..0ec7e246aa 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,71 +19,71 @@ font-weight: 700; } - .terminal-2998337404-matrix { + .terminal-3313499933-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2998337404-title { + .terminal-3313499933-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2998337404-r1 { fill: #c5c8c6 } -.terminal-2998337404-r2 { fill: #98a84b } -.terminal-2998337404-r3 { fill: #9a9b99 } -.terminal-2998337404-r4 { fill: #608ab1 } -.terminal-2998337404-r5 { fill: #d0b344 } -.terminal-2998337404-r6 { fill: #98a84b;font-weight: bold } -.terminal-2998337404-r7 { fill: #868887;font-weight: bold } -.terminal-2998337404-r8 { fill: #868887 } -.terminal-2998337404-r9 { fill: #4e707b;font-weight: bold } + .terminal-3313499933-r1 { fill: #c5c8c6 } +.terminal-3313499933-r2 { fill: #98a84b } +.terminal-3313499933-r3 { fill: #9a9b99 } +.terminal-3313499933-r4 { fill: #608ab1 } +.terminal-3313499933-r5 { fill: #d0b344 } +.terminal-3313499933-r6 { fill: #98a84b;font-weight: bold } +.terminal-3313499933-r7 { fill: #868887;font-weight: bold } +.terminal-3313499933-r8 { fill: #868887 } +.terminal-3313499933-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,23 +95,23 @@ - + - - $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 93 params) -INFO    [✓] Input parameters look valid + + $ nf-core schema validate nf-core-rnaseq/workflow nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 93 params) +INFO    [✓] Input parameters look valid diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index 5e8841a50e..872ffa60c7 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index 763a5a5d94..3b8aab4fa6 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot  --label process_low --meta  ---force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -Usage: nf-core subworkflows create [OPTIONS] subworkflow name                                       - -Try 'nf-core subworkflows create -h' for help. -╭─ Error ──────────────────────────────────────────────────────────────────────────────────────────╮ -│ No such option: --label Did you mean --help?                                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-info.svg b/docs/images/nf-core-subworkflows-info.svg index b00f918056..bace90622f 100644 --- a/docs/images/nf-core-subworkflows-info.svg +++ b/docs/images/nf-core-subworkflows-info.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core subworkflows info bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ -│/opt/hostedtoolcache/Python/3.11.1/x64/bin/nf-core:8 in <module>│ -││ -│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ -│❱ 8 │   sys.exit(run_nf_core())                                                                  │ -│9 │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/nf_core/__main__.py:105 in   │ -│run_nf_core│ -││ -│ 104 │   # Launch the click cli│ -│❱  105 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ -│ 106 │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1130 in        │ -│__call__│ -││ -│1129 │   │   """Alias for :meth:`main`."""│ -│❱ 1130 │   │   returnself.main(*args, **kwargs)                                                 │ -│1131 │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/rich_click/rich_group.py:21│ -│ in main│ -││ -│20 │   │   try:                                                                                │ -│❱ 21 │   │   │   rv = super().main(*args, standalone_mode=False, **kwargs)                       │ -│22 │   │   │   ifnot standalone_mode:                                                         │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1055 in main│ -││ -│1054 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                  │ -│❱ 1055 │   │   │   │   │   rv = self.invoke(ctx)                                                 │ -│1056 │   │   │   │   │   ifnot standalone_mode:                                               │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ -││ -│1656 │   │   │   │   with sub_ctx:                                                             │ -│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ -│1658 │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1657 in invoke│ -││ -│1656 │   │   │   │   with sub_ctx:                                                             │ -│❱ 1657 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ -│1658 │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:1404 in invoke│ -││ -│1403 │   │   ifself.callback isnotNone:                                                     │ -│❱ 1404 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ -│1405 │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/core.py:760 in invoke│ -││ -│ 759 │   │   │   with ctx:                                                                     │ -│❱  760 │   │   │   │   return __callback(*args, **kwargs)                                        │ -│ 761 │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/click/decorators.py:26 in    │ -│new_func│ -││ -│ 25 │   defnew_func(*args, **kwargs):  # type: ignore│ -│❱  26 │   │   return f(get_current_context(), *args, **kwargs)                                   │ -│ 27 │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/nf_core/__main__.py:1087 in  │ -│info│ -││ -│1086 │   │   )                                                                                 │ -│❱ 1087 │   │   stdout.print(subworkflow_info.get_component_info())                               │ -│1088 │   except (UserWarning, LookupError) as e:                                               │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/nf_core/components/info.py:1│ -│49 in get_component_info│ -││ -│148 │   │   │ -│❱ 149 │   │   returnself.generate_component_info_help()                                         │ -│150 │ -││ -│/opt/hostedtoolcache/Python/3.11.1/x64/lib/python3.11/site-packages/nf_core/components/info.py:2│ -│62 in generate_component_info_help│ -││ -│261 │   │   │   │   │   inputs_table.add_row(                                                  │ -│❱ 262 │   │   │   │   │   │   f"[orange1 on black] {key} [/][dim i] ({info['type']})",           │ -│263 │   │   │   │   │   │   Markdown(info["description"] if info["description"] else""),      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -TypeError: 'NoneType' object is not subscriptable + + + + $ nf-core subworkflows info bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +╭─ Subworkflow: bam_rseqc  ────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 📖 Description: Subworkflow to run multiple commands in the RSeqC package                        │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                       ╷                                                                   ╷         +📥 Inputs             │Description                                                        │Pattern +╺━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━╸ + meta  (map)          │Groovy Map containing sample information e.g. [ id:'test',         │ +                       │single_end:false ]                                                 │ +╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ + bam  (file)          │BAM file to calculate statistics                                   │*.{bam} +╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ + bai  (file)          │Index for input BAM file                                           │*.{bai} +╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ + bed  (file)          │BED file for the reference gene model                              │*.{bed} +╶──────────────────────┼───────────────────────────────────────────────────────────────────┼───────╴ + rseqc_modules  (list)│List of rseqc modules to run e.g. [ 'bam_stat', 'infer_experiment' │ +                       │]                                                                  │ +                       ╵                                                                   ╵         +                                     ╷                                   ╷                           +📤 Outputs                          │Description                        │                  Pattern +╺━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━╸ + bamstat_txt  (file)                │bam statistics report              │           *.bam_stat.txt +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_distance  (file)     │the inner distances                │     *.inner_distance.txt +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_freq  (file)         │frequencies of different insert    │*.inner_distance_freq.txt +                                     │sizes                              │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_mean  (file)         │mean/median values of inner        │*.inner_distance_mean.txt +                                     │distances                          │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_pdf  (file)          │distribution plot of inner         │*.inner_distance_plot.pdf +                                     │distances                          │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + innerdistance_rscript  (file)      │script to reproduce the plot       │  *.inner_distance_plot.R +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + inferexperiment_txt  (file)        │infer_experiment results report    │   *.infer_experiment.txt +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_bed  (file)     │bed file of annotated junctions    │           *.junction.bed +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_interact_bed   │Interact bed file                  │           *.Interact.bed +(file)                              │                                   │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_xls  (file)     │xls file with junction information │                    *.xls +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_pdf  (file)     │junction plot                      │           *.junction.pdf +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_events_pdf     │events plot                        │             *.events.pdf +(file)                              │                                   │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_rscript  (file) │Rscript to reproduce the plots     │                      *.r +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionannotation_log  (file)     │Log file generated by tool         │                    *.log +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionsaturation_pdf  (file)     │Junction saturation report         │                    *.pdf +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + junctionsaturation_rscript  (file) │Junction saturation R-script       │                      *.r +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readdistribution_txt  (file)       │the read distribution report       │  *.read_distribution.txt +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readduplication_seq_xls  (file)    │Read duplication rate determined   │         *seq.DupRate.xls +                                     │from mapping position of read      │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readduplication_pos_xls  (file)    │Read duplication rate determined   │         *pos.DupRate.xls +                                     │from sequence of read              │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readduplication_pdf  (file)        │plot of duplication rate           │                    *.pdf +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + readduplication_rscript  (file)    │script to reproduce the plot       │                      *.R +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + tin_txt  (file)                    │TXT file containing tin.py results │                    *.txt +                                     │summary                            │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + versions  (file)                   │File containing software versions  │             versions.yml +                                     ╵                                   ╵                           + + 💻  Installation command: nf-core subworkflows install bam_rseqc + diff --git a/docs/images/nf-core-subworkflows-install.svg b/docs/images/nf-core-subworkflows-install.svg index a440ce839e..7363919195 100644 --- a/docs/images/nf-core-subworkflows-install.svg +++ b/docs/images/nf-core-subworkflows-install.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core subworkflows install bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows install bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Installing 'bam_rseqc' diff --git a/docs/images/nf-core-subworkflows-list-local.svg b/docs/images/nf-core-subworkflows-list-local.svg index 208e28e40e..dc2836367d 100644 --- a/docs/images/nf-core-subworkflows-list-local.svg +++ b/docs/images/nf-core-subworkflows-list-local.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core subworkflows list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -[..truncated..] + + $ nf-core subworkflows list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     No nf-core subworkflows found in '.' + diff --git a/docs/images/nf-core-subworkflows-list-remote.svg b/docs/images/nf-core-subworkflows-list-remote.svg index 4e2ff9f8a5..025ee22a18 100644 --- a/docs/images/nf-core-subworkflows-list-remote.svg +++ b/docs/images/nf-core-subworkflows-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-4215096215-matrix { + .terminal-1279347052-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4215096215-title { + .terminal-1279347052-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4215096215-r1 { fill: #c5c8c6 } -.terminal-4215096215-r2 { fill: #98a84b } -.terminal-4215096215-r3 { fill: #9a9b99 } -.terminal-4215096215-r4 { fill: #608ab1 } -.terminal-4215096215-r5 { fill: #d0b344 } -.terminal-4215096215-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-4215096215-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-4215096215-r8 { fill: #868887;font-style: italic; } + .terminal-1279347052-r1 { fill: #c5c8c6 } +.terminal-1279347052-r2 { fill: #98a84b } +.terminal-1279347052-r3 { fill: #9a9b99 } +.terminal-1279347052-r4 { fill: #608ab1 } +.terminal-1279347052-r5 { fill: #d0b344 } +.terminal-1279347052-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-1279347052-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-1279347052-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core subworkflows list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Subworkflow Name                            ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ bam_dedup_stats_samtools_umitools            │ -│ bam_markduplicates_picard                    │ -│ bam_qc_picard                                │ -│ bam_rseqc                                    │ -│ bam_sort_stats_samtools                      │ -│ bam_stats_samtools                           │ -│ bcl_demultiplex                              │ -│ bed_scatter_bedtools                         │ -│ bedgraph_bedclip_bedgraphtobigwig            │ -[..truncated..] + + $ nf-core subworkflows list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Subworkflow Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ bam_create_som_pon_gatk                       │ +│ bam_dedup_stats_samtools_umitools             │ +│ bam_docounts_contamination_angsd              │ +│ bam_markduplicates_picard                     │ +│ bam_ngscheckmate                              │ +│ bam_qc_picard                                 │ +│ bam_rseqc                                     │ +│ bam_sort_stats_samtools                       │ +│ bam_split_by_region                           │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 33b93f5c19..250ea32973 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index 44d0a5fc48..3dc5cd9377 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index 4769128e73..07462dd9c0 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - - -WARNING 'repository_type' not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - » Pipeline -   nf-core/modules - - - -        ? Is this repository an nf-core pipeline or a fork of nf-core/modules? (Use arr -                                                                               o -w keys) - -Aborted. + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 99c164c146..7305dda430 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-832309805-matrix { + .terminal-1818757582-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-832309805-title { + .terminal-1818757582-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-832309805-r1 { fill: #c5c8c6 } -.terminal-832309805-r2 { fill: #98a84b } -.terminal-832309805-r3 { fill: #9a9b99 } -.terminal-832309805-r4 { fill: #608ab1 } -.terminal-832309805-r5 { fill: #d0b344 } -.terminal-832309805-r6 { fill: #98729f } -.terminal-832309805-r7 { fill: #ff2c7a } + .terminal-1818757582-r1 { fill: #c5c8c6 } +.terminal-1818757582-r2 { fill: #98a84b } +.terminal-1818757582-r3 { fill: #9a9b99 } +.terminal-1818757582-r4 { fill: #608ab1 } +.terminal-1818757582-r5 { fill: #d0b344 } +.terminal-1818757582-r6 { fill: #98729f } +.terminal-1818757582-r7 { fill: #ff2c7a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.7.2 - https://nf-co.re - - -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master' -INFO     Deleting all files in 'TEMPLATE' branch                                                     -INFO     Making a new template pipeline using pipeline variables                                     + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master' +INFO     Deleting all files in 'TEMPLATE' branch                                                     +INFO     Making a new template pipeline using pipeline variables                                     From 1879ff5aa2a279dd5b29e541960dab5eb0e1e97f Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 28 Apr 2023 09:59:28 +0200 Subject: [PATCH 240/246] update modules in template --- nf_core/pipeline-template/modules.json | 4 ++-- .../modules/nf-core/custom/dumpsoftwareversions/main.nf | 6 +++--- .../modules/nf-core/custom/dumpsoftwareversions/meta.yml | 5 ++--- .../dumpsoftwareversions/templates/dumpsoftwareversions.py | 0 nf_core/pipeline-template/modules/nf-core/multiqc/main.nf | 6 +++--- nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml | 3 ++- 6 files changed, 12 insertions(+), 12 deletions(-) mode change 100755 => 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 08116ecbac..f3d45764e5 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,7 +7,7 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", + "git_sha": "f2d63bd5b68925f98f572eed70993d205cc694b7", "installed_by": ["modules"] }, "fastqc": { @@ -17,7 +17,7 @@ }, "multiqc": { "branch": "master", - "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c", + "git_sha": "f2d63bd5b68925f98f572eed70993d205cc694b7", "installed_by": ["modules"] } } diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf index 3df21765b9..800a60991a 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.13" + conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path versions diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml index c1bccd5f76..c32657de7a 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -3,9 +3,8 @@ name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: - custom - - software - - versions - + - dump + - version tools: - custom: description: Custom module used to dump software versions within the nf-core pipeline template diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py old mode 100755 new mode 100644 diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index 68f66bea74..4b604749f5 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_single' - conda "bioconda::multiqc=1.13" + conda "bioconda::multiqc=1.14" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.14--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml index ebc29b279d..f93b5ee519 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: MultiQC description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: @@ -37,7 +38,7 @@ output: description: MultiQC report file pattern: "multiqc_report.html" - data: - type: dir + type: directory description: MultiQC data dir pattern: "multiqc_data" - plots: From 95d7ef37b73b1541d5beec5fd8112fcb73a5bcf7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 28 Apr 2023 10:24:20 +0200 Subject: [PATCH 241/246] allow labels between single quotes --- nf_core/modules/lint/main_nf.py | 2 +- tests/modules/lint.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index e5aaa9e0c4..bacc7d7fbd 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -403,7 +403,7 @@ def check_process_labels(self, lines): if len(all_labels) > 0: for label in all_labels: try: - label = re.match("^label\s+([a-zA-Z0-9_-]+)$", label).group(1) + label = re.match(r"^label\s+'?([a-zA-Z0-9_-]+)'?$", label).group(1) except AttributeError: self.warned.append( ( diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 90697fbf90..b7aaf610ca 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -122,7 +122,7 @@ def __init__(self): PROCESS_LABEL_GOOD = ( """ - label process_high + label 'process_high' cpus 12 """, 1, @@ -131,7 +131,7 @@ def __init__(self): ) PROCESS_LABEL_NON_ALPHANUMERIC = ( """ - label a:label:with:colons + label 'a:label:with:colons' cpus 12 """, 0, @@ -140,8 +140,8 @@ def __init__(self): ) PROCESS_LABEL_GOOD_CONFLICTING = ( """ - label process_high - label process_low + label 'process_high' + label 'process_low' cpus 12 """, 0, @@ -150,8 +150,8 @@ def __init__(self): ) PROCESS_LABEL_GOOD_DUPLICATES = ( """ - label process_high - label process_high + label 'process_high' + label 'process_high' cpus 12 """, 0, @@ -160,8 +160,8 @@ def __init__(self): ) PROCESS_LABEL_GOOD_AND_NONSTANDARD = ( """ - label process_high - label process_extra_label + label 'process_high' + label 'process_extra_label' cpus 12 """, 1, @@ -170,7 +170,7 @@ def __init__(self): ) PROCESS_LABEL_NONSTANDARD = ( """ - label process_extra_label + label 'process_extra_label' cpus 12 """, 0, From c40ad45518a60fd72f0386406c78ef87b5394954 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 28 Apr 2023 11:31:38 +0200 Subject: [PATCH 242/246] update dumpsoftwareversions permisions --- nf_core/pipeline-template/modules.json | 2 +- .../dumpsoftwareversions/templates/dumpsoftwareversions.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index f3d45764e5..b5c31d0939 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,7 +7,7 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "f2d63bd5b68925f98f572eed70993d205cc694b7", + "git_sha": "76cc4938c1f6ea5c7d83fed1eeffc146787f9543", "installed_by": ["modules"] }, "fastqc": { diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py old mode 100644 new mode 100755 From df27c284014f45af675b889a6b5ef94e4418a671 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 28 Apr 2023 11:00:37 +0000 Subject: [PATCH 243/246] Generate new screengrabs with rich-codex --- docs/images/nf-core-lint.svg | 267 ++++++------------ docs/images/nf-core-list-rna.svg | 114 ++++---- docs/images/nf-core-list-stars.svg | 104 +++---- docs/images/nf-core-list.svg | 108 +++---- docs/images/nf-core-modules-bump-version.svg | 110 ++++---- docs/images/nf-core-modules-create-test.svg | 98 +++---- docs/images/nf-core-modules-create.svg | 132 ++++++--- docs/images/nf-core-modules-lint.svg | 229 ++++----------- docs/images/nf-core-modules-list-local.svg | 130 ++++----- docs/images/nf-core-modules-update.svg | 86 +++--- .../nf-core-subworkflows-create-test.svg | 136 ++++++--- docs/images/nf-core-subworkflows-create.svg | 116 +++++--- docs/images/nf-core-subworkflows-remove.svg | 103 ++++--- docs/images/nf-core-subworkflows-test.svg | 86 +++--- docs/images/nf-core-subworkflows-update.svg | 72 ++--- 15 files changed, 928 insertions(+), 963 deletions(-) diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index 2aaf8d4fad..89e8c43d82 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Testing pipeline: . - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ -││ -│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ -│doi (after the first release).│ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 4 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│custom/dumpsoftwareversions│modules/nf-core/custom/…│New version available│ -│multiqc│modules/nf-core/multiqc │New version available│ -│samplesheet_check│modules/local/sampleshe…│Specified label appears │ -│││to contain │ -│││non-alphanumerics: label │ -│││'process_single'│ -│samplesheet_check│modules/local/sampleshe…│Standard process label │ -│││not found│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─[✗] 1 Module Test Failed───────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│custom/dumpsoftwareversions│modules/nf-core/custom/…│Local copy of module does│ -│││not match remote│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 179 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   5 Test Warnings│ -│[✗]   1 Test Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Testing pipeline: . + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ +││ +│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ +│doi (after the first release).│ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔] 183 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 419fe124a7..0aa93b0d91 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,96 +19,96 @@ font-weight: 700; } - .terminal-3490276454-matrix { + .terminal-38953927-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3490276454-title { + .terminal-38953927-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3490276454-r1 { fill: #c5c8c6 } -.terminal-3490276454-r2 { fill: #98a84b } -.terminal-3490276454-r3 { fill: #9a9b99 } -.terminal-3490276454-r4 { fill: #608ab1 } -.terminal-3490276454-r5 { fill: #d0b344 } -.terminal-3490276454-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3490276454-r7 { fill: #868887 } + .terminal-38953927-r1 { fill: #c5c8c6 } +.terminal-38953927-r2 { fill: #98a84b } +.terminal-38953927-r3 { fill: #9a9b99 } +.terminal-38953927-r4 { fill: #608ab1 } +.terminal-38953927-r5 { fill: #d0b344 } +.terminal-38953927-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-38953927-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -120,32 +120,32 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃Pipeline Name       ┃Stars┃Latest Release┃   Released┃Last Pulled┃Have latest release?┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ smrnaseq             │    49 │          2.2.0 │   yesterday │           - │ -                    │ -│ rnafusion            │    95 │          2.3.4 │   yesterday │           - │ -                    │ -│ rnaseq               │   604 │         3.11.2 │  2 days ago │           - │ -                    │ -│ dualrnaseq           │    12 │          1.0.0 │ 2 years ago │           - │ -                    │ -│ circrna              │    27 │            dev │           - │           - │ -                    │ -│ lncpipe              │    25 │            dev │           - │           - │ -                    │ -│ scflow               │    19 │            dev │           - │           - │ -                    │ -│ spatialtranscriptom… │    19 │            dev │           - │           - │ -                    │ -└──────────────────────┴───────┴────────────────┴─────────────┴─────────────┴──────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃Pipeline Name       ┃Stars┃Latest Release┃   Released┃Last Pulled┃Have latest release?┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ smrnaseq             │    49 │          2.2.0 │   yesterday │           - │ -                    │ +│ rnafusion            │    95 │          2.3.4 │  2 days ago │           - │ -                    │ +│ rnaseq               │   604 │         3.11.2 │  3 days ago │           - │ -                    │ +│ dualrnaseq           │    12 │          1.0.0 │ 2 years ago │           - │ -                    │ +│ circrna              │    27 │            dev │           - │           - │ -                    │ +│ lncpipe              │    25 │            dev │           - │           - │ -                    │ +│ scflow               │    19 │            dev │           - │           - │ -                    │ +│ spatialtranscriptom… │    19 │            dev │           - │           - │ -                    │ +└──────────────────────┴───────┴────────────────┴─────────────┴─────────────┴──────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 4fb4401571..bd24375ed5 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-3643603892-matrix { + .terminal-3741580213-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3643603892-title { + .terminal-3741580213-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3643603892-r1 { fill: #c5c8c6 } -.terminal-3643603892-r2 { fill: #98a84b } -.terminal-3643603892-r3 { fill: #9a9b99 } -.terminal-3643603892-r4 { fill: #608ab1 } -.terminal-3643603892-r5 { fill: #d0b344 } -.terminal-3643603892-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3643603892-r7 { fill: #868887 } -.terminal-3643603892-r8 { fill: #868887;font-style: italic; } + .terminal-3741580213-r1 { fill: #c5c8c6 } +.terminal-3741580213-r2 { fill: #98a84b } +.terminal-3741580213-r3 { fill: #9a9b99 } +.terminal-3741580213-r4 { fill: #608ab1 } +.terminal-3741580213-r5 { fill: #d0b344 } +.terminal-3741580213-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3741580213-r7 { fill: #868887 } +.terminal-3741580213-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq              │   604 │         3.11.2 │    2 days ago │           - │ -                   │ -│ sarek               │   235 │          3.1.2 │  4 months ago │           - │ -                   │ -│ chipseq             │   144 │          2.0.0 │  7 months ago │           - │ -                   │ -│ atacseq             │   134 │            2.0 │  5 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq              │   604 │         3.11.2 │    3 days ago │           - │ -                   │ +│ sarek               │   235 │          3.1.2 │  4 months ago │           - │ -                   │ +│ chipseq             │   144 │          2.0.0 │  7 months ago │           - │ -                   │ +│ atacseq             │   134 │            2.0 │  5 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 55a5cfd68c..ba60421aee 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-831160234-matrix { + .terminal-3853680407-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-831160234-title { + .terminal-3853680407-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-831160234-r1 { fill: #c5c8c6 } -.terminal-831160234-r2 { fill: #98a84b } -.terminal-831160234-r3 { fill: #9a9b99 } -.terminal-831160234-r4 { fill: #608ab1 } -.terminal-831160234-r5 { fill: #d0b344 } -.terminal-831160234-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-831160234-r7 { fill: #868887 } -.terminal-831160234-r8 { fill: #868887;font-style: italic; } + .terminal-3853680407-r1 { fill: #c5c8c6 } +.terminal-3853680407-r2 { fill: #98a84b } +.terminal-3853680407-r3 { fill: #9a9b99 } +.terminal-3853680407-r4 { fill: #608ab1 } +.terminal-3853680407-r5 { fill: #d0b344 } +.terminal-3853680407-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3853680407-r7 { fill: #868887 } +.terminal-3853680407-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ funcscan            │    29 │          1.1.0 │   9 hours ago │           - │ -                   │ -│ smrnaseq            │    49 │          2.2.0 │     yesterday │           - │ -                   │ -│ rnafusion           │    95 │          2.3.4 │     yesterday │           - │ -                   │ -│ rnaseq              │   604 │         3.11.2 │    2 days ago │           - │ -                   │ -│ demultiplex         │    25 │          1.2.0 │    2 days ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ funcscan            │    29 │          1.1.0 │  13 hours ago │           - │ -                   │ +│ smrnaseq            │    49 │          2.2.0 │     yesterday │           - │ -                   │ +│ rnafusion           │    95 │          2.3.4 │    2 days ago │           - │ -                   │ +│ rnaseq              │   604 │         3.11.2 │    3 days ago │           - │ -                   │ +│ demultiplex         │    25 │          1.2.0 │    3 days ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 5a0f2d672d..ed6519c271 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -1,4 +1,4 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module updated│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module updated:  0.11.9 --> 0.12.1                    │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 0f1a19330d..03b6c0eb27 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,84 +19,84 @@ font-weight: 700; } - .terminal-2971378258-matrix { + .terminal-1756208294-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2971378258-title { + .terminal-1756208294-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2971378258-r1 { fill: #c5c8c6 } -.terminal-2971378258-r2 { fill: #98a84b } -.terminal-2971378258-r3 { fill: #9a9b99 } -.terminal-2971378258-r4 { fill: #608ab1 } -.terminal-2971378258-r5 { fill: #d0b344 } -.terminal-2971378258-r6 { fill: #ff2c7a } -.terminal-2971378258-r7 { fill: #98729f } + .terminal-1756208294-r1 { fill: #c5c8c6 } +.terminal-1756208294-r2 { fill: #98a84b } +.terminal-1756208294-r3 { fill: #9a9b99 } +.terminal-1756208294-r4 { fill: #608ab1 } +.terminal-1756208294-r5 { fill: #d0b344 } +.terminal-1756208294-r6 { fill: #ff2c7a } +.terminal-1756208294-r7 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108,28 +108,28 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc_paired_end' -INFO     Running 'fastqc' test with command:                                                         -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc_paired_end -c  -./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmpm5j883iy -work-dir /tmp/tmp8cxlkocy + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmpgutqla5x -work-dir /tmp/tmpvumy01_b diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 3a97b353d1..192365d5ff 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' +INFO     Using Docker container: 'biocontainers/fastqc:0.12.1--hdfd78af_0' +INFO     Using Singularity container:                                                                +'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' +INFO     Created / edited following files:                                                           +           ./modules/nf-core/fastqc/main.nf +           ./modules/nf-core/fastqc/meta.yml +           ./tests/modules/nf-core/fastqc/main.nf +           ./tests/modules/nf-core/fastqc/test.yml +           ./tests/modules/nf-core/fastqc/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 625d164311..8f3b419489 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' - -╭─[!] 2 Module Test Warnings─────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│multiqc│modules/nf-core/multiqc…│Specified label appears │ -│││to contain │ -│││non-alphanumerics: label │ -│││'process_single'│ -│multiqc│modules/nf-core/multiqc…│Standard process label │ -│││not found│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─[✗] 1 Module Test Failed───────────────────────────────────────────────────────────────────────╮ -│                                           ╷                          ╷                           │ -│Module name                              │File path               │Test message             │ -│╶──────────────────────────────────────────┼──────────────────────────┼──────────────────────────╴│ -│multiqc│modules/nf-core/multiqc…│The meta.yml of the │ -│││module multiqc is not │ -│││valid: 'dir' is not one │ -│││of ['map', 'file', │ -│││'directory', 'string', │ -│││'integer', 'float'].│ -│││Check the entry for │ -│││output.│ -│                                           ╵                          ╵                           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  21 Tests Passed │ -│[!]   2 Test Warnings│ -│[✗]   1 Test Failed  │ -╰───────────────────────╯ + + + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' + +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  24 Tests Passed │ +│[!]   0 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index f96f3bbf0f..3128f1caad 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-161221737-matrix { + .terminal-4212330781-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-161221737-title { + .terminal-4212330781-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-161221737-r1 { fill: #c5c8c6 } -.terminal-161221737-r2 { fill: #98a84b } -.terminal-161221737-r3 { fill: #9a9b99 } -.terminal-161221737-r4 { fill: #608ab1 } -.terminal-161221737-r5 { fill: #d0b344 } -.terminal-161221737-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-161221737-r7 { fill: #868887;font-style: italic; } + .terminal-4212330781-r1 { fill: #c5c8c6 } +.terminal-4212330781-r2 { fill: #98a84b } +.terminal-4212330781-r3 { fill: #9a9b99 } +.terminal-4212330781-r4 { fill: #608ab1 } +.terminal-4212330781-r5 { fill: #d0b344 } +.terminal-4212330781-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4212330781-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                                   - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftwar… │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ -│                     │                    │                     │ syntax for all     │            │ -│                     │                    │                     │ modules (#2654)    │            │ -│ fastqc              │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ -│                     │                    │                     │ syntax for all     │            │ -│                     │                    │                     │ modules (#2654)    │            │ -│ multiqc             │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ -│                     │                    │                     │ syntax for all     │            │ -│                     │                    │                     │ modules (#2654)    │            │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                                   + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ custom/dumpsoftwar… │ https://github.co… │ 76cc4938c1f6ea5c7d… │ give execution     │ 2023-04-28 │ +│                     │                    │                     │ permissions to     │            │ +│                     │                    │                     │ dumpsoftwareversi… │            │ +│                     │                    │                     │ (#3347)            │            │ +│ fastqc              │ https://github.co… │ c8e35eb2055c099720… │ Bulk change conda  │ 2022-12-13 │ +│                     │                    │                     │ syntax for all     │            │ +│                     │                    │                     │ modules (#2654)    │            │ +│ multiqc             │ https://github.co… │ f2d63bd5b68925f98f… │ fix meta.ymls for  │ 2023-04-28 │ +│                     │                    │                     │ dumpsoftware and   │            │ +[..truncated..] diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 4f24ce1baf..a41cbbeec4 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-3169929543-matrix { + .terminal-3947254567-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3169929543-title { + .terminal-3947254567-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3169929543-r1 { fill: #c5c8c6 } -.terminal-3169929543-r2 { fill: #98a84b } -.terminal-3169929543-r3 { fill: #9a9b99 } -.terminal-3169929543-r4 { fill: #608ab1 } -.terminal-3169929543-r5 { fill: #d0b344 } + .terminal-3947254567-r1 { fill: #c5c8c6 } +.terminal-3947254567-r2 { fill: #98a84b } +.terminal-3947254567-r3 { fill: #9a9b99 } +.terminal-3947254567-r4 { fill: #608ab1 } +.terminal-3947254567-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                              -INFO     Updating 'nf-core/custom/dumpsoftwareversions' -INFO    'modules/nf-core/fastqc' is already up to date                                              -INFO     Updating 'nf-core/multiqc' -INFO     Updates complete ✨                                                                         + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                              +INFO    'modules/nf-core/custom/dumpsoftwareversions' is already up to date                         +INFO    'modules/nf-core/fastqc' is already up to date                                              +INFO    'modules/nf-core/multiqc' is already up to date                                             +INFO     Updates complete ✨                                                                         diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index 872ffa60c7..c887e37915 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +WARNING  Organisation path not defined in .nf-core.yml                                               +Warning: Input is not a terminal (fd=0). +? What is the organisation path under which modules and subworkflows are stored +                                                                               ? + nf-core + + + + + +         ? What is the organisation path under which modules and subworkflows are stored +                                                                               ? + nf-core + +Aborted. diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index 3b8aab4fa6..215fce11c0 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Created / edited following files:                                                           +           ./subworkflows/nf-core/bam_stats_samtools/main.nf +           ./subworkflows/nf-core/bam_stats_samtools/meta.yml +           ./tests/subworkflows/nf-core/bam_stats_samtools/main.nf +           ./tests/subworkflows/nf-core/bam_stats_samtools/test.yml +           ./tests/subworkflows/nf-core/bam_stats_samtools/nextflow.config +           ./tests/config/pytest_modules.yml diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 250ea32973..49b2b23a8f 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Removed files for 'rseqc/bamstat' and it's dependencies 'rseqc/bamstat'.                    +INFO     Removed files for 'rseqc/inferexperiment' and it's dependencies 'rseqc/inferexperiment'.    +INFO     Removed files for 'rseqc/innerdistance' and it's dependencies 'rseqc/innerdistance'.        +INFO     Removed files for 'rseqc/junctionannotation' and it's dependencies                          +'rseqc/junctionannotation'.                                                                 +INFO     Removed files for 'bam_rseqc' and it's dependencies 'bam_rseqc, rseqc_bamstat,  +rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation'.                      diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index 3dc5cd9377..bdbe06aaec 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - - - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     The 'modules.json' file is not up to date. Recreating the 'modules.json' file.              +CRITICAL Can't find a ./modules directory. Is this a DSL2 pipeline?                                  diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index 07462dd9c0..7829eacc90 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-3921689180-matrix { + .terminal-3223589889-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3921689180-title { + .terminal-3223589889-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3921689180-r1 { fill: #c5c8c6 } -.terminal-3921689180-r2 { fill: #98a84b } -.terminal-3921689180-r3 { fill: #9a9b99 } -.terminal-3921689180-r4 { fill: #608ab1 } -.terminal-3921689180-r5 { fill: #d0b344 } -.terminal-3921689180-r6 { fill: #cc555a;font-weight: bold } + .terminal-3223589889-r1 { fill: #c5c8c6 } +.terminal-3223589889-r2 { fill: #98a84b } +.terminal-3223589889-r3 { fill: #9a9b99 } +.terminal-3223589889-r4 { fill: #608ab1 } +.terminal-3223589889-r5 { fill: #d0b344 } +.terminal-3223589889-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +ERROR   'rseqc/junctionsaturation' From b4f103c5a97126dfdf3e426ca77d91567b541d9b Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 28 Apr 2023 13:24:39 +0200 Subject: [PATCH 244/246] update subworkflow commands for rich-codex --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b3b62beaf0..0de42e86e8 100644 --- a/README.md +++ b/README.md @@ -1183,11 +1183,9 @@ To help developers build new subworkflows, the `nf-core subworkflows create-test After you have written a minimal Nextflow script to test your subworkflow in `/tests/subworkflow//main.nf`, this command will run the tests for you and create the `/tests/subworkflow///test.yml` file. ![`nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force`](docs/images/nf-core-subworkflows-create-test.svg) @@ -1199,12 +1197,10 @@ To run unit tests of a subworkflow that you have installed or the test created b You can specify the subworkflow name in the form TOOL/SUBTOOL in command line or provide it later by prompts. ![`nf-core subworkflows test bam_rseqc --no-prompts`](docs/images/nf-core-subworkflows-test.svg) From e48bb3514263e891895c4bf268a21004efe1783c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 28 Apr 2023 11:47:56 +0000 Subject: [PATCH 245/246] Generate new screengrabs with rich-codex --- docs/images/nf-core-list.svg | 108 +++++++------- docs/images/nf-core-modules-bump-version.svg | 132 ++++++----------- docs/images/nf-core-modules-create-test.svg | 98 ++++++------- docs/images/nf-core-modules-create.svg | 132 ++++++----------- .../nf-core-subworkflows-create-test.svg | 135 +++++++----------- docs/images/nf-core-subworkflows-remove.svg | 100 ++++++------- docs/images/nf-core-subworkflows-test.svg | 79 +++++----- docs/images/nf-core-subworkflows-update.svg | 72 +++++----- 8 files changed, 367 insertions(+), 489 deletions(-) diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index ba60421aee..3c4a4cd4df 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-3853680407-matrix { + .terminal-3979640600-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3853680407-title { + .terminal-3979640600-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3853680407-r1 { fill: #c5c8c6 } -.terminal-3853680407-r2 { fill: #98a84b } -.terminal-3853680407-r3 { fill: #9a9b99 } -.terminal-3853680407-r4 { fill: #608ab1 } -.terminal-3853680407-r5 { fill: #d0b344 } -.terminal-3853680407-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3853680407-r7 { fill: #868887 } -.terminal-3853680407-r8 { fill: #868887;font-style: italic; } + .terminal-3979640600-r1 { fill: #c5c8c6 } +.terminal-3979640600-r2 { fill: #98a84b } +.terminal-3979640600-r3 { fill: #9a9b99 } +.terminal-3979640600-r4 { fill: #608ab1 } +.terminal-3979640600-r5 { fill: #d0b344 } +.terminal-3979640600-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3979640600-r7 { fill: #868887 } +.terminal-3979640600-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ funcscan            │    29 │          1.1.0 │  13 hours ago │           - │ -                   │ -│ smrnaseq            │    49 │          2.2.0 │     yesterday │           - │ -                   │ -│ rnafusion           │    95 │          2.3.4 │    2 days ago │           - │ -                   │ -│ rnaseq              │   604 │         3.11.2 │    3 days ago │           - │ -                   │ -│ demultiplex         │    25 │          1.2.0 │    3 days ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ funcscan            │    29 │          1.1.0 │  14 hours ago │           - │ -                   │ +│ smrnaseq            │    49 │          2.2.0 │     yesterday │           - │ -                   │ +│ rnafusion           │    95 │          2.3.4 │    2 days ago │           - │ -                   │ +│ rnaseq              │   604 │         3.11.2 │    3 days ago │           - │ -                   │ +│ demultiplex         │    25 │          1.2.0 │    3 days ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index ed6519c271..0a9094ad26 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - + - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + diff --git a/docs/images/nf-core-modules-create-test.svg b/docs/images/nf-core-modules-create-test.svg index 03b6c0eb27..e49f6dcffa 100644 --- a/docs/images/nf-core-modules-create-test.svg +++ b/docs/images/nf-core-modules-create-test.svg @@ -19,84 +19,84 @@ font-weight: 700; } - .terminal-1756208294-matrix { + .terminal-1372103280-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1756208294-title { + .terminal-1372103280-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1756208294-r1 { fill: #c5c8c6 } -.terminal-1756208294-r2 { fill: #98a84b } -.terminal-1756208294-r3 { fill: #9a9b99 } -.terminal-1756208294-r4 { fill: #608ab1 } -.terminal-1756208294-r5 { fill: #d0b344 } -.terminal-1756208294-r6 { fill: #ff2c7a } -.terminal-1756208294-r7 { fill: #98729f } + .terminal-1372103280-r1 { fill: #c5c8c6 } +.terminal-1372103280-r2 { fill: #98a84b } +.terminal-1372103280-r3 { fill: #9a9b99 } +.terminal-1372103280-r4 { fill: #608ab1 } +.terminal-1372103280-r5 { fill: #d0b344 } +.terminal-1372103280-r6 { fill: #ff2c7a } +.terminal-1372103280-r7 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -108,28 +108,28 @@ - + - - $ nf-core modules create-test-yml fastqc --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' -──────────────────────────────────────────────────────────────────────────────────────────────────── -INFO     Building test meta for entry point 'test_fastqc' -INFO     Running 'fastqc' test with command:                                                         -nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc -c  -./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  -/tmp/tmpgutqla5x -work-dir /tmp/tmpvumy01_b + + $ nf-core modules create-test-yml fastqc --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Looking for test workflow entry points: 'tests/modules/nf-core/fastqc/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_fastqc_paired_end' +INFO     Running 'fastqc' test with command:                                                         +nextflow run ./tests/modules/nf-core/fastqc -entry test_fastqc_paired_end -c  +./tests/config/nextflow.config -c ./tests/modules/nf-core/fastqc/nextflow.config --outdir  +/tmp/tmpnqfm1ogi -work-dir /tmp/tmpcv36s2sh diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 192365d5ff..3a97b353d1 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' -INFO     Using Docker container: 'biocontainers/fastqc:0.12.1--hdfd78af_0' -INFO     Using Singularity container:                                                                -'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' -INFO     Created / edited following files:                                                           -           ./modules/nf-core/fastqc/main.nf -           ./modules/nf-core/fastqc/meta.yml -           ./tests/modules/nf-core/fastqc/main.nf -           ./tests/modules/nf-core/fastqc/test.yml -           ./tests/modules/nf-core/fastqc/nextflow.config -           ./tests/config/pytest_modules.yml + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' diff --git a/docs/images/nf-core-subworkflows-create-test.svg b/docs/images/nf-core-subworkflows-create-test.svg index c887e37915..074b32dd15 100644 --- a/docs/images/nf-core-subworkflows-create-test.svg +++ b/docs/images/nf-core-subworkflows-create-test.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -WARNING  Organisation path not defined in .nf-core.yml                                               -Warning: Input is not a terminal (fd=0). -? What is the organisation path under which modules and subworkflows are stored -                                                                               ? - nf-core - - - - - -         ? What is the organisation path under which modules and subworkflows are stored -                                                                               ? - nf-core - -Aborted. + + $ nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Looking for test workflow entry points:                                                     +'tests/subworkflows/nf-core/bam_stats_samtools/main.nf' +──────────────────────────────────────────────────────────────────────────────────────────────────── +INFO     Building test meta for entry point 'test_bam_stats_samtools' +INFO     Running 'bam_stats_samtools' test with command:                                             +nextflow run ./tests/subworkflows/nf-core/bam_stats_samtools -entry test_bam_stats_samtools +-c ./tests/config/nextflow.config --outdir /tmp/tmp4iszml15 -work-dir /tmp/tmp1r4spwkd diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 49b2b23a8f..637ad9101c 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - + - + - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     Removed files for 'rseqc/bamstat' and it's dependencies 'rseqc/bamstat'.                    -INFO     Removed files for 'rseqc/inferexperiment' and it's dependencies 'rseqc/inferexperiment'.    -INFO     Removed files for 'rseqc/innerdistance' and it's dependencies 'rseqc/innerdistance'.        -INFO     Removed files for 'rseqc/junctionannotation' and it's dependencies                          -'rseqc/junctionannotation'.                                                                 -INFO     Removed files for 'bam_rseqc' and it's dependencies 'bam_rseqc, rseqc_bamstat,  -rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation'.                      + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +INFO     Removed files for 'rseqc/bamstat' and it's dependencies 'rseqc/bamstat'.                    +INFO     Removed files for 'rseqc/inferexperiment' and it's dependencies 'rseqc/inferexperiment'.    +INFO     Removed files for 'rseqc/innerdistance' and it's dependencies 'rseqc/innerdistance'.        +INFO     Removed files for 'bam_rseqc' and it's dependencies 'bam_rseqc, rseqc_bamstat,  +rseqc_inferexperiment, rseqc_innerdistance'.                                                diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index bdbe06aaec..f9ba18cc79 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -19,70 +19,67 @@ font-weight: 700; } - .terminal-2571809710-matrix { + .terminal-3101413882-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2571809710-title { + .terminal-3101413882-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2571809710-r1 { fill: #c5c8c6 } -.terminal-2571809710-r2 { fill: #98a84b } -.terminal-2571809710-r3 { fill: #9a9b99 } -.terminal-2571809710-r4 { fill: #608ab1 } -.terminal-2571809710-r5 { fill: #d0b344 } -.terminal-2571809710-r6 { fill: #292929;font-weight: bold } -.terminal-2571809710-r7 { fill: #98729f } -.terminal-2571809710-r8 { fill: #ff2c7a } + .terminal-3101413882-r1 { fill: #c5c8c6 } +.terminal-3101413882-r2 { fill: #98a84b } +.terminal-3101413882-r3 { fill: #9a9b99 } +.terminal-3101413882-r4 { fill: #608ab1 } +.terminal-3101413882-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -94,23 +91,23 @@ - - - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -INFO     The 'modules.json' file is not up to date. Recreating the 'modules.json' file.              -CRITICAL Can't find a ./modules directory. Is this a DSL2 pipeline?                                  + + + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +──────────────────────────────────────────── bam_rseqc ───────────────────────────────────────────── +INFO     Running pytest for subworkflow 'bam_rseqc' diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index 7829eacc90..a9844d80a9 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-3223589889-matrix { + .terminal-3054113778-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3223589889-title { + .terminal-3054113778-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3223589889-r1 { fill: #c5c8c6 } -.terminal-3223589889-r2 { fill: #98a84b } -.terminal-3223589889-r3 { fill: #9a9b99 } -.terminal-3223589889-r4 { fill: #608ab1 } -.terminal-3223589889-r5 { fill: #d0b344 } -.terminal-3223589889-r6 { fill: #cc555a;font-weight: bold } + .terminal-3054113778-r1 { fill: #c5c8c6 } +.terminal-3054113778-r2 { fill: #98a84b } +.terminal-3054113778-r3 { fill: #9a9b99 } +.terminal-3054113778-r4 { fill: #608ab1 } +.terminal-3054113778-r5 { fill: #d0b344 } +.terminal-3054113778-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.8 - https://nf-co.re - - - -ERROR   'rseqc/junctionsaturation' + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.8 - https://nf-co.re + + + +ERROR   'rseqc/junctionannotation' From 7c6c51d2ecfde9474784c89407bfaff4f38f9b2d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 28 Apr 2023 14:24:10 +0200 Subject: [PATCH 246/246] fix template new lines --- nf_core/pipeline-template/lib/WorkflowPipeline.groovy | 2 +- nf_core/pipeline-template/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy index 600a655439..6e577a669c 100755 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy @@ -63,8 +63,8 @@ class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { return description_html } + {%- if igenomes %} - {%- if igenomes -%} // // Exit pipeline if incorrect --genome key provided // diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 74dddd590d..52d8b1bb38 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -4,7 +4,7 @@ {{ name }} ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/{{ name }} -{% if branded %} +{%- if branded %} Website: https://nf-co.re/{{ short_name }} Slack : https://nfcore.slack.com/channels/{{ short_name }} {% endif -%}