From 8d4528996a2f83446bc27ced3fa449760b3206c1 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Jan 2024 10:28:02 +0100 Subject: [PATCH 001/178] fix conflicts between editor config and prettier --- .prettierrc.yml | 1 + nf_core/pipeline-template/.editorconfig | 2 +- nf_core/pipeline-template/.prettierrc.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.prettierrc.yml b/.prettierrc.yml index c81f9a7660..e7df631ca8 100644 --- a/.prettierrc.yml +++ b/.prettierrc.yml @@ -1 +1,2 @@ printWidth: 120 +editorconfig: true diff --git a/nf_core/pipeline-template/.editorconfig b/nf_core/pipeline-template/.editorconfig index 9b990088ab..dd991a0e95 100644 --- a/nf_core/pipeline-template/.editorconfig +++ b/nf_core/pipeline-template/.editorconfig @@ -28,5 +28,5 @@ indent_size = unset indent_style = unset # ignore python -[*.{py}] +[*.{py,md}] indent_style = unset diff --git a/nf_core/pipeline-template/.prettierrc.yml b/nf_core/pipeline-template/.prettierrc.yml index c81f9a7660..e7df631ca8 100644 --- a/nf_core/pipeline-template/.prettierrc.yml +++ b/nf_core/pipeline-template/.prettierrc.yml @@ -1 +1,2 @@ printWidth: 120 +editorconfig: true From ad20579297d78af704eba45e391933bec5afa090 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Jan 2024 10:28:40 +0100 Subject: [PATCH 002/178] remove trailing whitespace in template --- .../pipeline-template/.github/workflows/download_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 07ac82c1ed..0274092543 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -64,4 +64,4 @@ jobs: env: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results {% endraw %} + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results{% endraw %} From 0c9e01ad45bef207b0621ea3c816be5dd2a85cc4 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Jan 2024 10:29:49 +0100 Subject: [PATCH 003/178] automatically rerun actions on github instead of self-hosted --- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/pytest.yml | 4 ++-- .github/workflows/sync.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index c4d0358982..138df9fe60 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -26,7 +26,7 @@ env: jobs: MakeTestWorkflow: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} env: NXF_ANSI_LOG: false strategy: diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 7d586fc3d2..92ed13caf2 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -39,7 +39,7 @@ jobs: - "template_skip_igenomes.yml" - "template_skip_ci.yml" runner: - - ${{ github.event.inputs.runners || 'self-hosted' }} + - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} profile: ["self_hosted_runner"] include: - TEMPLATE: "template_skip_all.yml" diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 9301d1224e..2f3e8b16e1 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -26,7 +26,7 @@ env: jobs: RunTestWorkflow: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} env: NXF_ANSI_LOG: false strategy: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 8a4afe9161..0f62fc8a80 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -72,7 +72,7 @@ jobs: name: Run ${{matrix.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} needs: [setup, list_tests] if: ${{ needs.setup.outputs.run-tests }} - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} strategy: matrix: ${{ fromJson(needs.list_tests.outputs.tests) }} fail-fast: false # run all tests even if one fails @@ -162,7 +162,7 @@ jobs: coverage: needs: test - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} steps: - name: go to subdirectory run: | diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index aeb24839f1..83f7bba5be 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -23,7 +23,7 @@ concurrency: jobs: get-pipelines: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: @@ -38,7 +38,7 @@ jobs: sync: needs: get-pipelines - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} strategy: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false From 61508daf4018078b8e4340dddfe95971c7afe20f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Jan 2024 10:57:40 +0100 Subject: [PATCH 004/178] remove ignored option --- .prettierrc.yml | 1 - nf_core/pipeline-template/.prettierrc.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.prettierrc.yml b/.prettierrc.yml index e7df631ca8..c81f9a7660 100644 --- a/.prettierrc.yml +++ b/.prettierrc.yml @@ -1,2 +1 @@ printWidth: 120 -editorconfig: true diff --git a/nf_core/pipeline-template/.prettierrc.yml b/nf_core/pipeline-template/.prettierrc.yml index e7df631ca8..c81f9a7660 100644 --- a/nf_core/pipeline-template/.prettierrc.yml +++ b/nf_core/pipeline-template/.prettierrc.yml @@ -1,2 +1 @@ printWidth: 120 -editorconfig: true From 2f7b3715c651dd0056ba8fb8534457269839c2d9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Jan 2024 12:19:26 +0100 Subject: [PATCH 005/178] run test_components always on self-hosted runners --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0f62fc8a80..2cfe7f7ee1 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -72,7 +72,7 @@ jobs: name: Run ${{matrix.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} needs: [setup, list_tests] if: ${{ needs.setup.outputs.run-tests }} - runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} + runs-on: ${{ matrix.test == 'test_components.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }} strategy: matrix: ${{ fromJson(needs.list_tests.outputs.tests) }} fail-fast: false # run all tests even if one fails From 5ab5597e31999cb1f026731ea8e20e8d9492afc2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Jan 2024 12:41:29 +0100 Subject: [PATCH 006/178] add comment explaining runner logic --- .github/workflows/create-test-lint-wf-template.yml | 1 + .github/workflows/create-test-wf.yml | 1 + .github/workflows/pytest.yml | 2 ++ .github/workflows/sync.yml | 1 + 4 files changed, 5 insertions(+) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 92ed13caf2..4bd7187a76 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -39,6 +39,7 @@ jobs: - "template_skip_igenomes.yml" - "template_skip_ci.yml" runner: + # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} profile: ["self_hosted_runner"] include: diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 2f3e8b16e1..3ad8e58b01 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -26,6 +26,7 @@ env: jobs: RunTestWorkflow: + # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} env: NXF_ANSI_LOG: false diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2cfe7f7ee1..4b40513da6 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -72,6 +72,7 @@ jobs: name: Run ${{matrix.test}} with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} needs: [setup, list_tests] if: ${{ needs.setup.outputs.run-tests }} + # run on self-hosted runners for test_components.py (because of the gitlab branch), based on the input if it is dispatched manually, on github if it is a rerun or on self-hosted by default runs-on: ${{ matrix.test == 'test_components.py' && 'self-hosted' || (github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted') }} strategy: matrix: ${{ fromJson(needs.list_tests.outputs.tests) }} @@ -162,6 +163,7 @@ jobs: coverage: needs: test + # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} steps: - name: go to subdirectory diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 83f7bba5be..cc08b9242b 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -38,6 +38,7 @@ jobs: sync: needs: get-pipelines + # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} strategy: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} From e47c05b9928dc0d1390689badb1725808f33b1a8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 30 Jan 2024 12:42:19 +0100 Subject: [PATCH 007/178] run sync setup locally to start quicker --- .github/workflows/sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index cc08b9242b..8367b320c5 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -23,7 +23,7 @@ concurrency: jobs: get-pipelines: - runs-on: ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }} + runs-on: "ubuntu-latest" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: From b029dea4236d645ccee9a17061ab76bbd9ef8499 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 1 Feb 2024 11:55:37 +0100 Subject: [PATCH 008/178] bump to 2.13dev --- .gitpod.yml | 2 +- CHANGELOG.md | 10 ++++++++++ setup.py | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index a4f1c17169..39fb530da2 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 70cf2de07a..aae1fecb69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # nf-core/tools: Changelog +## v2.13dev + +### Template + +### Linting + +### Modules + +### General + ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] ### Linting diff --git a/setup.py b/setup.py index 87677b45c1..e6e7b40762 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.12.1" +version = "2.13dev" with open("README.md") as f: readme = f.read() From 596695790c4fa480f6e8af5640e9c483ab25a458 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 1 Feb 2024 13:09:17 +0100 Subject: [PATCH 009/178] fix ignoring changes in partial template files --- nf_core/lint/files_unchanged.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 399830faae..7f63973f16 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -176,7 +176,7 @@ def _tf(file_path: Union[str, Path]) -> Path: # Files that can be added to, but that must contain the template contents for files in files_partial: # Ignore if file specified in linting config - if any([f in ignore_files for f in files]): + if any([str(f) in ignore_files for f in files]): ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}") # Ignore if we can't find the file From cffda640a6c1c4d2a37217b80bbad58765a7114e Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 1 Feb 2024 12:14:15 +0000 Subject: [PATCH 010/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aae1fecb69..a8d6f8eb8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- fix ignoring changes in partial files ([#2722](https://github.com/nf-core/tools/pull/2722)) + ### Linting ### Modules From a6514457aa924121580e46df802362ca16d133f8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 1 Feb 2024 13:16:34 +0100 Subject: [PATCH 011/178] fix changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8d6f8eb8f..52f24f7da2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,14 @@ ### Template -- fix ignoring changes in partial files ([#2722](https://github.com/nf-core/tools/pull/2722)) - ### Linting ### Modules ### General +- fix ignoring changes in partially templated files (e.g. `.gitignore`) ([#2722](https://github.com/nf-core/tools/pull/2722)) + ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] ### Linting From 9e219ebd623d467ab6c125d197acebb859710f73 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 1 Feb 2024 13:23:11 +0100 Subject: [PATCH 012/178] fix one more path --- nf_core/lint/files_exist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 1bd6dba74c..ecd7bc4bbd 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -236,7 +236,7 @@ def pf(file_path: Union[str, Path]) -> Path: # Files that cause an error if they exist for file in files_fail_ifexists: - if file in ignore_files: + if str(file) in ignore_files: continue if pf(file).is_file(): failed.append(f"File must be removed: {self._wrap_quotes(file)}") From 32d3488d7028d8ef2516318d0102f6a7536e4f5f Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 2 Feb 2024 09:42:42 +0100 Subject: [PATCH 013/178] update ruff for dev tools [skip changelog] --- .pre-commit-config.yaml | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1fdda4978a..46db084c35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.15 + rev: v0.2.0 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix diff --git a/pyproject.toml b/pyproject.toml index d75ae89df6..9c551f3297 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,10 +18,10 @@ target-version = "py38" select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] cache-dir = "~/.cache/ruff" -[tool.ruff.isort] +[tool.ruff.lint.isort] known-first-party = ["nf_core"] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401"] [tool.ruff.lint.pep8-naming] From 08e44f3dc1d0452b12829a8af0cd3693a2d4bdc6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 2 Feb 2024 10:03:22 +0100 Subject: [PATCH 014/178] update ruff in template and add it to pre-commit step --- nf_core/pipeline-template/.pre-commit-config.yaml | 6 ++++++ nf_core/pipeline-template/pyproject.toml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index af57081f60..36058eeb1b 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -1,4 +1,10 @@ repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.2.0 + hooks: + - id: ruff # linter + args: [--fix, --exit-non-zero-on-fix] # sort imports and fix + - id: ruff-format # formatter - repo: https://github.com/pre-commit/mirrors-prettier rev: "v3.1.0" hooks: diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 7d08e1c8ef..84c16f27cd 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -6,8 +6,8 @@ target-version = "py38" select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] cache-dir = "~/.cache/ruff" -[tool.ruff.isort] +[tool.ruff.lint.isort] known-first-party = ["nf_core"] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401"] From c75959f5408afa9e4f72a8bf7da8a4c12e8e9711 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 2 Feb 2024 09:05:01 +0000 Subject: [PATCH 015/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aae1fecb69..5dc4d17f78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ ### General +- update ruff to 0.2.0 and add it to pre-commit step ([#2725](https://github.com/nf-core/tools/pull/2725)) + ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] ### Linting From 94476251568a787a276cf9d9d68c15ba290f38a5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 2 Feb 2024 11:04:41 +0100 Subject: [PATCH 016/178] fix docstring --- nf_core/lint/multiqc_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index b2f1a89a1b..2ad786690d 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -9,6 +9,7 @@ def multiqc_config(self) -> Dict[str, List[str]]: Basic template: .. code-block:: yaml + report_comment: > This report has been generated by the nf-core/quantms analysis pipeline. For information about how to interpret these results, please see the @@ -18,8 +19,8 @@ def multiqc_config(self) -> Dict[str, List[str]]: order: -1000 nf-core-quantms-summary: order: -1001 - export_plots: true + """ passed: List[str] = [] failed: List[str] = [] From f18983532b6d9d51e4aa7aa467c44d4ba90b0a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 2 Feb 2024 13:04:21 +0100 Subject: [PATCH 017/178] Update nf_core/pipeline-template/.pre-commit-config.yaml --- nf_core/pipeline-template/.pre-commit-config.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index 36058eeb1b..af57081f60 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -1,10 +1,4 @@ repos: - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.0 - hooks: - - id: ruff # linter - args: [--fix, --exit-non-zero-on-fix] # sort imports and fix - - id: ruff-format # formatter - repo: https://github.com/pre-commit/mirrors-prettier rev: "v3.1.0" hooks: From 59808f2071df268c18e12431d4b59af5be2253e9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 22:32:11 +0000 Subject: [PATCH 018/178] Update codecov/codecov-action digest to e0b68c6 --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 12c8a34000..28ccfe973e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -195,7 +195,7 @@ jobs: coverage report coverage xml - - uses: codecov/codecov-action@f30e4959ba63075080d4f7f90cacc18d9f3fafd7 # v4 + - uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4 with: files: coverage.xml env: From aabbb35acddbf24b569f29b15bcdf19b54d37bf7 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 3 Feb 2024 04:33:05 +0000 Subject: [PATCH 019/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aae1fecb69..e47af79a8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ ### General +- Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) + ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] ### Linting From fe25da0d280d69870ae27d69ca0a2fc1df16f36b Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Mon, 5 Feb 2024 11:13:42 +0100 Subject: [PATCH 020/178] Update README.md need an extra empty line, otherwise html and markdown collide --- nf_core/pipeline-template/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 5160ccb04d..1c4c34cd41 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -6,6 +6,7 @@ {{ name }} + {% endif -%} {% if github_badges -%} [![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) From b2ee77d262153b5ae3adb12806a6c076284eced6 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 5 Feb 2024 10:14:34 +0000 Subject: [PATCH 021/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e47af79a8b..d07756d5c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### General - Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) +- Update README.md ([#2729](https://github.com/nf-core/tools/pull/2729)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From 28a2f242bd4b6be2559449d5778cd605f8a2983d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Mon, 5 Feb 2024 10:19:20 +0000 Subject: [PATCH 022/178] fix changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d07756d5c1..ea5ec3a8ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Add empty line in README.md to fix badges. ([#2729](https://github.com/nf-core/tools/pull/2729)) + ### Linting ### Modules @@ -11,7 +13,6 @@ ### General - Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) -- Update README.md ([#2729](https://github.com/nf-core/tools/pull/2729)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From 157860cd844e832f0e47d99f765e9e87c50be5d1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 6 Feb 2024 01:43:41 +0000 Subject: [PATCH 023/178] Pin dependencies --- .github/workflows/branch.yml | 2 +- .github/workflows/changelog.yml | 6 +++--- .github/workflows/clean-up.yml | 2 +- .github/workflows/create-lint-wf.yml | 8 ++++---- .../workflows/create-test-lint-wf-template.yml | 8 ++++---- .github/workflows/create-test-wf.yml | 8 ++++---- .github/workflows/deploy-pypi.yml | 4 ++-- .github/workflows/lint-code.yml | 4 ++-- .github/workflows/push_dockerhub_dev.yml | 2 +- .github/workflows/push_dockerhub_release.yml | 2 +- .github/workflows/pytest.yml | 16 ++++++++-------- .github/workflows/rich-codex.yml | 8 ++++---- .github/workflows/sync.yml | 10 +++++----- .github/workflows/tools-api-docs-dev.yml | 6 +++--- .github/workflows/tools-api-docs-release.yml | 6 +++--- Dockerfile | 2 +- nf_core/gitpod/gitpod.Dockerfile | 2 +- .../.github/workflows/awsfulltest.yml | 4 ++-- .../.github/workflows/awstest.yml | 4 ++-- .../.github/workflows/branch.yml | 2 +- .../pipeline-template/.github/workflows/ci.yml | 4 ++-- .../.github/workflows/clean-up.yml | 2 +- .../.github/workflows/download_pipeline.yml | 6 +++--- .../.github/workflows/linting.yml | 12 ++++++------ .../.github/workflows/linting_comment.yml | 4 ++-- .../.github/workflows/release-announcements.yml | 4 ++-- 26 files changed, 69 insertions(+), 69 deletions(-) diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 54dee6df16..bbac1cc6ff 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -18,7 +18,7 @@ jobs: # If the above check failed, post a comment on the PR explaining the failure - name: Post PR comment if: failure() - uses: mshick/add-pr-comment@v2 + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 with: message: | ## This PR is against the `master` branch :x: diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 63dcf2e428..6316d62f33 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -17,7 +17,7 @@ jobs: ) steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} @@ -34,7 +34,7 @@ jobs: fi gh pr checkout $PR_NUMBER - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" @@ -63,7 +63,7 @@ jobs: echo "File changed: ${{ env.changed }}" - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 cache: "pip" diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml index ff311f9df8..a9cd4e930c 100644 --- a/.github/workflows/clean-up.yml +++ b/.github/workflows/clean-up.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v9 + - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9 with: stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index c4d0358982..19aab21b97 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -42,12 +42,12 @@ jobs: export NXF_WORK=$(pwd) # Get the repo code - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository # Set up nf-core/tools - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 cache: pip @@ -59,7 +59,7 @@ jobs: # Set up Nextflow - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 with: version: ${{ matrix.NXF_VER }} @@ -147,7 +147,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-lint-wf/log.txt diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 7d586fc3d2..4043fed79a 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -57,11 +57,11 @@ jobs: cd create-lint-wf-template export NXF_WORK=$(pwd) - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 @@ -71,7 +71,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 with: version: latest-everything @@ -154,7 +154,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: nf-core-log-file-${{ matrix.TEMPLATE }} path: create-test-lint-wf/artifact_files.tar diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 9301d1224e..60636b5f03 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -41,11 +41,11 @@ jobs: cd create-test-wf export NXF_WORK=$(pwd) - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 @@ -55,7 +55,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 with: version: ${{ matrix.NXF_VER }} @@ -68,7 +68,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-test-wf/log.txt diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 8d3a154d80..e53d2f2f5a 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,11 +13,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index d9847dd365..8ed52a0582 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -18,10 +18,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 cache: "pip" diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index 169a917d83..af11a79990 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 49ce17dd84..0b3f381d86 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 28ccfe973e..c2becc9279 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -58,7 +58,7 @@ jobs: name: Get test file matrix runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - name: List tests @@ -83,11 +83,11 @@ jobs: cd pytest export NXF_WORK=$(pwd) - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - name: Set up Python ${{ needs.setup.outputs.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: ${{ needs.setup.outputs.python-version }} cache: "pip" @@ -111,7 +111,7 @@ jobs: run: echo "date=$(date +'%Y-%m')" >> $GITHUB_ENV - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 - name: Look if nf-test is already installed and write to env variable id: check-nftest @@ -155,7 +155,7 @@ jobs: fi - name: Upload coverage - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: coverage_${{ matrix.test }} path: .coverage @@ -169,9 +169,9 @@ jobs: mkdir -p pytest cd pytest - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 env: AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/ with: @@ -188,7 +188,7 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index ecbae8ffec..73719f2630 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -6,16 +6,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.x cache: pip cache-dependency-path: setup.py - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 - name: Cache nf-test installation id: cache-software @@ -36,7 +36,7 @@ jobs: run: pip install git+https://github.com/nf-core/tools.git@dev - name: Generate terminal images with rich-codex - uses: ewels/rich-codex@v1 + uses: ewels/rich-codex@8ce988cc253c240a3027ba58e33e47640935dd8b # v1 env: COLUMNS: 100 HIDE_PROGRESS: "true" diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index aeb24839f1..7d02206ba9 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -43,10 +43,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out nf-core/tools - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} @@ -56,7 +56,7 @@ jobs: fetch-depth: "0" - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 @@ -66,7 +66,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 with: version: "latest-everything" @@ -85,7 +85,7 @@ jobs: - name: Upload sync log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: sync_log_${{ matrix.pipeline }} path: sync_log_${{ matrix.pipeline }}.txt diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index 91396b2a25..19a1621092 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -24,10 +24,10 @@ jobs: steps: - name: Check out source-code repository - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 @@ -43,7 +43,7 @@ jobs: - name: Sync dev docs # Only sync with the website if it was a push from nf-core/tools dev branch if: github.repository == 'nf-core/tools' && github.event_name == 'push' && github.event.ref == 'refs/heads/dev' - uses: SamKirkland/FTP-Deploy-Action@v4.3.4 + uses: SamKirkland/FTP-Deploy-Action@8a24039354ee91000cb948cb4a1dbdf1a1b94a3c # v4.3.4 with: server: ${{ secrets.ftp_server }} username: ${{ secrets.ftp_username}} diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index 2183db3fcf..45ce237f13 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -19,10 +19,10 @@ jobs: - ${{ github.event.release.tag_name }} steps: - name: Check out source-code repository - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 @@ -37,7 +37,7 @@ jobs: - name: Sync release docs if: github.repository == 'nf-core/tools' - uses: SamKirkland/FTP-Deploy-Action@v4.3.4 + uses: SamKirkland/FTP-Deploy-Action@8a24039354ee91000cb948cb4a1dbdf1a1b94a3c # v4.3.4 with: server: ${{ secrets.ftp_server }} username: ${{ secrets.ftp_username}} diff --git a/Dockerfile b/Dockerfile index 9c9770c25f..128b43643c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM python:3.11-slim@sha256:53d6284a40eae6b625f22870f5faba6c54f2a28db9027408f4dee111f1e885a2 LABEL authors="phil.ewels@scilifelab.se,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for the nfcore tools" diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index ad4bed5052..27ab60127f 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base +FROM gitpod/workspace-base@sha256:7a130ed80741c3f838a7607f88296d26ac477e713c12b638ab93874d3169f7f1 USER root diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 4c9fd69fcc..52aa8fa196 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v2 + uses: seqeralabs/action-tower-launch@922e5c8d5ac4e918107ec311d2ebbd65e5982b3d # v2 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters {%- raw %} @@ -31,7 +31,7 @@ jobs: } profiles: test_full - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: Tower debug log file path: | diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 25726aa1c9..5488095197 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -12,7 +12,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action {%- raw %} - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v2 + uses: seqeralabs/action-tower-launch@922e5c8d5ac4e918107ec311d2ebbd65e5982b3d # v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} @@ -25,7 +25,7 @@ jobs: } profiles: test - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: Tower debug log file path: | diff --git a/nf_core/pipeline-template/.github/workflows/branch.yml b/nf_core/pipeline-template/.github/workflows/branch.yml index 057016e4be..df1a627b15 100644 --- a/nf_core/pipeline-template/.github/workflows/branch.yml +++ b/nf_core/pipeline-template/.github/workflows/branch.yml @@ -19,7 +19,7 @@ jobs: # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - name: Post PR comment if: failure() - uses: mshick/add-pr-comment@v2 + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 with: message: | ## This PR is against the `master` branch :x: diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 3edd49f09d..bd3435120e 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -28,10 +28,10 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 with: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" diff --git a/nf_core/pipeline-template/.github/workflows/clean-up.yml b/nf_core/pipeline-template/.github/workflows/clean-up.yml index 8feb3fb017..b3b5c05d60 100644 --- a/nf_core/pipeline-template/.github/workflows/clean-up.yml +++ b/nf_core/pipeline-template/.github/workflows/clean-up.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v9 + - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9 with: stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 07ac82c1ed..31cf61f358 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -23,13 +23,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" architecture: "x64" - - uses: eWaterCycle/setup-singularity@v7 + - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 with: singularity-version: 3.8.3 diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index a267f1ec14..232789b8d4 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,10 +14,10 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 cache: "pip" @@ -32,12 +32,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" architecture: "x64" @@ -60,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: linting-logs path: | diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index e5528b29cf..c49fb53955 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@v3 + uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d # v3 with: workflow: linting.yml workflow_conclusion: completed @@ -21,7 +21,7 @@ jobs: run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment - uses: marocchino/sticky-pull-request-comment@v2 + uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 # v2 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} number: ${{ steps.pr_number.outputs.pr_number }} diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index 1dd48b123f..f293032771 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.10" - name: Install dependencies @@ -56,7 +56,7 @@ jobs: bsky-post: runs-on: ubuntu-latest steps: - - uses: zentered/bluesky-post-action@v0.1.0 + - uses: zentered/bluesky-post-action@80dbe0a7697de18c15ad22f4619919ceb5ccf597 # v0.1.0 with: post: | Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! From bf7d0b5e4418914385c701762e954062f6f204b5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 6 Feb 2024 01:43:44 +0000 Subject: [PATCH 024/178] Update pre-commit hook astral-sh/ruff-pre-commit to v0.2.1 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46db084c35..9bb8c3df95 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.0 + rev: v0.2.1 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 7e3ecc56765b3a1641943871cda9cb118405f211 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 6 Feb 2024 01:44:32 +0000 Subject: [PATCH 025/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea5ec3a8ed..d766871387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ### General - Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.2.1 ([#2730](https://github.com/nf-core/tools/pull/2730)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From b2f7c481f6e6d14c1dcef7fad163a1b44b09c600 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 6 Feb 2024 14:25:45 +0100 Subject: [PATCH 026/178] make creat-lint-wf composable --- .github/actions/create-lint-wf.yml | 141 +++++++++++++++++++++++++++ .github/workflows/create-lint-wf.yml | 102 +------------------ 2 files changed, 146 insertions(+), 97 deletions(-) create mode 100644 .github/actions/create-lint-wf.yml diff --git a/.github/actions/create-lint-wf.yml b/.github/actions/create-lint-wf.yml new file mode 100644 index 0000000000..23856c8a42 --- /dev/null +++ b/.github/actions/create-lint-wf.yml @@ -0,0 +1,141 @@ +name: Create a pipeline and run nf-core linting +on: + push: + branches: + - dev + pull_request: + release: + types: [published] + workflow_dispatch: + inputs: + runners: + description: "Runners to test on" + type: choice + options: + - "ubuntu-latest" + - "self-hosted" + default: "self-hosted" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + NXF_ANSI_LOG: false + +jobs: + MakeTestWorkflow: + runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} + env: + NXF_ANSI_LOG: false + strategy: + matrix: + NXF_VER: + - "23.04.0" + - "latest-everything" + steps: + - name: go to subdirectory and change nextflow workdir + run: | + mkdir -p create-lint-wf + cd create-lint-wf + export NXF_WORK=$(pwd) + + # Set up Nextflow + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + with: + version: ${{ matrix.NXF_VER }} + + # Build a pipeline from the template + - name: nf-core create + run: | + mkdir create-lint-wf && cd create-lint-wf + export NXF_WORK=$(pwd) + nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain + + # Try syncing it before we change anything + - name: nf-core sync + run: nf-core --log-file log.txt sync --dir nf-core-testpipeline/ + working-directory: create-lint-wf + + # Build a module from the template + - name: nf-core modules create + run: nf-core --log-file log.txt modules create bpipe --dir nf-core-testpipeline --author @nf-core-bot --label process_low --meta + working-directory: create-lint-wf + + # Run code style linting + - name: run pre-commit + run: pre-commit run --all-files + working-directory: create-lint-wf + + # Update modules to the latest version + - name: nf-core modules update + run: nf-core --log-file log.txt modules update --dir nf-core-testpipeline --all --no-preview + working-directory: create-lint-wf + + # Remove TODO statements + - name: remove TODO + run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; + working-directory: create-lint-wf + + # Replace zenodo.XXXXXX to pass readme linting + - name: replace zenodo.XXXXXX + run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; + working-directory: create-lint-wf + + # Run nf-core linting + - name: nf-core lint + run: nf-core --verbose --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned + working-directory: create-lint-wf + + # Run the other nf-core commands + - name: nf-core list + run: nf-core --log-file log.txt list + working-directory: create-lint-wf + + # - name: nf-core licences + # run: nf-core --log-file log.txt licences nf-core-testpipeline + + - name: nf-core schema + run: nf-core --log-file log.txt schema build --dir nf-core-testpipeline/ --no-prompts + working-directory: create-lint-wf + + - name: nf-core bump-version + run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1 + working-directory: create-lint-wf + + - name: nf-core lint in release mode + run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release + working-directory: create-lint-wf + + - name: nf-core modules install + run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force + working-directory: create-lint-wf + + - name: nf-core modules install gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastp --dir nf-core-testpipeline/ + working-directory: create-lint-wf + + - name: nf-core modules list local + run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ + working-directory: create-lint-wf + + - name: nf-core modules list remote + run: nf-core --log-file log.txt modules list remote + working-directory: create-lint-wf + + - name: nf-core modules list remote gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote + working-directory: create-lint-wf + + - name: Upload log file artifact + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: nf-core-log-file-${{ matrix.NXF_VER }} + path: create-lint-wf/log.txt + + - name: Cleanup work directory + run: sudo rm -rf create-lint-wf + if: always() diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index c4d0358982..b41104ac67 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -57,101 +57,9 @@ jobs: python -m pip install --upgrade pip pip install . - # Set up Nextflow - - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + - name: run create-lint-wf + uses: ./.github/actions/create-lint-wf with: - version: ${{ matrix.NXF_VER }} - - # Build a pipeline from the template - - name: nf-core create - run: | - mkdir create-lint-wf && cd create-lint-wf - export NXF_WORK=$(pwd) - nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - - # Try syncing it before we change anything - - name: nf-core sync - run: nf-core --log-file log.txt sync --dir nf-core-testpipeline/ - working-directory: create-lint-wf - - # Build a module from the template - - name: nf-core modules create - run: nf-core --log-file log.txt modules create bpipe --dir nf-core-testpipeline --author @nf-core-bot --label process_low --meta - working-directory: create-lint-wf - - # Run code style linting - - name: run pre-commit - run: pre-commit run --all-files - working-directory: create-lint-wf - - # Update modules to the latest version - - name: nf-core modules update - run: nf-core --log-file log.txt modules update --dir nf-core-testpipeline --all --no-preview - working-directory: create-lint-wf - - # Remove TODO statements - - name: remove TODO - run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; - working-directory: create-lint-wf - - # Replace zenodo.XXXXXX to pass readme linting - - name: replace zenodo.XXXXXX - run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; - working-directory: create-lint-wf - - # Run nf-core linting - - name: nf-core lint - run: nf-core --verbose --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned - working-directory: create-lint-wf - - # Run the other nf-core commands - - name: nf-core list - run: nf-core --log-file log.txt list - working-directory: create-lint-wf - - # - name: nf-core licences - # run: nf-core --log-file log.txt licences nf-core-testpipeline - - - name: nf-core schema - run: nf-core --log-file log.txt schema build --dir nf-core-testpipeline/ --no-prompts - working-directory: create-lint-wf - - - name: nf-core bump-version - run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1 - working-directory: create-lint-wf - - - name: nf-core lint in release mode - run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release - working-directory: create-lint-wf - - - name: nf-core modules install - run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - working-directory: create-lint-wf - - - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastp --dir nf-core-testpipeline/ - working-directory: create-lint-wf - - - name: nf-core modules list local - run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ - working-directory: create-lint-wf - - - name: nf-core modules list remote - run: nf-core --log-file log.txt modules list remote - working-directory: create-lint-wf - - - name: nf-core modules list remote gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote - working-directory: create-lint-wf - - - name: Upload log file artifact - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: nf-core-log-file-${{ matrix.NXF_VER }} - path: create-lint-wf/log.txt - - - name: Cleanup work directory - run: sudo rm -rf create-lint-wf - if: always() + NXF_VER: ${{ matrix.NXF_VER }} + NXF_WORK: ${{ env.NXF_WORK }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 0fe8896c1caa7ca4404dfc08e0a87f873a07cfbb Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 6 Feb 2024 13:27:42 +0000 Subject: [PATCH 027/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea5ec3a8ed..1f4cfb92ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Linting +- make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733)) + ### Modules ### General From eec8293be86fd8f574e1aa2f2627fa8c4a0bf709 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 6 Feb 2024 14:42:33 +0100 Subject: [PATCH 028/178] move to correct folder --- .../action.yml} | 49 ++----------------- .github/workflows/create-lint-wf.yml | 3 ++ .../create-test-lint-wf-template.yml | 3 ++ .github/workflows/create-test-wf.yml | 3 ++ 4 files changed, 12 insertions(+), 46 deletions(-) rename .github/actions/{create-lint-wf.yml => create-lint-wf/action.yml} (59%) diff --git a/.github/actions/create-lint-wf.yml b/.github/actions/create-lint-wf/action.yml similarity index 59% rename from .github/actions/create-lint-wf.yml rename to .github/actions/create-lint-wf/action.yml index 23856c8a42..d408383e6d 100644 --- a/.github/actions/create-lint-wf.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -3,6 +3,9 @@ on: push: branches: - dev + paths-ignore: + - "docs/**" + - "CHANGELOG.md" pull_request: release: types: [published] @@ -54,16 +57,6 @@ jobs: export NXF_WORK=$(pwd) nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - # Try syncing it before we change anything - - name: nf-core sync - run: nf-core --log-file log.txt sync --dir nf-core-testpipeline/ - working-directory: create-lint-wf - - # Build a module from the template - - name: nf-core modules create - run: nf-core --log-file log.txt modules create bpipe --dir nf-core-testpipeline --author @nf-core-bot --label process_low --meta - working-directory: create-lint-wf - # Run code style linting - name: run pre-commit run: pre-commit run --all-files @@ -89,46 +82,10 @@ jobs: run: nf-core --verbose --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned working-directory: create-lint-wf - # Run the other nf-core commands - - name: nf-core list - run: nf-core --log-file log.txt list - working-directory: create-lint-wf - - # - name: nf-core licences - # run: nf-core --log-file log.txt licences nf-core-testpipeline - - - name: nf-core schema - run: nf-core --log-file log.txt schema build --dir nf-core-testpipeline/ --no-prompts - working-directory: create-lint-wf - - - name: nf-core bump-version - run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1 - working-directory: create-lint-wf - - name: nf-core lint in release mode run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release working-directory: create-lint-wf - - name: nf-core modules install - run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force - working-directory: create-lint-wf - - - name: nf-core modules install gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastp --dir nf-core-testpipeline/ - working-directory: create-lint-wf - - - name: nf-core modules list local - run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ - working-directory: create-lint-wf - - - name: nf-core modules list remote - run: nf-core --log-file log.txt modules list remote - working-directory: create-lint-wf - - - name: nf-core modules list remote gitlab - run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote - working-directory: create-lint-wf - - name: Upload log file artifact if: ${{ always() }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index b41104ac67..31e6ff5dc9 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -3,6 +3,9 @@ on: push: branches: - dev + paths-ignore: + - "docs/**" + - "CHANGELOG.md" pull_request: release: types: [published] diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 7d586fc3d2..33c37eee57 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -3,6 +3,9 @@ on: push: branches: - dev + paths-ignore: + - "docs/**" + - "CHANGELOG.md" paths: - nf_core/pipeline-template/** pull_request: diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 9301d1224e..ba734eeea7 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -3,6 +3,9 @@ on: push: branches: - dev + paths-ignore: + - "docs/**" + - "CHANGELOG.md" pull_request: release: types: [published] From bf8257852c98f86aa0e9d258aa9661023244e498 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 6 Feb 2024 15:12:30 +0100 Subject: [PATCH 029/178] fix composite action syntax --- .github/actions/create-lint-wf/action.yml | 145 +++++++++------------- .github/workflows/create-lint-wf.yml | 1 - 2 files changed, 58 insertions(+), 88 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index d408383e6d..25447de41a 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -1,98 +1,69 @@ -name: Create a pipeline and run nf-core linting -on: - push: - branches: - - dev - paths-ignore: - - "docs/**" - - "CHANGELOG.md" - pull_request: - release: - types: [published] - workflow_dispatch: - inputs: - runners: - description: "Runners to test on" - type: choice - options: - - "ubuntu-latest" - - "self-hosted" - default: "self-hosted" +name: "Create and lint nf-core pipeline" +description: "Create and lint nf-core pipeline" +inputs: + NXF_VER: + description: "Nextflow version" + required: true -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true +runs: + using: "composite" + steps: + - name: go to subdirectory and change nextflow workdir + run: | + mkdir -p create-lint-wf + cd create-lint-wf + export NXF_WORK=$(pwd) -env: - NXF_ANSI_LOG: false + # Set up Nextflow + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + with: + version: ${{ matrix.NXF_VER }} -jobs: - MakeTestWorkflow: - runs-on: ${{ github.event.inputs.runners || 'self-hosted' }} - env: - NXF_ANSI_LOG: false - strategy: - matrix: - NXF_VER: - - "23.04.0" - - "latest-everything" - steps: - - name: go to subdirectory and change nextflow workdir - run: | - mkdir -p create-lint-wf - cd create-lint-wf - export NXF_WORK=$(pwd) + # Build a pipeline from the template + - name: nf-core create + run: | + mkdir create-lint-wf && cd create-lint-wf + export NXF_WORK=$(pwd) + nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - # Set up Nextflow - - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 - with: - version: ${{ matrix.NXF_VER }} + # Run code style linting + - name: run pre-commit + shell: bash + run: pre-commit run --all-files + working-directory: create-lint-wf - # Build a pipeline from the template - - name: nf-core create - run: | - mkdir create-lint-wf && cd create-lint-wf - export NXF_WORK=$(pwd) - nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain + # Update modules to the latest version + - name: nf-core modules update + run: nf-core --log-file log.txt modules update --dir nf-core-testpipeline --all --no-preview + working-directory: create-lint-wf - # Run code style linting - - name: run pre-commit - run: pre-commit run --all-files - working-directory: create-lint-wf + # Remove TODO statements + - name: remove TODO + run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; + working-directory: create-lint-wf - # Update modules to the latest version - - name: nf-core modules update - run: nf-core --log-file log.txt modules update --dir nf-core-testpipeline --all --no-preview - working-directory: create-lint-wf + # Replace zenodo.XXXXXX to pass readme linting + - name: replace zenodo.XXXXXX + run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; + working-directory: create-lint-wf - # Remove TODO statements - - name: remove TODO - run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; - working-directory: create-lint-wf + # Run nf-core linting + - name: nf-core lint + run: nf-core --verbose --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned + working-directory: create-lint-wf - # Replace zenodo.XXXXXX to pass readme linting - - name: replace zenodo.XXXXXX - run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; - working-directory: create-lint-wf + - name: nf-core lint in release mode + run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release + working-directory: create-lint-wf - # Run nf-core linting - - name: nf-core lint - run: nf-core --verbose --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned - working-directory: create-lint-wf + - name: Upload log file artifact + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: nf-core-log-file-${{ matrix.NXF_VER }} + path: create-lint-wf/log.txt - - name: nf-core lint in release mode - run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release - working-directory: create-lint-wf - - - name: Upload log file artifact - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: nf-core-log-file-${{ matrix.NXF_VER }} - path: create-lint-wf/log.txt - - - name: Cleanup work directory - run: sudo rm -rf create-lint-wf - if: always() + - name: Cleanup work directory + run: sudo rm -rf create-lint-wf + if: always() diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 31e6ff5dc9..4ab8dc8828 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -64,5 +64,4 @@ jobs: uses: ./.github/actions/create-lint-wf with: NXF_VER: ${{ matrix.NXF_VER }} - NXF_WORK: ${{ env.NXF_WORK }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 23316a37e863e34d41057ba13cc1d4b568be0d8d Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 6 Feb 2024 15:17:10 +0100 Subject: [PATCH 030/178] fix action syntax --- .github/actions/create-lint-wf/action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 25447de41a..1147bf2620 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -9,6 +9,7 @@ runs: using: "composite" steps: - name: go to subdirectory and change nextflow workdir + shell: bash run: | mkdir -p create-lint-wf cd create-lint-wf @@ -22,6 +23,7 @@ runs: # Build a pipeline from the template - name: nf-core create + shell: bash run: | mkdir create-lint-wf && cd create-lint-wf export NXF_WORK=$(pwd) @@ -35,25 +37,30 @@ runs: # Update modules to the latest version - name: nf-core modules update + shell: bash run: nf-core --log-file log.txt modules update --dir nf-core-testpipeline --all --no-preview working-directory: create-lint-wf # Remove TODO statements - name: remove TODO + shell: bash run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; working-directory: create-lint-wf # Replace zenodo.XXXXXX to pass readme linting - name: replace zenodo.XXXXXX + shell: bash run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; working-directory: create-lint-wf # Run nf-core linting - name: nf-core lint + shell: bash run: nf-core --verbose --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned working-directory: create-lint-wf - name: nf-core lint in release mode + shell: bash run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release working-directory: create-lint-wf @@ -65,5 +72,6 @@ runs: path: create-lint-wf/log.txt - name: Cleanup work directory + shell: bash run: sudo rm -rf create-lint-wf if: always() From c965539eb5dbc731a21f3b9082108c9705796c76 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 6 Feb 2024 15:21:11 +0100 Subject: [PATCH 031/178] clean up actions --- .github/actions/create-lint-wf/action.yml | 2 +- .github/workflows/create-lint-wf.yml | 1 - .github/workflows/pytest.yml | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 1147bf2620..80af3016bf 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -25,7 +25,7 @@ runs: - name: nf-core create shell: bash run: | - mkdir create-lint-wf && cd create-lint-wf + mkdir -p create-lint-wf && cd create-lint-wf export NXF_WORK=$(pwd) nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 4ab8dc8828..e429e4a094 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -64,4 +64,3 @@ jobs: uses: ./.github/actions/create-lint-wf with: NXF_VER: ${{ matrix.NXF_VER }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 28ccfe973e..4681eabfb2 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -12,6 +12,7 @@ on: paths-ignore: - "docs/**" - "CHANGELOG.md" + - ".github/**" release: types: [published] workflow_dispatch: From 1b6e2147f215e4e0bc142531c2cd9a1eec35c4b0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 6 Feb 2024 15:29:57 +0100 Subject: [PATCH 032/178] add step with bump to release version --- .github/actions/create-lint-wf/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 80af3016bf..3b87c767a1 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -59,6 +59,11 @@ runs: run: nf-core --verbose --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned working-directory: create-lint-wf + - name: nf-core bump-version to release + shell: bash + run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1 + working-directory: create-lint-wf + - name: nf-core lint in release mode shell: bash run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release From f3f7abcca13fbf68a60cb364d074c5e576904726 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 2 Feb 2024 12:41:12 +0100 Subject: [PATCH 033/178] Download CI Test: Set dev as the default branch and add customisation option for manual dispatch. --- .../.github/workflows/download_pipeline.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 31cf61f358..a6619b7bd9 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -6,6 +6,11 @@ name: Test successful pipeline download with 'nf-core download' # - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. on: workflow_dispatch: + inputs: + testbranch: + description: "The branch to test the pipeline download." + required: true + default: "dev" pull_request: types: - opened @@ -42,7 +47,7 @@ jobs: run: | echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} - echo "REPO_BRANCH=${GITHUB_REF#refs/heads/}" >> ${GITHUB_ENV} + echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} - name: Download the pipeline env: From de73d39fbd7391cda5af4de2de0bce6cea3a605a Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 2 Feb 2024 16:18:37 +0100 Subject: [PATCH 034/178] Update Changelog. --- CHANGELOG.md | 3 ++- .../pipeline-template/.github/workflows/download_pipeline.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2de939183c..b23787d61d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Add empty line in README.md to fix badges. ([#2729](https://github.com/nf-core/tools/pull/2729)) +- Replace automatic branch detection in `nf-core download` CI test with hardcoded `dev` and input. ([#2727](https://github.com/nf-core/tools/pull/2727)) ### Linting @@ -35,7 +36,7 @@ ### Template -- Add a Github Action Workflow to the pipeline template that tests a successful download with 'nf-core download' ([#2618](https://github.com/nf-core/tools/pull/2618)) +- Add a Github Action Workflow to the pipeline template that tests a successful download with `nf-core download` ([#2618](https://github.com/nf-core/tools/pull/2618)) - Use `pre-commit` to lint files in GitHub CI ([#2635](https://github.com/nf-core/tools/pull/2635)) - Use pdiff also on gitpod for nf-test ([#2640](https://github.com/nf-core/tools/pull/2640)) - switch to new image syntax in readme ([#2645](https://github.com/nf-core/tools/pull/2645)) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index a6619b7bd9..a828882f7f 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: inputs: testbranch: - description: "The branch to test the pipeline download." + description: "The specific branch you wish to utilize for the test execution of nf-core download." required: true default: "dev" pull_request: From 90dd79372ca6fc82f09692e5e1b5baa86d8ae2fd Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 6 Feb 2024 17:56:25 +0100 Subject: [PATCH 035/178] fix trailing whitespace --- .../pipeline-template/.github/workflows/download_pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index a828882f7f..08fc30c936 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -69,4 +69,4 @@ jobs: env: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results {% endraw %} + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results{% endraw %} From c2f95efeced24b004b3ac130d8982d45abb842e4 Mon Sep 17 00:00:00 2001 From: gregorysprenger <42686628+gregorysprenger@users.noreply.github.com> Date: Tue, 6 Feb 2024 13:33:34 -0500 Subject: [PATCH 036/178] Place checkouts in try block and force if needed --- nf_core/synced_repo.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index ac0f467e66..44ac540503 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -208,7 +208,10 @@ def checkout_branch(self): """ Checks out the specified branch of the repository """ - self.repo.git.checkout(self.branch) + try: + self.repo.git.checkout(self.branch) + except GitCommandError: + self.repo.git.checkout(self.branch, force=True) def checkout(self, commit): """ @@ -217,7 +220,10 @@ def checkout(self, commit): Args: commit (str): Git SHA of the commit """ - self.repo.git.checkout(commit) + try: + self.repo.git.checkout(commit) + except GitCommandError: + self.repo.git.checkout(commit, force=True) def component_exists(self, component_name, component_type, checkout=True, commit=None): """ From ebdfff0994ed6230a3ef4f972792dfc15323e3d2 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 6 Feb 2024 18:42:49 +0000 Subject: [PATCH 037/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2de939183c..36d6149cca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - update ruff to 0.2.0 and add it to pre-commit step ([#2725](https://github.com/nf-core/tools/pull/2725)) - Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) +- Force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From 47713a292e1810c074c513a02a85c2c7ea30e5ed Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 7 Feb 2024 09:32:54 +0100 Subject: [PATCH 038/178] only use force on specific git errors --- nf_core/synced_repo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 44ac540503..eaef930418 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -210,8 +210,10 @@ def checkout_branch(self): """ try: self.repo.git.checkout(self.branch) - except GitCommandError: - self.repo.git.checkout(self.branch, force=True) + except GitCommandError as e: + if "Your local changes to the following files would be overwritten by checkout" in str(e): + log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") + self.repo.git.checkout(self.branch, force=True) def checkout(self, commit): """ From 5163e4a63e1104786ddb4c8723b8f9e763c5a32f Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 7 Feb 2024 09:34:56 +0100 Subject: [PATCH 039/178] add logic to second occurrence of commands --- nf_core/synced_repo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index eaef930418..36dd22bc80 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -224,8 +224,10 @@ def checkout(self, commit): """ try: self.repo.git.checkout(commit) - except GitCommandError: - self.repo.git.checkout(commit, force=True) + except GitCommandError as e: + if "Your local changes to the following files would be overwritten by checkout" in str(e): + log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") + self.repo.git.checkout(self.branch, force=True) def component_exists(self, component_name, component_type, checkout=True, commit=None): """ From 372a9f2faad0eaf8ab0c91f22b2dbc3453df0c33 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 7 Feb 2024 10:37:22 +0100 Subject: [PATCH 040/178] fix tests --- nf_core/synced_repo.py | 4 ++++ tests/modules/modules_json.py | 25 ++++++++++++------------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 36dd22bc80..c668be28d2 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -214,6 +214,8 @@ def checkout_branch(self): if "Your local changes to the following files would be overwritten by checkout" in str(e): log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") self.repo.git.checkout(self.branch, force=True) + else: + raise e def checkout(self, commit): """ @@ -228,6 +230,8 @@ def checkout(self, commit): if "Your local changes to the following files would be overwritten by checkout" in str(e): log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") self.repo.git.checkout(self.branch, force=True) + else: + raise e def component_exists(self, component_name, component_type, checkout=True, commit=None): """ diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index a054b6b131..e0100adfb7 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -1,6 +1,5 @@ import copy import json -import os import shutil from pathlib import Path @@ -16,7 +15,7 @@ def test_get_modules_json(self): """Checks that the get_modules_json function returns the correct result""" - mod_json_path = os.path.join(self.pipeline_dir, "modules.json") + mod_json_path = Path(self.pipeline_dir, "modules.json") with open(mod_json_path) as fh: try: mod_json_sb = json.load(fh) @@ -49,16 +48,16 @@ def test_mod_json_update(self): def test_mod_json_create(self): """Test creating a modules.json file from scratch""" - mod_json_path = os.path.join(self.pipeline_dir, "modules.json") + mod_json_path = Path(self.pipeline_dir, "modules.json") # Remove the existing modules.json file - os.remove(mod_json_path) + mod_json_path.unlink() # Create the new modules.json file # (There are no prompts as long as there are only nf-core modules) ModulesJson(self.pipeline_dir).create() # Check that the file exists - assert os.path.exists(mod_json_path) + assert (mod_json_path).exists() # Get the contents of the file mod_json_obj = ModulesJson(self.pipeline_dir) @@ -94,7 +93,7 @@ def test_mod_json_create_with_patch(self): patch_obj.patch("fastqc") # Remove the existing modules.json file - os.remove(mod_json_path) + mod_json_path.unlink() # Create the new modules.json file ModulesJson(self.pipeline_dir).create() @@ -137,7 +136,7 @@ def test_mod_json_up_to_date_module_removed(self): but is missing in the pipeline """ # Remove the fastqc module - fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") + fastqc_path = Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") shutil.rmtree(fastqc_path) # Check that the modules.json file is up to date, and reinstall the module @@ -146,9 +145,9 @@ def test_mod_json_up_to_date_module_removed(self): # Check that the module has been reinstalled files = ["main.nf", "meta.yml"] - assert os.path.exists(fastqc_path) + assert fastqc_path.exists() for f in files: - assert os.path.exists(os.path.join(fastqc_path, f)) + assert Path(fastqc_path, f).exists() def test_mod_json_up_to_date_reinstall_fails(self): @@ -161,7 +160,7 @@ def test_mod_json_up_to_date_reinstall_fails(self): mod_json_obj.update("modules", ModulesRepo(), "fastqc", "INVALID_GIT_SHA", "modules", write_file=True) # Remove the fastqc module - fastqc_path = os.path.join(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") + fastqc_path = Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "fastqc") shutil.rmtree(fastqc_path) # Check that the modules.json file is up to date, and remove the fastqc module entry @@ -206,12 +205,12 @@ def test_mod_json_dump(self): mod_json_obj = ModulesJson(self.pipeline_dir) mod_json = mod_json_obj.get_modules_json() # Remove the modules.json file - mod_json_path = os.path.join(self.pipeline_dir, "modules.json") - os.remove(mod_json_path) + mod_json_path = Path(self.pipeline_dir, "modules.json") + mod_json_path.unlink() # Check that the dump function creates the file mod_json_obj.dump() - assert os.path.exists(mod_json_path) + assert mod_json_path.exists() # Check that the dump function writes the correct content with open(mod_json_path) as f: From 3e95d02e402ac74b636008f2756cf3af104d41cf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 7 Feb 2024 09:49:35 +0000 Subject: [PATCH 041/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e040a3dbf..1a6ec96a1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,14 @@ ### Modules +- Handle dirty local module repos by force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) + ### General - fix ignoring changes in partially templated files (e.g. `.gitignore`) ([#2722](https://github.com/nf-core/tools/pull/2722)) - update ruff to 0.2.0 and add it to pre-commit step ([#2725](https://github.com/nf-core/tools/pull/2725)) - Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.2.1 ([#2730](https://github.com/nf-core/tools/pull/2730)) -- Force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From c161ebb8ba011c58f38eef0027999e8c1d8574e9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 04:05:20 +0000 Subject: [PATCH 042/178] Update python:3.11-slim Docker digest to 2a746e2 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 128b43643c..3e561d151d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim@sha256:53d6284a40eae6b625f22870f5faba6c54f2a28db9027408f4dee111f1e885a2 +FROM python:3.11-slim@sha256:2a746e2b9dfd9c155e1218ee5bcaad64c3c8816258c0ee7d25f3893ed2252a1e LABEL authors="phil.ewels@scilifelab.se,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for the nfcore tools" From 5ed48a9c4a705f2900b150626f41f4927d98ba7b Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Feb 2024 10:43:44 +0100 Subject: [PATCH 043/178] Install utils_subworkflows_* from nf-core/modules --- nf_core/pipeline-template/modules.json | 19 + .../nf-core/utils_nextflow_pipeline/main.nf | 126 ++++++ .../nf-core/utils_nextflow_pipeline/meta.yml | 38 ++ .../tests/main.function.nf.test | 54 +++ .../tests/main.function.nf.test.snap | 12 + .../tests/main.workflow.nf.test | 123 ++++++ .../tests/nextflow.config | 9 + .../utils_nextflow_pipeline/tests/tags.yml | 2 + .../nf-core/utils_nfcore_pipeline/main.nf | 395 ++++++++++++++++++ .../nf-core/utils_nfcore_pipeline/meta.yml | 20 + .../tests/main.function.nf.test | 114 +++++ .../tests/main.function.nf.test.snap | 134 ++++++ .../tests/main.workflow.nf.test | 21 + .../tests/main.workflow.nf.test.snap | 15 + .../tests/nextflow.config | 9 + .../utils_nfcore_pipeline/tests/tags.yml | 2 + .../nf-core/utils_nfvalidation_plugin/main.nf | 62 +++ .../utils_nfvalidation_plugin/meta.yml | 44 ++ .../tests/main.nf.test | 200 +++++++++ .../tests/nextflow_schema.json | 96 +++++ .../utils_nfvalidation_plugin/tests/tags.yml | 2 + 21 files changed, 1497 insertions(+) create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json create mode 100644 nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 8660da2d42..5c30ac92fc 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -21,6 +21,25 @@ "installed_by": ["modules"] } } + }, + "subworkflows": { + "nf-core": { + "utils_nextflow_pipeline": { + "branch": "master", + "git_sha": "cd08c91373cd00a73255081340e4914485846ba1", + "installed_by": ["subworkflows"] + }, + "utils_nfcore_pipeline": { + "branch": "master", + "git_sha": "73b63c8e510107010ab13b68cf31189d6cdca8cb", + "installed_by": ["subworkflows"] + }, + "utils_nfvalidation_plugin": { + "branch": "master", + "git_sha": "cd08c91373cd00a73255081340e4914485846ba1", + "installed_by": ["subworkflows"] + } + } } } } diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf new file mode 100644 index 0000000000..ac31f28f66 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -0,0 +1,126 @@ +// +// Subworkflow with functionality that may be useful for any Nextflow pipeline +// + +import org.yaml.snakeyaml.Yaml +import groovy.json.JsonOutput +import nextflow.extension.FilesEx + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NEXTFLOW_PIPELINE { + + take: + print_version // boolean: print version + dump_parameters // boolean: dump parameters + outdir // path: base directory used to publish pipeline results + check_conda_channels // boolean: check conda channels + + main: + + // + // Print workflow version and exit on --version + // + if (print_version) { + log.info "${workflow.manifest.name} ${getWorkflowVersion()}" + System.exit(0) + } + + // + // Dump pipeline parameters to a JSON file + // + if (dump_parameters && outdir) { + dumpParametersToJSON(outdir) + } + + // + // When running with Conda, warn if channels have not been set-up appropriately + // + if (check_conda_channels) { + checkCondaChannels() + } + + emit: + dummy_emit = true +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ + +// +// Generate version string +// +def getWorkflowVersion() { + String version_string = "" + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Dump pipeline parameters to a JSON file +// +def dumpParametersToJSON(outdir) { + def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") + def jsonStr = JsonOutput.toJson(params) + temp_pf.text = JsonOutput.prettyPrint(jsonStr) + + FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.delete() +} + +// +// When running with -profile conda, warn if channels have not been set-up appropriately +// +def checkCondaChannels() { + Yaml parser = new Yaml() + def channels = [] + try { + def config = parser.load("conda config --show channels".execute().text) + channels = config.channels + } catch(NullPointerException | IOException e) { + log.warn "Could not verify conda channel configuration." + return + } + + // Check that all channels are present + // This channel list is ordered by required channel priority. + def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean + + // Check that they are in the right order + def channel_priority_violation = false + def n = required_channels_in_order.size() + for (int i = 0; i < n - 1; i++) { + channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) + } + + if (channels_missing | channel_priority_violation) { + log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + " There is a problem with your Conda configuration!\n\n" + + " You will need to set-up the conda-forge and bioconda channels correctly.\n" + + " Please refer to https://bioconda.github.io/\n" + + " The observed channel order is \n" + + " ${channels}\n" + + " but the following channel order is required:\n" + + " ${required_channels_in_order}\n" + + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + } +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml new file mode 100644 index 0000000000..e5c3a0a828 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml @@ -0,0 +1,38 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NEXTFLOW_PIPELINE" +description: Subworkflow with functionality that may be useful for any Nextflow pipeline +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - print_version: + type: boolean + description: | + Print the version of the pipeline and exit + - dump_parameters: + type: boolean + description: | + Dump the parameters of the pipeline to a JSON file + - output_directory: + type: directory + description: Path to output dir to write JSON file to. + pattern: "results/" + - check_conda_channel: + type: boolean + description: | + Check if the conda channel priority is correct. +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@adamrtalbot" + - "@drpatelh" +maintainers: + - "@adamrtalbot" + - "@drpatelh" + - "@maxulysse" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test new file mode 100644 index 0000000000..8ed4310cac --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test @@ -0,0 +1,54 @@ + +nextflow_function { + + name "Test Functions" + script "subworkflows/nf-core/utils_nextflow_pipeline/main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Test Function getWorkflowVersion") { + + function "getWorkflowVersion" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dumpParametersToJSON") { + + function "dumpParametersToJSON" + + when { + function { + """ + // define inputs of the function here. Example: + input[0] = "$outputDir" + """.stripIndent() + } + } + + then { + assertAll( + { assert function.success } + ) + } + } + + test("Test Function checkCondaChannels") { + + function "checkCondaChannels" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 0000000000..db2030f8b0 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,12 @@ +{ + "Test Function getWorkflowVersion": { + "content": [ + "v9.9.9" + ], + "timestamp": "2024-01-19T11:32:36.031083" + }, + "Test Function checkCondaChannels": { + "content": null, + "timestamp": "2024-01-19T11:32:50.456" + } +} \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test new file mode 100644 index 0000000000..f7c54bc68f --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,123 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NEXTFLOW_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + workflow "UTILS_NEXTFLOW_PIPELINE" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Should run no inputs") { + + when { + params { + outdir = "tests/results" + } + workflow { + """ + print_version = false + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should print version") { + + when { + params { + outdir = "tests/results" + } + workflow { + """ + print_version = true + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.stdout.contains("nextflow_workflow v9.9.9") } + ) + } + } + + test("Should dump params") { + + when { + params { + outdir = "$outputDir" + } + workflow { + """ + print_version = false + dump_parameters = true + outdir = params.outdir + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = params.outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should not create params JSON if no output directory") { + + when { + params { + outdir = "$outputDir" + } + workflow { + """ + print_version = false + dump_parameters = true + outdir = params.outdir + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = null + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config new file mode 100644 index 0000000000..53574ffec4 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml new file mode 100644 index 0000000000..f84761125a --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nextflow_pipeline: + - subworkflows/nf-core/utils_nextflow_pipeline/** diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf new file mode 100644 index 0000000000..6d805bb3fd --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -0,0 +1,395 @@ +// +// Subworkflow with utility functions specific to the nf-core pipeline template +// + +import org.yaml.snakeyaml.Yaml +import nextflow.extension.FilesEx + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NFCORE_PIPELINE { + + main: + valid_config = checkConfigProvided() + + emit: + valid_config +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ + +// +// Warn if a -profile or Nextflow config has not been provided to run the pipeline +// +def checkConfigProvided() { + valid_config = true + if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { + log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + + " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + + " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + + " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + + "Please refer to the quick start section and usage docs for the pipeline.\n " + valid_config = false + } + return valid_config +} + +// +// Citation string for pipeline +// +def workflowCitation() { + return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + + "* The pipeline\n" + + " ${workflow.manifest.doi}\n\n" + + "* The nf-core framework\n" + + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + + "* Software dependencies\n" + + " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" +} + +// +// Generate workflow version string +// +def getWorkflowVersion() { + String version_string = "" + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Get software versions for pipeline +// +def processVersionsFromYAML(yaml_file) { + Yaml yaml = new Yaml() + versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } + return yaml.dumpAsMap(versions).trim() +} + +// +// Get workflow version for pipeline +// +def workflowVersionToYAML() { + return """ + Workflow: + $workflow.manifest.name: ${getWorkflowVersion()} + Nextflow: $workflow.nextflow.version + """.stripIndent().trim() +} + +// +// Get channel of software versions used in pipeline in YAML format +// +def softwareVersionsToYAML(ch_versions) { + return ch_versions + .unique() + .map { processVersionsFromYAML(it) } + .unique() + .mix(Channel.of(workflowVersionToYAML())) +} + +// +// Get workflow summary for MultiQC +// +def paramsSummaryMultiqc(summary_params) { + def summary_section = '' + for (group in summary_params.keySet()) { + def group_params = summary_params.get(group) // This gets the parameters of that particular group + if (group_params) { + summary_section += "

$group

\n" + summary_section += "
\n" + for (param in group_params.keySet()) { + summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" + } + summary_section += "
\n" + } + } + + String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" + yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" + yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" + yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" + yaml_file_text += "plot_type: 'html'\n" + yaml_file_text += "data: |\n" + yaml_file_text += "${summary_section}" + + return yaml_file_text +} + +// +// nf-core logo +// +def nfCoreLogo(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + String.format( + """\n + ${dashedLine(monochrome_logs)} + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} + ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} + ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} + ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} + ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} + ${dashedLine(monochrome_logs)} + """.stripIndent() + ) +} + +// +// Return dashed line +// +def dashedLine(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + return "-${colors.dim}----------------------------------------------------${colors.reset}-" +} + +// +// ANSII colours used for terminal logging +// +def logColours(monochrome_logs=true) { + Map colorcodes = [:] + + // Reset / Meta + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" + colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" + colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" + colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" + + // Regular Colors + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + + // Bold + colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" + colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" + colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" + colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" + colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" + colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" + colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" + colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" + + // Underline + colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" + colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" + colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" + colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" + colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" + colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" + colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" + colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" + + // High Intensity + colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" + colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" + colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" + colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" + colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" + colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" + colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" + colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" + + // Bold High Intensity + colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" + colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" + colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" + colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" + colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" + colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" + colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" + + return colorcodes +} + +// +// Construct and send completion email +// +def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true) { + + // Set up the e-mail variables + def subject = "[$workflow.manifest.name] Successful: $workflow.runName" + if (!workflow.success) { + subject = "[$workflow.manifest.name] FAILED: $workflow.runName" + } + + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['Date Started'] = workflow.start + misc_fields['Date Completed'] = workflow.complete + misc_fields['Pipeline script file path'] = workflow.scriptFile + misc_fields['Pipeline script hash ID'] = workflow.scriptId + if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository + if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId + if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision + misc_fields['Nextflow Version'] = workflow.nextflow.version + misc_fields['Nextflow Build'] = workflow.nextflow.build + misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp + + def email_fields = [:] + email_fields['version'] = getWorkflowVersion() + email_fields['runName'] = workflow.runName + email_fields['success'] = workflow.success + email_fields['dateComplete'] = workflow.complete + email_fields['duration'] = workflow.duration + email_fields['exitStatus'] = workflow.exitStatus + email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + email_fields['errorReport'] = (workflow.errorReport ?: 'None') + email_fields['commandLine'] = workflow.commandLine + email_fields['projectDir'] = workflow.projectDir + email_fields['summary'] = summary << misc_fields + + // Check if we are only sending emails on failure + def email_address = email + if (!email && email_on_fail && !workflow.success) { + email_address = email_on_fail + } + + // Render the TXT template + def engine = new groovy.text.GStringTemplateEngine() + def tf = new File("${workflow.projectDir}/assets/email_template.txt") + def txt_template = engine.createTemplate(tf).make(email_fields) + def email_txt = txt_template.toString() + + // Render the HTML template + def hf = new File("${workflow.projectDir}/assets/email_template.html") + def html_template = engine.createTemplate(hf).make(email_fields) + def email_html = html_template.toString() + + // Render the sendmail template + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}" ] + def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") + def sendmail_template = engine.createTemplate(sf).make(smail_fields) + def sendmail_html = sendmail_template.toString() + + // Send the HTML e-mail + Map colors = logColours(monochrome_logs) + if (email_address) { + try { + if (plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + // Try to send HTML e-mail using sendmail + def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + sendmail_tf.withWriter { w -> w << sendmail_html } + [ 'sendmail', '-t' ].execute() << sendmail_html + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" + } catch (all) { + // Catch failures and try with plaintext + def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] + mail_cmd.execute() << email_html + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" + } + } + + // Write summary e-mail HTML to a file + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") + output_hf.withWriter { w -> w << email_html } + FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") + output_tf.withWriter { w -> w << email_txt } + FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); + output_tf.delete() +} + +// +// Print pipeline summary on completion +// +def completionSummary(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + if (workflow.success) { + if (workflow.stats.ignoredCount == 0) { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" + } else { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" + } + } else { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" + } +} + +// +// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack +// +def imNotification(summary_params, hook_url) { + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['start'] = workflow.start + misc_fields['complete'] = workflow.complete + misc_fields['scriptfile'] = workflow.scriptFile + misc_fields['scriptid'] = workflow.scriptId + if (workflow.repository) misc_fields['repository'] = workflow.repository + if (workflow.commitId) misc_fields['commitid'] = workflow.commitId + if (workflow.revision) misc_fields['revision'] = workflow.revision + misc_fields['nxf_version'] = workflow.nextflow.version + misc_fields['nxf_build'] = workflow.nextflow.build + misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp + + def msg_fields = [:] + msg_fields['version'] = getWorkflowVersion() + msg_fields['runName'] = workflow.runName + msg_fields['success'] = workflow.success + msg_fields['dateComplete'] = workflow.complete + msg_fields['duration'] = workflow.duration + msg_fields['exitStatus'] = workflow.exitStatus + msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + msg_fields['errorReport'] = (workflow.errorReport ?: 'None') + msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") + msg_fields['projectDir'] = workflow.projectDir + msg_fields['summary'] = summary << misc_fields + + // Render the JSON template + def engine = new groovy.text.GStringTemplateEngine() + // Different JSON depending on the service provider + // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format + def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" + def hf = new File("${workflow.projectDir}/assets/${json_path}") + def json_template = engine.createTemplate(hf).make(msg_fields) + def json_message = json_template.toString() + + // POST + def post = new URL(hook_url).openConnection(); + post.setRequestMethod("POST") + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(json_message.getBytes("UTF-8")); + def postRC = post.getResponseCode(); + if (! postRC.equals(200)) { + log.warn(post.getErrorStream().getText()); + } +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml new file mode 100644 index 0000000000..dd1462b232 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFCORE_PIPELINE" +description: Subworkflow with utility functions specific to the nf-core pipeline template +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: [] +output: + - success: + type: boolean + description: | + Dummy output to indicate success +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test new file mode 100644 index 0000000000..f7ea05fcd7 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -0,0 +1,114 @@ + +nextflow_function { + + name "Test Functions" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Test Function checkConfigProvided") { + + function "checkConfigProvided" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function workflowCitation") { + + function "workflowCitation" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function nfCoreLogo") { + + function "nfCoreLogo" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dashedLine") { + + function "dashedLine" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function without logColours") { + + function "logColours" + + when { + function { + """ + input[0] = true + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function with logColours") { + function "logColours" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 0000000000..afb9ab4dca --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,134 @@ +{ + "Test Function checkConfigProvided": { + "content": [ + true + ], + "timestamp": "2024-01-19T11:34:13.548431224" + }, + "Test Function nfCoreLogo": { + "content": [ + "\n\n-\u001b[2m----------------------------------------------------\u001b[0m-\n \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n \u001b[0;32m`._,._,'\u001b[0m\n\u001b[0;35m nextflow_workflow v9.9.9\u001b[0m\n-\u001b[2m----------------------------------------------------\u001b[0m-\n" + ], + "timestamp": "2024-01-19T11:34:38.840454873" + }, + "Test Function workflowCitation": { + "content": [ + "If you use nextflow_workflow for your analysis please cite:\n\n* The pipeline\n https://doi.org/10.5281/zenodo.5070524\n\n* The nf-core framework\n https://doi.org/10.1038/s41587-020-0439-x\n\n* Software dependencies\n https://github.com/nextflow_workflow/blob/master/CITATIONS.md" + ], + "timestamp": "2024-01-19T11:34:22.24352016" + }, + "Test Function without logColours": { + "content": [ + { + "reset": "", + "bold": "", + "dim": "", + "underlined": "", + "blink": "", + "reverse": "", + "hidden": "", + "black": "", + "red": "", + "green": "", + "yellow": "", + "blue": "", + "purple": "", + "cyan": "", + "white": "", + "bblack": "", + "bred": "", + "bgreen": "", + "byellow": "", + "bblue": "", + "bpurple": "", + "bcyan": "", + "bwhite": "", + "ublack": "", + "ured": "", + "ugreen": "", + "uyellow": "", + "ublue": "", + "upurple": "", + "ucyan": "", + "uwhite": "", + "iblack": "", + "ired": "", + "igreen": "", + "iyellow": "", + "iblue": "", + "ipurple": "", + "icyan": "", + "iwhite": "", + "biblack": "", + "bired": "", + "bigreen": "", + "biyellow": "", + "biblue": "", + "bipurple": "", + "bicyan": "", + "biwhite": "" + } + ], + "timestamp": "2024-01-19T11:35:04.418416984" + }, + "Test Function dashedLine": { + "content": [ + "-\u001b[2m----------------------------------------------------\u001b[0m-" + ], + "timestamp": "2024-01-19T11:34:55.420000755" + }, + "Test Function with logColours": { + "content": [ + { + "reset": "\u001b[0m", + "bold": "\u001b[1m", + "dim": "\u001b[2m", + "underlined": "\u001b[4m", + "blink": "\u001b[5m", + "reverse": "\u001b[7m", + "hidden": "\u001b[8m", + "black": "\u001b[0;30m", + "red": "\u001b[0;31m", + "green": "\u001b[0;32m", + "yellow": "\u001b[0;33m", + "blue": "\u001b[0;34m", + "purple": "\u001b[0;35m", + "cyan": "\u001b[0;36m", + "white": "\u001b[0;37m", + "bblack": "\u001b[1;30m", + "bred": "\u001b[1;31m", + "bgreen": "\u001b[1;32m", + "byellow": "\u001b[1;33m", + "bblue": "\u001b[1;34m", + "bpurple": "\u001b[1;35m", + "bcyan": "\u001b[1;36m", + "bwhite": "\u001b[1;37m", + "ublack": "\u001b[4;30m", + "ured": "\u001b[4;31m", + "ugreen": "\u001b[4;32m", + "uyellow": "\u001b[4;33m", + "ublue": "\u001b[4;34m", + "upurple": "\u001b[4;35m", + "ucyan": "\u001b[4;36m", + "uwhite": "\u001b[4;37m", + "iblack": "\u001b[0;90m", + "ired": "\u001b[0;91m", + "igreen": "\u001b[0;92m", + "iyellow": "\u001b[0;93m", + "iblue": "\u001b[0;94m", + "ipurple": "\u001b[0;95m", + "icyan": "\u001b[0;96m", + "iwhite": "\u001b[0;97m", + "biblack": "\u001b[1;90m", + "bired": "\u001b[1;91m", + "bigreen": "\u001b[1;92m", + "biyellow": "\u001b[1;93m", + "biblue": "\u001b[1;94m", + "bipurple": "\u001b[1;95m", + "bicyan": "\u001b[1;96m", + "biwhite": "\u001b[1;97m" + } + ], + "timestamp": "2024-01-19T11:35:13.436366565" + } +} \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test new file mode 100644 index 0000000000..c5f7776a67 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,21 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap new file mode 100644 index 0000000000..d07ce54c51 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap @@ -0,0 +1,15 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "timestamp": "2024-01-19T11:35:22.538940073" + } +} \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config new file mode 100644 index 0000000000..d0a926bf6d --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml new file mode 100644 index 0000000000..ac8523c9a2 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nfcore_pipeline: + - subworkflows/nf-core/utils_nfcore_pipeline/** diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf new file mode 100644 index 0000000000..2585b65d1b --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf @@ -0,0 +1,62 @@ +// +// Subworkflow that uses the nf-validation plugin to render help text and parameter summary +// + +/* +======================================================================================== + IMPORT NF-VALIDATION PLUGIN +======================================================================================== +*/ + +include { paramsHelp } from 'plugin/nf-validation' +include { paramsSummaryLog } from 'plugin/nf-validation' +include { validateParameters } from 'plugin/nf-validation' + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NFVALIDATION_PLUGIN { + + take: + print_help // boolean: print help + workflow_command // string: default commmand used to run pipeline + pre_help_text // string: string to be printed before help text and summary log + post_help_text // string: string to be printed after help text and summary log + validate_params // boolean: validate parameters + schema_filename // path: JSON schema file, null to use default value + + main: + + log.debug "Using schema file: ${schema_filename}" + + // Default values for strings + pre_help_text = pre_help_text ?: '' + post_help_text = post_help_text ?: '' + workflow_command = workflow_command ?: '' + + // + // Print help message if needed + // + if (print_help) { + log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text + System.exit(0) + } + + // + // Print parameter summary to stdout + // + log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text + + // + // Validate parameters relative to the parameter JSON schema + // + if (validate_params){ + validateParameters(parameters_schema: schema_filename) + } + + emit: + dummy_emit = true +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml new file mode 100644 index 0000000000..3d4a6b04f5 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml @@ -0,0 +1,44 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFVALIDATION_PLUGIN" +description: Use nf-validation to initiate and validate a pipeline +keywords: + - utility + - pipeline + - initialise + - validation +components: [] +input: + - print_help: + type: boolean + description: | + Print help message and exit + - workflow_command: + type: string + description: | + The command to run the workflow e.g. "nextflow run main.nf" + - pre_help_text: + type: string + description: | + Text to print before the help message + - post_help_text: + type: string + description: | + Text to print after the help message + - validate_params: + type: boolean + description: | + Validate the parameters and error if invalid. + - schema_filename: + type: string + description: | + The filename of the schema to validate against. +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test new file mode 100644 index 0000000000..517ee54e48 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test @@ -0,0 +1,200 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFVALIDATION_PLUGIN" + script "../main.nf" + workflow "UTILS_NFVALIDATION_PLUGIN" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "plugin/nf-validation" + tag "'plugin/nf-validation'" + tag "utils_nfvalidation_plugin" + tag "subworkflows/utils_nfvalidation_plugin" + + test("Should run nothing") { + + when { + + params { + monochrome_logs = true + test_data = '' + } + + workflow { + """ + help = false + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should run help") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } } + ) + } + } + + test("Should run help with command") { + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = "nextflow run noorg/doesntexist" + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } } + ) + } + } + + test("Should run help with extra text") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = "nextflow run noorg/doesntexist" + pre_help_text = "pre-help-text" + post_help_text = "post-help-text" + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('pre-help-text') } }, + { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } }, + { assert workflow.stdout.any { it.contains('post-help-text') } } + ) + } + } + + test("Should validate params") { + + when { + + params { + monochrome_logs = true + test_data = '' + outdir = 1 + } + workflow { + """ + help = false + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = true + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } } + ) + } + } +} \ No newline at end of file diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json new file mode 100644 index 0000000000..7626c1c93e --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", + "title": ". pipeline parameters", + "description": "", + "type": "object", + "definitions": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["outdir"], + "properties": { + "validate_params": { + "type": "boolean", + "description": "Validate parameters?", + "default": true, + "hidden": true + }, + "outdir": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" + }, + "test_data_base": { + "type": "string", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/modules", + "description": "Base for test data directory", + "hidden": true + }, + "test_data": { + "type": "string", + "description": "Fake test data param", + "hidden": true + } + } + }, + "generic_options": { + "title": "Generic options", + "type": "object", + "fa_icon": "fas fa-file-import", + "description": "Less common options for the pipeline, typically set in a config file.", + "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", + "properties": { + "help": { + "type": "boolean", + "description": "Display help text.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "logo": { + "type": "boolean", + "default": true, + "description": "Display nf-core logo in console output.", + "fa_icon": "fas fa-image", + "hidden": true + }, + "singularity_pull_docker_container": { + "type": "boolean", + "description": "Pull Singularity container from Docker?", + "hidden": true + }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "hidden": true + }, + "monochrome_logs": { + "type": "boolean", + "description": "Use monochrome_logs", + "hidden": true + } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/input_output_options" + }, + { + "$ref": "#/definitions/generic_options" + } + ] +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml new file mode 100644 index 0000000000..60b1cfff49 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nfvalidation_plugin: + - subworkflows/nf-core/utils_nfvalidation_plugin/** From f6572df81cffedc97c87e1fc6c3087dcb82f6c85 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 8 Feb 2024 11:08:39 +0100 Subject: [PATCH 044/178] add regular component update action --- .../workflows/update_components_template.yml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/update_components_template.yml diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml new file mode 100644 index 0000000000..9b14bdce98 --- /dev/null +++ b/.github/workflows/update_components_template.yml @@ -0,0 +1,46 @@ +name: Update Modules Template + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +jobs: + update_modules: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + + - name: Install nf-core + run: pip install nf-core + + - name: Update modules + run: nf-core modules update -all + working-directory: nf-core/pipeline-template + + - name: Update subworkflows + run: nf-core subworkflows update -all + working-directory: nf-core/pipeline-template + + # Commit the changes + - name: Commit changes + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git add . + git status + git commit -m "[automated] Fix code linting" + + # Open a new PR to dev with the changes + - name: Create PR + run: | + git checkout -b update-modules + git push origin update-modules + gh pr create --title "Update modules in template" --body "This PR updates the modules in the pipeline template" --base dev --head update-modules From 2768222600bacb836943c7b9d14d3bf54e7829a8 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 8 Feb 2024 11:13:05 +0000 Subject: [PATCH 045/178] [automated] Fix code linting --- .github/workflows/create-lint-wf.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 1e27c7126c..deeb225511 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -68,4 +68,3 @@ jobs: - name: Cleanup work directory run: sudo rm -rf create-lint-wf if: always() - From a1417c0718fe73d8b3dadd868c62098ad3a7ce53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 8 Feb 2024 11:37:13 +0000 Subject: [PATCH 046/178] add test nf-core commands only in the tools repo --- .github/actions/create-lint-wf/action.yml | 25 +++++++++++++++++++++++ .github/workflows/create-lint-wf.yml | 19 +++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 3b87c767a1..781b63f1b4 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -29,6 +29,11 @@ runs: export NXF_WORK=$(pwd) nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain + # Try syncing it before we change anything + - name: nf-core sync + run: nf-core --log-file log.txt sync --dir nf-core-testpipeline/ + working-directory: create-lint-wf + # Run code style linting - name: run pre-commit shell: bash @@ -76,6 +81,26 @@ runs: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-lint-wf/log.txt + - name: nf-core modules install + run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force + working-directory: create-lint-wf + + - name: nf-core modules install gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastp --dir nf-core-testpipeline/ + working-directory: create-lint-wf + + - name: nf-core modules list local + run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ + working-directory: create-lint-wf + + - name: nf-core modules list remote + run: nf-core --log-file log.txt modules list remote + working-directory: create-lint-wf + + - name: nf-core modules list remote gitlab + run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote + working-directory: create-lint-wf + - name: Cleanup work directory shell: bash run: sudo rm -rf create-lint-wf diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index deeb225511..d9d0437b08 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -65,6 +65,25 @@ jobs: with: NXF_VER: ${{ matrix.NXF_VER }} + # Build a module from the template + - name: nf-core modules create + run: nf-core --log-file log.txt modules create bpipe --dir nf-core-testpipeline --author @nf-core-bot --label process_low --meta + working-directory: create-lint-wf + + # Remove TODO statements + - name: remove TODO + run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \; + working-directory: create-lint-wf + + # Run the other nf-core commands + - name: nf-core list + run: nf-core --log-file log.txt list + working-directory: create-lint-wf + + - name: nf-core schema + run: nf-core --log-file log.txt schema build --dir nf-core-testpipeline/ --no-prompts + working-directory: create-lint-wf + - name: Cleanup work directory run: sudo rm -rf create-lint-wf if: always() From 53487ec543a4fc5446fc29fd4a4ed2d84517116b Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Feb 2024 12:44:32 +0100 Subject: [PATCH 047/178] Remove lib directory from pipeline template --- .../lib/NfcoreTemplate.groovy | 356 ------------------ nf_core/pipeline-template/lib/Utils.groovy | 47 --- .../pipeline-template/lib/WorkflowMain.groovy | 80 ---- .../lib/WorkflowPipeline.groovy | 123 ------ 4 files changed, 606 deletions(-) delete mode 100755 nf_core/pipeline-template/lib/NfcoreTemplate.groovy delete mode 100644 nf_core/pipeline-template/lib/Utils.groovy delete mode 100755 nf_core/pipeline-template/lib/WorkflowMain.groovy delete mode 100755 nf_core/pipeline-template/lib/WorkflowPipeline.groovy diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy deleted file mode 100755 index 0f8d021a03..0000000000 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ /dev/null @@ -1,356 +0,0 @@ -// -// This file holds several functions used within the nf-core pipeline template. -// - -import org.yaml.snakeyaml.Yaml -import groovy.json.JsonOutput -import nextflow.extension.FilesEx - -class NfcoreTemplate { - - // - // Check AWS Batch related parameters have been specified correctly - // - public static void awsBatch(workflow, params) { - if (workflow.profile.contains('awsbatch')) { - // Check params.awsqueue and params.awsregion have been set if running on AWSBatch - assert (params.awsqueue && params.awsregion) : "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" - // Check outdir paths to be S3 buckets if running on AWSBatch - assert params.outdir.startsWith('s3:') : "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" - } - } - - // - // Warn if a -profile or Nextflow config has not been provided to run the pipeline - // - public static void checkConfigProvided(workflow, log) { - if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { - log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + - "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + - " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + - " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + - " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + - "Please refer to the quick start section and usage docs for the pipeline.\n " - } - } - - // - // Generate version string - // - public static String version(workflow) { - String version_string = "" - - if (workflow.manifest.version) { - def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' - version_string += "${prefix_v}${workflow.manifest.version}" - } - - if (workflow.commitId) { - def git_shortsha = workflow.commitId.substring(0, 7) - version_string += "-g${git_shortsha}" - } - - return version_string - } - - // - // Construct and send completion email - // - public static void email(workflow, params, summary_params, projectDir, log, multiqc_report=[]) { - - // Set up the e-mail variables - def subject = "[$workflow.manifest.name] Successful: $workflow.runName" - if (!workflow.success) { - subject = "[$workflow.manifest.name] FAILED: $workflow.runName" - } - - def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['Date Started'] = workflow.start - misc_fields['Date Completed'] = workflow.complete - misc_fields['Pipeline script file path'] = workflow.scriptFile - misc_fields['Pipeline script hash ID'] = workflow.scriptId - if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository - if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId - if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision - misc_fields['Nextflow Version'] = workflow.nextflow.version - misc_fields['Nextflow Build'] = workflow.nextflow.build - misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp - - def email_fields = [:] - email_fields['version'] = NfcoreTemplate.version(workflow) - email_fields['runName'] = workflow.runName - email_fields['success'] = workflow.success - email_fields['dateComplete'] = workflow.complete - email_fields['duration'] = workflow.duration - email_fields['exitStatus'] = workflow.exitStatus - email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - email_fields['errorReport'] = (workflow.errorReport ?: 'None') - email_fields['commandLine'] = workflow.commandLine - email_fields['projectDir'] = workflow.projectDir - email_fields['summary'] = summary << misc_fields - - // On success try attach the multiqc report - def mqc_report = null - try { - if (workflow.success) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" - } - mqc_report = mqc_report[0] - } - } - } catch (all) { - if (multiqc_report) { - log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" - } - } - - // Check if we are only sending emails on failure - def email_address = params.email - if (!params.email && params.email_on_fail && !workflow.success) { - email_address = params.email_on_fail - } - - // Render the TXT template - def engine = new groovy.text.GStringTemplateEngine() - def tf = new File("$projectDir/assets/email_template.txt") - def txt_template = engine.createTemplate(tf).make(email_fields) - def email_txt = txt_template.toString() - - // Render the HTML template - def hf = new File("$projectDir/assets/email_template.html") - def html_template = engine.createTemplate(hf).make(email_fields) - def email_html = html_template.toString() - - // Render the sendmail template - def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] - def sf = new File("$projectDir/assets/sendmail_template.txt") - def sendmail_template = engine.createTemplate(sf).make(smail_fields) - def sendmail_html = sendmail_template.toString() - - // Send the HTML e-mail - Map colors = logColours(params.monochrome_logs) - if (email_address) { - try { - if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } - // Try to send HTML e-mail using sendmail - def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") - sendmail_tf.withWriter { w -> w << sendmail_html } - [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" - } catch (all) { - // Catch failures and try with plaintext - def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report != null && mqc_report.size() <= max_multiqc_email_size.toBytes() ) { - mail_cmd += [ '-A', mqc_report ] - } - mail_cmd.execute() << email_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" - } - } - - // Write summary e-mail HTML to a file - def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") - output_hf.withWriter { w -> w << email_html } - FilesEx.copyTo(output_hf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.html"); - output_hf.delete() - - // Write summary e-mail TXT to a file - def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") - output_tf.withWriter { w -> w << email_txt } - FilesEx.copyTo(output_tf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.txt"); - output_tf.delete() - } - - // - // Construct and send a notification to a web server as JSON - // e.g. Microsoft Teams and Slack - // - public static void IM_notification(workflow, params, summary_params, projectDir, log) { - def hook_url = params.hook_url - - def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) misc_fields['repository'] = workflow.repository - if (workflow.commitId) misc_fields['commitid'] = workflow.commitId - if (workflow.revision) misc_fields['revision'] = workflow.revision - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = NfcoreTemplate.version(workflow) - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("$projectDir/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection(); - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")); - def postRC = post.getResponseCode(); - if (! postRC.equals(200)) { - log.warn(post.getErrorStream().getText()); - } - } - - // - // Dump pipeline parameters in a json file - // - public static void dump_parameters(workflow, params) { - def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') - def filename = "params_${timestamp}.json" - def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") - def jsonStr = JsonOutput.toJson(params) - temp_pf.text = JsonOutput.prettyPrint(jsonStr) - - FilesEx.copyTo(temp_pf.toPath(), "${params.outdir}/pipeline_info/params_${timestamp}.json") - temp_pf.delete() - } - - // - // Print pipeline summary on completion - // - public static void summary(workflow, params, log) { - Map colors = logColours(params.monochrome_logs) - if (workflow.success) { - if (workflow.stats.ignoredCount == 0) { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" - } - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" - } - } - - // - // ANSII Colours used for terminal logging - // - public static Map logColours(Boolean monochrome_logs) { - Map colorcodes = [:] - - // Reset / Meta - colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" - colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" - colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" - colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" - colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" - colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" - colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" - - // Regular Colors - colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" - colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" - colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" - colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" - colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" - colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" - colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" - colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" - - // Bold - colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" - colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" - colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" - colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" - colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" - colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" - colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" - colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" - - // Underline - colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" - colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" - colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" - colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" - colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" - colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" - colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" - colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" - - // High Intensity - colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" - colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" - colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" - colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" - colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" - colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" - colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" - colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" - - // Bold High Intensity - colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" - colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" - colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" - colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" - colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" - colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" - colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" - colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" - - return colorcodes - } - - // - // Does what is says on the tin - // - public static String dashedLine(monochrome_logs) { - Map colors = logColours(monochrome_logs) - return "-${colors.dim}----------------------------------------------------${colors.reset}-" - } - - // - // nf-core logo - // - public static String logo(workflow, monochrome_logs) { - Map colors = logColours(monochrome_logs) - String workflow_version = NfcoreTemplate.version(workflow) - String.format( - """\n - ${dashedLine(monochrome_logs)}{% if branded %} - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} - ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} - ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} - ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset}{% endif %} - ${colors.purple} ${workflow.manifest.name} ${workflow_version}${colors.reset} - ${dashedLine(monochrome_logs)} - """.stripIndent() - ) - } -} diff --git a/nf_core/pipeline-template/lib/Utils.groovy b/nf_core/pipeline-template/lib/Utils.groovy deleted file mode 100644 index 8d030f4e84..0000000000 --- a/nf_core/pipeline-template/lib/Utils.groovy +++ /dev/null @@ -1,47 +0,0 @@ -// -// This file holds several Groovy functions that could be useful for any Nextflow pipeline -// - -import org.yaml.snakeyaml.Yaml - -class Utils { - - // - // When running with -profile conda, warn if channels have not been set-up appropriately - // - public static void checkCondaChannels(log) { - Yaml parser = new Yaml() - def channels = [] - try { - def config = parser.load("conda config --show channels".execute().text) - channels = config.channels - } catch(NullPointerException | IOException e) { - log.warn "Could not verify conda channel configuration." - return - } - - // Check that all channels are present - // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] - def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean - - // Check that they are in the right order - def channel_priority_violation = false - def n = required_channels_in_order.size() - for (int i = 0; i < n - 1; i++) { - channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) - } - - if (channels_missing | channel_priority_violation) { - log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " There is a problem with your Conda configuration!\n\n" + - " You will need to set-up the conda-forge and bioconda channels correctly.\n" + - " Please refer to https://bioconda.github.io/\n" + - " The observed channel order is \n" + - " ${channels}\n" + - " but the following channel order is required:\n" + - " ${required_channels_in_order}\n" + - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - } - } -} diff --git a/nf_core/pipeline-template/lib/WorkflowMain.groovy b/nf_core/pipeline-template/lib/WorkflowMain.groovy deleted file mode 100755 index a254b2b22a..0000000000 --- a/nf_core/pipeline-template/lib/WorkflowMain.groovy +++ /dev/null @@ -1,80 +0,0 @@ -// -// This file holds several functions specific to the main.nf workflow in the {{ name }} pipeline -// - -import nextflow.Nextflow - -class WorkflowMain { - - // - // Citation string for pipeline - // - public static String citation(workflow) { - return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + - // TODO nf-core: Add Zenodo DOI for pipeline after first release - //"* The pipeline\n" + - //" https://doi.org/10.5281/zenodo.XXXXXXX\n\n" + - "* The nf-core framework\n" + - " https://doi.org/10.1038/s41587-020-0439-x\n\n" + - "* Software dependencies\n" + - " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" - } - - - // - // Validate parameters and print summary to screen - // - public static void initialise(workflow, params, log, args) { - - // Print workflow version and exit on --version - if (params.version) { - String workflow_version = NfcoreTemplate.version(workflow) - log.info "${workflow.manifest.name} ${workflow_version}" - System.exit(0) - } - - // Check that a -profile or Nextflow config has been provided to run the pipeline - NfcoreTemplate.checkConfigProvided(workflow, log) - // Check that the profile doesn't contain spaces and doesn't end with a trailing comma - checkProfile(workflow.profile, args, log) - - // Check that conda channels are set-up correctly - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - Utils.checkCondaChannels(log) - } - - // Check AWS batch settings - NfcoreTemplate.awsBatch(workflow, params) - - // Check input has been provided - if (!params.input) { - Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'") - } - } - - {%- if igenomes %} - // - // Get attribute from genome config file e.g. fasta - // - public static Object getGenomeAttribute(params, attribute) { - if (params.genomes && params.genome && params.genomes.containsKey(params.genome)) { - if (params.genomes[ params.genome ].containsKey(attribute)) { - return params.genomes[ params.genome ][ attribute ] - } - } - return null - } - {%- endif %} - - // - // Exit pipeline if --profile contains spaces - // - private static void checkProfile(profile, args, log) { - if (profile.endsWith(',')) { - Nextflow.error "Profile cannot end with a trailing comma. Please remove the comma from the end of the profile string.\nHint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." - } - if (args[0]) { - log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${args[0]}` has been detected.\n Hint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." - } - } -} diff --git a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy b/nf_core/pipeline-template/lib/WorkflowPipeline.groovy deleted file mode 100755 index f9a7859ef5..0000000000 --- a/nf_core/pipeline-template/lib/WorkflowPipeline.groovy +++ /dev/null @@ -1,123 +0,0 @@ -// -// This file holds several functions specific to the workflow/{{ short_name }}.nf in the {{ name }} pipeline -// - -import nextflow.Nextflow -import groovy.text.SimpleTemplateEngine - -class Workflow{{ short_name[0]|upper }}{{ short_name[1:] }} { - - // - // Check and validate parameters - // - public static void initialise(params, log) { -{% if igenomes %} - genomeExistsError(params, log) -{% endif %} - - if (!params.fasta) { - Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." - } - } - - // - // Get workflow summary for MultiQC - // - public static String paramsSummaryMultiqc(workflow, summary) { - String summary_section = '' - for (group in summary.keySet()) { - def group_params = summary.get(group) // This gets the parameters of that particular group - if (group_params) { - summary_section += "

$group

\n" - summary_section += "
\n" - for (param in group_params.keySet()) { - summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" - } - summary_section += "
\n" - } - } - - String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" - yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" - yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" - yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" - yaml_file_text += "plot_type: 'html'\n" - yaml_file_text += "data: |\n" - yaml_file_text += "${summary_section}" - return yaml_file_text - } - - // - // Generate methods description for MultiQC - // - - public static String toolCitationText(params) { - - // TODO nf-core: Optionally add in-text citation tools to this list. - // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "Tool (Foo et al. 2023)" : "", - // Uncomment function in methodsDescriptionText to render in MultiQC report - def citation_text = [ - "Tools used in the workflow included:", - "FastQC (Andrews 2010),", - "MultiQC (Ewels et al. 2016)", - "." - ].join(' ').trim() - - return citation_text - } - - public static String toolBibliographyText(params) { - - // TODO Optionally add bibliographic entries to this list. - // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "
  • Author (2023) Pub name, Journal, DOI
    • " : "", - // Uncomment function in methodsDescriptionText to render in MultiQC report - def reference_text = [ - "
  • Andrews S, (2010) FastQC, URL: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).
    • ", - "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • " - ].join(' ').trim() - - return reference_text - } - - public static String methodsDescriptionText(run_workflow, mqc_methods_yaml, params) { - // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file - def meta = [:] - meta.workflow = run_workflow.toMap() - meta["manifest_map"] = run_workflow.manifest.toMap() - - // Pipeline DOI - meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" - meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " - - // Tool references - meta["tool_citations"] = "" - meta["tool_bibliography"] = "" - - // TODO Only uncomment below if logic in toolCitationText/toolBibliographyText has been filled! - //meta["tool_citations"] = toolCitationText(params).replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") - //meta["tool_bibliography"] = toolBibliographyText(params) - - - def methods_text = mqc_methods_yaml.text - - def engine = new SimpleTemplateEngine() - def description_html = engine.createTemplate(methods_text).make(meta) - - return description_html - } - {%- if igenomes %} - - // - // Exit pipeline if incorrect --genome key provided - // - private static void genomeExistsError(params, log) { - if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { - def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + - " Currently, the available genome keys are:\n" + - " ${params.genomes.keySet().join(", ")}\n" + - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - Nextflow.error(error_string) - } - } -{% endif -%}} From fdd020d45d7c45b163a863cf3425d6b402d46c4f Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 8 Feb 2024 12:44:34 +0100 Subject: [PATCH 048/178] fix syntax --- .github/actions/create-lint-wf/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 781b63f1b4..aa711b73c2 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -82,22 +82,27 @@ runs: path: create-lint-wf/log.txt - name: nf-core modules install + shell: bash run: nf-core --log-file log.txt modules install fastqc --dir nf-core-testpipeline/ --force working-directory: create-lint-wf - name: nf-core modules install gitlab + shell: bash run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git --branch branch-tester install fastp --dir nf-core-testpipeline/ working-directory: create-lint-wf - name: nf-core modules list local + shell: bash run: nf-core --log-file log.txt modules list local --dir nf-core-testpipeline/ working-directory: create-lint-wf - name: nf-core modules list remote + shell: bash run: nf-core --log-file log.txt modules list remote working-directory: create-lint-wf - name: nf-core modules list remote gitlab + shell: bash run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote working-directory: create-lint-wf From 99fb69f5263c5a63244682821fcf777bc6c899d3 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Feb 2024 12:44:56 +0100 Subject: [PATCH 049/178] Change output channel name in input_check subworkflow --- nf_core/pipeline-template/subworkflows/local/input_check.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/input_check.nf b/nf_core/pipeline-template/subworkflows/local/input_check.nf index 0aecf87fb7..562f44cc69 100644 --- a/nf_core/pipeline-template/subworkflows/local/input_check.nf +++ b/nf_core/pipeline-template/subworkflows/local/input_check.nf @@ -13,10 +13,10 @@ workflow INPUT_CHECK { .csv .splitCsv ( header:true, sep:',' ) .map { create_fastq_channel(it) } - .set { reads } + .set { samplesheet } emit: - reads // channel: [ val(meta), [ reads ] ] + samplesheet // channel: [ val(meta), [ reads ] ] versions = SAMPLESHEET_CHECK.out.versions // channel: [ versions.yml ] } From bae90f0f350ac2bc48991b1c938b98e28f7e0d75 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Feb 2024 12:46:07 +0100 Subject: [PATCH 050/178] Delete custom/dumpsoftwareversions module --- nf_core/pipeline-template/modules.json | 27 +++-- .../dumpsoftwareversions/environment.yml | 7 -- .../custom/dumpsoftwareversions/main.nf | 24 ----- .../custom/dumpsoftwareversions/meta.yml | 37 ------- .../templates/dumpsoftwareversions.py | 102 ------------------ .../dumpsoftwareversions/tests/main.nf.test | 43 -------- .../tests/main.nf.test.snap | 33 ------ .../dumpsoftwareversions/tests/tags.yml | 2 - 8 files changed, 16 insertions(+), 259 deletions(-) delete mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml delete mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf delete mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml delete mode 100755 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py delete mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test delete mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap delete mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 5c30ac92fc..c558969239 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -5,20 +5,19 @@ "https://github.com/nf-core/modules.git": { "modules": { "nf-core": { - "custom/dumpsoftwareversions": { - "branch": "master", - "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", - "installed_by": ["modules"] - }, "fastqc": { "branch": "master", "git_sha": "c9488585ce7bd35ccd2a30faa2371454c8112fb9", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "multiqc": { "branch": "master", "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] } } }, @@ -27,20 +26,26 @@ "utils_nextflow_pipeline": { "branch": "master", "git_sha": "cd08c91373cd00a73255081340e4914485846ba1", - "installed_by": ["subworkflows"] + "installed_by": [ + "subworkflows" + ] }, "utils_nfcore_pipeline": { "branch": "master", "git_sha": "73b63c8e510107010ab13b68cf31189d6cdca8cb", - "installed_by": ["subworkflows"] + "installed_by": [ + "subworkflows" + ] }, "utils_nfvalidation_plugin": { "branch": "master", "git_sha": "cd08c91373cd00a73255081340e4914485846ba1", - "installed_by": ["subworkflows"] + "installed_by": [ + "subworkflows" + ] } } } } } -} +} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml deleted file mode 100644 index 9b3272bc11..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: custom_dumpsoftwareversions -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - bioconda::multiqc=1.19 diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf deleted file mode 100644 index f2187611cc..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ /dev/null @@ -1,24 +0,0 @@ -process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_single' - - // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : - 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" - - input: - path versions - - output: - path "software_versions.yml" , emit: yml - path "software_versions_mqc.yml", emit: mqc_yml - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - template 'dumpsoftwareversions.py' -} diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml deleted file mode 100644 index 5f15a5fde0..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ /dev/null @@ -1,37 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json -name: custom_dumpsoftwareversions -description: Custom module used to dump software versions within the nf-core pipeline template -keywords: - - custom - - dump - - version -tools: - - custom: - description: Custom module used to dump software versions within the nf-core pipeline template - homepage: https://github.com/nf-core/tools - documentation: https://github.com/nf-core/tools - licence: ["MIT"] -input: - - versions: - type: file - description: YML file containing software versions - pattern: "*.yml" -output: - - yml: - type: file - description: Standard YML file containing software versions - pattern: "software_versions.yml" - - mqc_yml: - type: file - description: MultiQC custom content YML file containing software versions - pattern: "software_versions_mqc.yml" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@drpatelh" - - "@grst" -maintainers: - - "@drpatelh" - - "@grst" diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py deleted file mode 100755 index e55b8d43a9..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python - - -"""Provide functions to merge multiple versions.yml files.""" - - -import platform -from textwrap import dedent - -import yaml - - -def _make_versions_html(versions): - """Generate a tabular HTML output of all versions for MultiQC.""" - html = [ - dedent( - """\\ - - - - - - - - - - """ - ) - ] - for process, tmp_versions in sorted(versions.items()): - html.append("") - for i, (tool, version) in enumerate(sorted(tmp_versions.items())): - html.append( - dedent( - f"""\\ - - - - - - """ - ) - ) - html.append("") - html.append("
    Process Name Software Version
    {process if (i == 0) else ''}{tool}{version}
    ") - return "\\n".join(html) - - -def main(): - """Load all version files and generate merged output.""" - versions_this_module = {} - versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, - } - - with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - - # aggregate versions by the module name (derived from fully-qualified process name) - versions_by_module = {} - for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - - versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", - } - - versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), - } - - with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) - with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - - with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) - - -if __name__ == "__main__": - main() diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test deleted file mode 100644 index b1e1630bb3..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test +++ /dev/null @@ -1,43 +0,0 @@ -nextflow_process { - - name "Test Process CUSTOM_DUMPSOFTWAREVERSIONS" - script "../main.nf" - process "CUSTOM_DUMPSOFTWAREVERSIONS" - tag "modules" - tag "modules_nfcore" - tag "custom" - tag "dumpsoftwareversions" - tag "custom/dumpsoftwareversions" - - test("Should run without failures") { - when { - process { - """ - def tool1_version = ''' - TOOL1: - tool1: 0.11.9 - '''.stripIndent() - - def tool2_version = ''' - TOOL2: - tool2: 1.9 - '''.stripIndent() - - input[0] = Channel.of(tool1_version, tool2_version).collectFile() - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot( - process.out.versions, - file(process.out.mqc_yml[0]).readLines()[0..10], - file(process.out.yml[0]).readLines()[0..7] - ).match() - } - ) - } - } -} diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap deleted file mode 100644 index 5f59a936d7..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap +++ /dev/null @@ -1,33 +0,0 @@ -{ - "Should run without failures": { - "content": [ - [ - "versions.yml:md5,76d454d92244589d32455833f7c1ba6d" - ], - [ - "data: \"\\n\\n \\n \\n \\n \\n \\n \\n \\n\\", - " \\n\\n\\n \\n \\n\\", - " \\ \\n\\n\\n\\n \\n \\", - " \\ \\n \\n\\n\\n\\n\\", - " \\n\\n \\n \\n\\", - " \\ \\n\\n\\n\\n\\n\\n \\n\\", - " \\ \\n \\n\\n\\n\\n\\", - " \\n\\n \\n \\n\\" - ], - [ - "CUSTOM_DUMPSOFTWAREVERSIONS:", - " python: 3.11.7", - " yaml: 5.4.1", - "TOOL1:", - " tool1: 0.11.9", - "TOOL2:", - " tool2: '1.9'", - "Workflow:" - ] - ], - "timestamp": "2024-01-09T23:01:18.710682" - } -} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml deleted file mode 100644 index 405aa24ae3..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -custom/dumpsoftwareversions: - - modules/nf-core/custom/dumpsoftwareversions/** From a78d41b2a1647b70f4d5bf63c7bd6738f5412c16 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Feb 2024 12:46:25 +0100 Subject: [PATCH 051/178] Delete fasta from test.config --- nf_core/pipeline-template/conf/test.config | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 49bfe8a6db..499c72054b 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -24,11 +24,8 @@ params { // TODO nf-core: Give any required params for the test so that command line flags are not needed input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' - {% if igenomes -%} + {% if igenomes %} // Genome references genome = 'R64-1-1' - {%- else -%} - // Fasta references - fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' {%- endif %} } From 7343ce4adc481671e9cb50252aa6a84f45597cf3 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Feb 2024 12:46:46 +0100 Subject: [PATCH 052/178] Add local subworkflow for pipeline utilities --- .../utils_nfcore_pipeline_pipeline/main.nf | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf new file mode 100644 index 0000000000..945d270741 --- /dev/null +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -0,0 +1,177 @@ +// +// Subworkflow with functionality specific to the nf-core/pipeline pipeline +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' +include { paramsSummaryMap } from 'plugin/nf-validation' +include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' +include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' +include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' +include { dashedLine } from '../../nf-core/utils_nfcore_pipeline' +include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' +include { imNotification } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' +include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' +include { INPUT_CHECK } from '../../local/input_check' + +/* +======================================================================================== + SUBWORKFLOW TO INITIALISE PIPELINE +======================================================================================== +*/ + +workflow PIPELINE_INITIALISATION { + + take: + version // boolean: Display version and exit + help // boolean: Display help text + validate_params // boolean: Boolean whether to validate parameters against the schema at runtime + monochrome_logs // boolean: Do not use coloured log outputs + outdir // string: The output directory where the results will be saved + input // string: Path to input samplesheet + + main: + + ch_versions = Channel.empty() + + // + // Print version and exit if required and dump pipeline parameters to JSON file + // + UTILS_NEXTFLOW_PIPELINE ( + version, + true, + outdir, + workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1 + ) + + // + // Validate parameters and generate parameter summary to stdout + // + pre_help_text = nfCoreLogo(monochrome_logs) + post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs) + def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " + UTILS_NFVALIDATION_PLUGIN ( + help, + workflow_command, + pre_help_text, + post_help_text, + validate_params, + "nextflow_schema.json" + ) + + // + // Check config provided to the pipeline + // + UTILS_NFCORE_PIPELINE () + + // + // Custom validation for pipeline parameters + // + validateInputParameters() + + // + // SUBWORKFLOW: Read in samplesheet, validate and stage input files + // + INPUT_CHECK ( + file(input) + ) + ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) + // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the samplesheet with Channel.fromSamplesheet("input") + // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ + // ! There is currently no tooling to help you write a sample sheet schema + + emit: + samplesheet = INPUT_CHECK.out.samplesheet + versions = ch_versions +} + +/* +======================================================================================== + SUBWORKFLOW FOR PIPELINE COMPLETION +======================================================================================== +*/ + +workflow PIPELINE_COMPLETION { + + take: + email // string: email address + email_on_fail // string: email address sent on pipeline failure + plaintext_email // boolean: Send plain-text email instead of HTML + outdir // path: Path to output directory where results will be published + monochrome_logs // boolean: Disable ANSI colour codes in log output + hook_url // string: hook URL for notifications + + main: + + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + + // + // Completion email and summary + // + workflow.onComplete { + if (email || email_on_fail) { + completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs) + } + + completionSummary(monochrome_logs) + + if (hook_url) { + imNotification(summary_params, hook_url) + } + } + + // + // Log warning if exceeding resource requests + // + workflow.onError { + if (workflow.errorReport.contains("Process requirement exceeds available memory")) { + log.warning "🛑 Default resources exceed availability 🛑 " + log.warning "💡 See here on how to configure pipeline: https://nf-co.re/docs/usage/configuration#tuning-workflow-resources 💡" + } + } +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ + +// +// Check and validate pipeline parameters +// +def validateInputParameters() { + genomeExistsError() +} + +// +// Get attribute from genome config file e.g. fasta +// +def getGenomeAttribute(attribute) { + if (params.genomes && params.genome && params.genomes.containsKey(params.genome)) { + if (params.genomes[ params.genome ].containsKey(attribute)) { + return params.genomes[ params.genome ][ attribute ] + } + } + return null +} + +// +// Exit pipeline if incorrect --genome key provided +// +def genomeExistsError() { + if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { + def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + + " Currently, the available genome keys are:\n" + + " ${params.genomes.keySet().join(", ")}\n" + + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + error(error_string) + } +} \ No newline at end of file From 3f86b93cd7a903e32552fc1cf181f608a4c5fa4a Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 8 Feb 2024 12:47:42 +0100 Subject: [PATCH 053/178] one more --- .github/actions/create-lint-wf/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index aa711b73c2..d89c40d044 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -31,6 +31,7 @@ runs: # Try syncing it before we change anything - name: nf-core sync + shell: bash run: nf-core --log-file log.txt sync --dir nf-core-testpipeline/ working-directory: create-lint-wf From 090a395097cc233dff30a82f8ea726cb1a417219 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Feb 2024 12:47:59 +0100 Subject: [PATCH 054/178] Update main workflows in pipeline to use utils_* subworkflows --- nf_core/pipeline-template/main.nf | 95 ++++++++----- .../pipeline-template/workflows/pipeline.nf | 127 ++++-------------- 2 files changed, 89 insertions(+), 133 deletions(-) diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 78da158856..2fb327b221 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -12,7 +12,19 @@ */ nextflow.enable.dsl = 2 + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +include { {{ short_name|upper }} } from './workflows/{{ short_name }}' +include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_pipeline_pipeline' +include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_pipeline_pipeline' {% if igenomes %} +include { getGenomeAttribute } from './subworkflows/local/utils_nfcore_pipeline_pipeline' + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ GENOME PARAMETER VALUES @@ -22,59 +34,72 @@ nextflow.enable.dsl = 2 // TODO nf-core: Remove this line if you don't need a FASTA file // This is an example of how to use getGenomeAttribute() to fetch parameters // from igenomes.config using `--genome` -params.fasta = WorkflowMain.getGenomeAttribute(params, 'fasta') +params.fasta = getGenomeAttribute('fasta') {% endif %} /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - VALIDATE & PRINT PARAMETER SUMMARY + NAMED WORKFLOWS FOR PIPELINE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { validateParameters; paramsHelp } from 'plugin/nf-validation' +// +// WORKFLOW: Run main analysis pipeline depending on type of input +// +workflow {{ prefix_nodash|upper }}_{{ short_name|upper }} { -// Print help message if needed -if (params.help) { - def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) - def citation = '\n' + WorkflowMain.citation(workflow) + '\n' - def String command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" - log.info logo + paramsHelp(command) + citation + NfcoreTemplate.dashedLine(params.monochrome_logs) - System.exit(0) -} + take: + samplesheet // channel: samplesheet read in from --input -// Validate input parameters -if (params.validate_params) { - validateParameters() -} + main: -WorkflowMain.initialise(workflow, params, log, args) + // + // WORKFLOW: Run pipeline + // + {{ short_name|upper }} ( + samplesheet + ) +} /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - NAMED WORKFLOW FOR PIPELINE + RUN MAIN WORKFLOW ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { {{ short_name|upper }} } from './workflows/{{ short_name }}' +workflow { -// -// WORKFLOW: Run main {{ name }} analysis pipeline -// -workflow {{ prefix_nodash|upper }}_{{ short_name|upper }} { - {{ short_name|upper }} () -} + main: -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - RUN ALL WORKFLOWS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ + // + // SUBWORKFLOW: Run initialisation tasks + // + PIPELINE_INITIALISATION ( + params.version, + params.help, + params.validate_params, + params.monochrome_logs, + params.outdir, + params.input + ) -// -// WORKFLOW: Execute a single named workflow for the pipeline -// See: https://github.com/nf-core/rnaseq/issues/619 -// -workflow { - {{ prefix_nodash|upper }}_{{ short_name|upper }} () + // + // WORKFLOW: Run main workflow + // + {{ prefix_nodash|upper }}_{{ short_name|upper }} ( + PIPELINE_INITIALISATION.out.samplesheet + ) + + // + // SUBWORKFLOW: Run completion tasks + // + PIPELINE_COMPLETION ( + params.email, + params.email_on_fail, + params.plaintext_email, + params.outdir, + params.monochrome_logs, + params.hook_url + ) } /* diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 4583f2a9d6..85c9dd22a1 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -1,54 +1,14 @@ /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - PRINT PARAMS SUMMARY + IMPORT MODULES / SUBWORKFLOWS / FUNCTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' - -def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) -def citation = '\n' + WorkflowMain.citation(workflow) + '\n' -def summary_params = paramsSummaryMap(workflow) - -// Print parameter summary log to screen -log.info logo + paramsSummaryLog(workflow) + citation - -Workflow{{ short_name[0]|upper }}{{ short_name[1:] }}.initialise(params, log) - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - CONFIG FILES -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() -ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() -ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - IMPORT LOCAL MODULES/SUBWORKFLOWS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -// -// SUBWORKFLOW: Consisting of a mix of local and nf-core/modules -// -include { INPUT_CHECK } from '../subworkflows/local/input_check' - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - IMPORT NF-CORE MODULES/SUBWORKFLOWS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -// -// MODULE: Installed directly from nf-core/modules -// -include { FASTQC } from '../modules/nf-core/fastqc/main' -include { MULTIQC } from '../modules/nf-core/multiqc/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' +include { FASTQC } from '../modules/nf-core/fastqc/main' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { paramsSummaryMap } from 'plugin/nf-validation' +include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -56,50 +16,42 @@ include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoft ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -// Info required for completion email and summary -def multiqc_report = [] - workflow {{ short_name|upper }} { - ch_versions = Channel.empty() + take: + ch_samplesheet // channel: samplesheet read in from --input - // - // SUBWORKFLOW: Read in samplesheet, validate and stage input files - // - INPUT_CHECK ( - file(params.input) - ) - ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) - // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the samplesheet with Channel.fromSamplesheet("input") - // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ - // ! There is currently no tooling to help you write a sample sheet schema + main: + + ch_versions = Channel.empty() + ch_multiqc_files = Channel.empty() // // MODULE: Run FastQC // FASTQC ( - INPUT_CHECK.out.reads + ch_samplesheet ) + ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}) ch_versions = ch_versions.mix(FASTQC.out.versions.first()) - CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique().collectFile(name: 'collated_versions.yml') - ) + // + // Collate and save software versions + // + softwareVersionsToYAML(ch_versions) + .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) + .set { ch_collated_versions } // // MODULE: MultiQC // - workflow_summary = Workflow{{ short_name[0]|upper }}{{ short_name[1:] }}.paramsSummaryMultiqc(workflow, summary_params) - ch_workflow_summary = Channel.value(workflow_summary) - - methods_description = Workflow{{ short_name[0]|upper }}{{ short_name[1:] }}.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) - ch_methods_description = Channel.value(methods_description) - - ch_multiqc_files = Channel.empty() + ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() + ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) - ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) MULTIQC ( ch_multiqc_files.collect(), @@ -107,31 +59,10 @@ workflow {{ short_name|upper }} { ch_multiqc_custom_config.toList(), ch_multiqc_logo.toList() ) - multiqc_report = MULTIQC.out.report.toList() -} - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - COMPLETION EMAIL AND SUMMARY -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -workflow.onComplete { - if (params.email || params.email_on_fail) { - NfcoreTemplate.email(workflow, params, summary_params, projectDir, log, multiqc_report) - } - NfcoreTemplate.dump_parameters(workflow, params) - NfcoreTemplate.summary(workflow, params, log) - if (params.hook_url) { - NfcoreTemplate.IM_notification(workflow, params, summary_params, projectDir, log) - } -} -workflow.onError { - if (workflow.errorReport.contains("Process requirement exceeds available memory")) { - println("🛑 Default resources exceed availability 🛑 ") - println("💡 See here on how to configure pipeline: https://nf-co.re/docs/usage/configuration#tuning-workflow-resources 💡") - } + emit: + multiqc_report = MULTIQC.out.report // channel: /path/to/multiqc_report.html + versions = ch_versions // channel: [ path(versions.yml) ] } /* From 7a1e4bcd5403c7201682a5f0782549ab34c4406e Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 8 Feb 2024 14:00:05 +0100 Subject: [PATCH 055/178] debug workflow --- .github/workflows/create-lint-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index d9d0437b08..e69229c051 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -67,7 +67,7 @@ jobs: # Build a module from the template - name: nf-core modules create - run: nf-core --log-file log.txt modules create bpipe --dir nf-core-testpipeline --author @nf-core-bot --label process_low --meta + run: nf-core --verbose --log-file log.txt modules create bpipe --dir nf-core-testpipeline --author @nf-core-bot --label process_low --meta working-directory: create-lint-wf # Remove TODO statements From a0fe75901423d3d0c0e7f880065a98e6ea04205b Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 8 Feb 2024 14:21:08 +0100 Subject: [PATCH 056/178] add shell --- .github/workflows/create-lint-wf.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index e69229c051..0a78428c9a 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -67,6 +67,7 @@ jobs: # Build a module from the template - name: nf-core modules create + shell: bash run: nf-core --verbose --log-file log.txt modules create bpipe --dir nf-core-testpipeline --author @nf-core-bot --label process_low --meta working-directory: create-lint-wf From b7e693550589e55e80fbf84a90dd85ae60630e78 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 8 Feb 2024 14:24:06 +0100 Subject: [PATCH 057/178] remove clean up action --- .github/actions/create-lint-wf/action.yml | 5 ----- .github/workflows/create-lint-wf.yml | 1 - 2 files changed, 6 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index d89c40d044..7eb3995599 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -106,8 +106,3 @@ runs: shell: bash run: nf-core --log-file log.txt modules --git-remote https://gitlab.com/nf-core/modules-test.git list remote working-directory: create-lint-wf - - - name: Cleanup work directory - shell: bash - run: sudo rm -rf create-lint-wf - if: always() diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 0a78428c9a..e69229c051 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -67,7 +67,6 @@ jobs: # Build a module from the template - name: nf-core modules create - shell: bash run: nf-core --verbose --log-file log.txt modules create bpipe --dir nf-core-testpipeline --author @nf-core-bot --label process_low --meta working-directory: create-lint-wf From 31f7eb3417648828cb44671b6527042b9acb53a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 8 Feb 2024 14:34:39 +0100 Subject: [PATCH 058/178] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- .github/workflows/update_components_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index 9b14bdce98..e331ae0226 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -22,11 +22,11 @@ jobs: run: pip install nf-core - name: Update modules - run: nf-core modules update -all + run: nf-core modules update --all working-directory: nf-core/pipeline-template - name: Update subworkflows - run: nf-core subworkflows update -all + run: nf-core subworkflows update --all working-directory: nf-core/pipeline-template # Commit the changes From 41249437af110f7c8e7e82fc1b13b63bd85443f6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 13:35:35 +0000 Subject: [PATCH 059/178] Update actions/setup-python action to v5 --- .github/workflows/update_components_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index e331ae0226..8617c8b650 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.x" From 7b4720a3acbb0aa4e83815232be1e7906b7017f3 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 8 Feb 2024 13:36:47 +0000 Subject: [PATCH 060/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4218425e5d..b05f284ebb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - update ruff to 0.2.0 and add it to pre-commit step ([#2725](https://github.com/nf-core/tools/pull/2725)) - Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.2.1 ([#2730](https://github.com/nf-core/tools/pull/2730)) +- Update actions/setup-python action to v5 ([#2739](https://github.com/nf-core/tools/pull/2739)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From b6c39658283a25920c280439ee8cbdec95956f94 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 8 Feb 2024 14:36:59 +0100 Subject: [PATCH 061/178] only paths or paths-ignore is allowed --- .github/workflows/create-test-lint-wf-template.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 6aae0596fe..818bb810d8 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -3,9 +3,6 @@ on: push: branches: - dev - paths-ignore: - - "docs/**" - - "CHANGELOG.md" paths: - nf_core/pipeline-template/** pull_request: From fdf3e547c10405cd21aa66a1e1ff0c8663fd6f70 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Feb 2024 14:55:35 +0100 Subject: [PATCH 062/178] Disable native version detection in MultiQC --- nf_core/pipeline-template/assets/multiqc_config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/assets/multiqc_config.yml b/nf_core/pipeline-template/assets/multiqc_config.yml index 39943ffe49..b13b7ae074 100644 --- a/nf_core/pipeline-template/assets/multiqc_config.yml +++ b/nf_core/pipeline-template/assets/multiqc_config.yml @@ -17,3 +17,5 @@ report_section_order: order: -1002 export_plots: true + +disable_version_detection: true From 63e67fc8e43b4ffc2ed8e65c8081083c06bc88ff Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 8 Feb 2024 15:14:21 +0100 Subject: [PATCH 063/178] fix jinja rendering of download_pipeline.yml --- .../pipeline-template/.github/workflows/download_pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 08fc30c936..ba6ea3f517 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -47,13 +47,13 @@ jobs: run: | echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} - echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} + echo "{% raw %}REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} - name: Download the pipeline env: NXF_SINGULARITY_CACHEDIR: ./ run: | - nf-core download {% raw %} ${{ env.REPO_LOWERCASE }} \ + nf-core download ${{ env.REPO_LOWERCASE }} \ --revision ${{ env.REPO_BRANCH }} \ --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ --compress "none" \ From a8f5274b47904d1e9fa4b0741163c3f7865d10c8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 14:14:34 +0000 Subject: [PATCH 064/178] Update actions/checkout action to v4 --- .github/workflows/update_components_template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index 8617c8b650..f357bed03b 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 From 1ad52e4a5ebd225773b9a3ed122b4f56b9621f63 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 14:15:04 +0000 Subject: [PATCH 065/178] Pin dependencies --- .github/actions/create-lint-wf/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 7eb3995599..3b63f198cc 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -17,7 +17,7 @@ runs: # Set up Nextflow - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 with: version: ${{ matrix.NXF_VER }} @@ -77,7 +77,7 @@ runs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-lint-wf/log.txt From 52149765512cc0a5c2c749f82fc1656536a4e4d4 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Feb 2024 16:13:54 +0100 Subject: [PATCH 066/178] Add checkProfileProvided function back in --- nf_core/pipeline-template/main.nf | 1 + .../utils_nfcore_pipeline_pipeline/main.nf | 17 ++++++++++------- .../nf-core/utils_nfcore_pipeline/main.nf | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 2fb327b221..c56dc5d443 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -78,6 +78,7 @@ workflow { params.help, params.validate_params, params.monochrome_logs, + args, params.outdir, params.input ) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 945d270741..d33aae5330 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -29,12 +29,13 @@ include { INPUT_CHECK } from '../../local/input_check' workflow PIPELINE_INITIALISATION { take: - version // boolean: Display version and exit - help // boolean: Display help text - validate_params // boolean: Boolean whether to validate parameters against the schema at runtime - monochrome_logs // boolean: Do not use coloured log outputs - outdir // string: The output directory where the results will be saved - input // string: Path to input samplesheet + version // boolean: Display version and exit + help // boolean: Display help text + validate_params // boolean: Boolean whether to validate parameters against the schema at runtime + monochrome_logs // boolean: Do not use coloured log outputs + nextflow_cli_args // array: List of positional nextflow CLI args + outdir // string: The output directory where the results will be saved + input // string: Path to input samplesheet main: @@ -68,7 +69,9 @@ workflow PIPELINE_INITIALISATION { // // Check config provided to the pipeline // - UTILS_NFCORE_PIPELINE () + UTILS_NFCORE_PIPELINE ( + nextflow_cli_args + ) // // Custom validation for pipeline parameters diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 6d805bb3fd..c1aecf6e42 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -13,8 +13,12 @@ import nextflow.extension.FilesEx workflow UTILS_NFCORE_PIPELINE { + take: + nextflow_cli_args + main: valid_config = checkConfigProvided() + checkProfileProvided(nextflow_cli_args) emit: valid_config @@ -43,6 +47,20 @@ def checkConfigProvided() { return valid_config } +// +// Exit pipeline if --profile contains spaces +// +def checkProfileProvided(nextflow_cli_args) { + if (workflow.profile.endsWith(',')) { + error "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + } + if (nextflow_cli_args[0]) { + log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + } +} + // // Citation string for pipeline // From a52503b5c3594c19bd8674aca79d82627f546c96 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 9 Feb 2024 07:25:11 +0000 Subject: [PATCH 067/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cdb73d30b..14dbe6fd62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - update ruff to 0.2.0 and add it to pre-commit step ([#2725](https://github.com/nf-core/tools/pull/2725)) - Update codecov/codecov-action digest to e0b68c6 ([#2728](https://github.com/nf-core/tools/pull/2728)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.2.1 ([#2730](https://github.com/nf-core/tools/pull/2730)) +- Update python:3.11-slim Docker digest to 2a746e2 ([#2743](https://github.com/nf-core/tools/pull/2743)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From d646ae6c0913c5aa99b480cd05838cbde226d830 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 9 Feb 2024 14:05:42 +0100 Subject: [PATCH 068/178] linting: add looser comparison when linting pipeline logos --- nf_core/lint/files_unchanged.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 7f63973f16..dcbac75152 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -1,5 +1,6 @@ import filecmp import logging +import os import shutil import tempfile from pathlib import Path @@ -162,7 +163,15 @@ def _tf(file_path: Union[str, Path]) -> Path: if filecmp.cmp(_pf(f), _tf(f), shallow=True): passed.append(f"`{f}` matches the template") else: - if "files_unchanged" in self.fix: + if ( + f.name.endswith(".png") + and os.stat(_pf(f)).st_mode == os.stat(_tf(f)).st_mode + and int(os.stat(_pf(f)).st_size / 100) == int(os.stat(_tf(f)).st_size / 100) + ): + # almost the same file, good enough for the logo + log.debug(f"Files are almost the same. Will pass: {f}") + passed.append(f"`{f}` matches the template") + elif "files_unchanged" in self.fix: # Try to fix the problem by overwriting the pipeline file shutil.copy(_tf(f), _pf(f)) passed.append(f"`{f}` matches the template") From 8ca34cb85d048b198827c7453a18ac11e492778e Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 9 Feb 2024 13:07:59 +0000 Subject: [PATCH 069/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fecdc74e01..fc6dcb47eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Linting - make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733)) +- add looser comparison when pipeline logos ([#2744](https://github.com/nf-core/tools/pull/2744)) ### Modules From 201562ca25f06ec0f167ec9ba0074ea02aa444e5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 9 Feb 2024 18:16:41 +0000 Subject: [PATCH 070/178] Update nf-core/setup-nextflow digest to b9f764e --- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/pytest.yml | 2 +- .github/workflows/rich-codex.yml | 2 +- .github/workflows/sync.yml | 2 +- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- .../pipeline-template/.github/workflows/download_pipeline.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 818bb810d8..6f27236462 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -72,7 +72,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 with: version: latest-everything diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index de2a52fa22..0166931be7 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -59,7 +59,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 with: version: ${{ matrix.NXF_VER }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f661911f55..5351bb985c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -113,7 +113,7 @@ jobs: run: echo "date=$(date +'%Y-%m')" >> $GITHUB_ENV - name: Install Nextflow - uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 - name: Look if nf-test is already installed and write to env variable id: check-nftest diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 73719f2630..0e8a1c5e9b 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -15,7 +15,7 @@ jobs: cache-dependency-path: setup.py - name: Install Nextflow - uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 - name: Cache nf-test installation id: cache-software diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 2bee312b16..d89e255bfb 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -67,7 +67,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 with: version: "latest-everything" diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index bd3435120e..1290bfafc0 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 with: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index ba6ea3f517..20b811ab2b 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 232789b8d4..3ec259b5ed 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@dccbf847604cbc34610280d79644a02035d924ac # v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: From 7832a441f3fda016d55f989095f9aae11d4ff4a8 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 9 Feb 2024 20:02:31 +0100 Subject: [PATCH 071/178] update utils_nfcore_pipeline in the template --- nf_core/pipeline-template/conf/test.config | 2 +- nf_core/pipeline-template/modules.json | 24 ++++++------------- .../nf-core/utils_nfcore_pipeline/meta.yml | 6 ++++- .../tests/main.function.nf.test | 20 ++++++++++++++++ .../tests/main.function.nf.test.snap | 4 ++++ .../tests/main.workflow.nf.test | 8 +++++++ 6 files changed, 45 insertions(+), 19 deletions(-) diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 499c72054b..32b9619ebb 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -24,7 +24,7 @@ params { // TODO nf-core: Give any required params for the test so that command line flags are not needed input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' - {% if igenomes %} + {% if igenomes -%} // Genome references genome = 'R64-1-1' {%- endif %} diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index c558969239..d1e54dbbf7 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -8,16 +8,12 @@ "fastqc": { "branch": "master", "git_sha": "c9488585ce7bd35ccd2a30faa2371454c8112fb9", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "multiqc": { "branch": "master", "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] } } }, @@ -26,26 +22,20 @@ "utils_nextflow_pipeline": { "branch": "master", "git_sha": "cd08c91373cd00a73255081340e4914485846ba1", - "installed_by": [ - "subworkflows" - ] + "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "73b63c8e510107010ab13b68cf31189d6cdca8cb", - "installed_by": [ - "subworkflows" - ] + "git_sha": "db321a792c88d145afa0ae214fc7da70fcd7d591", + "installed_by": ["subworkflows"] }, "utils_nfvalidation_plugin": { "branch": "master", "git_sha": "cd08c91373cd00a73255081340e4914485846ba1", - "installed_by": [ - "subworkflows" - ] + "installed_by": ["subworkflows"] } } } } } -} \ No newline at end of file +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml index dd1462b232..d08d24342d 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml @@ -7,7 +7,11 @@ keywords: - initialise - version components: [] -input: [] +input: + - nextflow_cli_args: + type: list + description: | + Nextflow CLI positional arguments output: - success: type: boolean diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test index f7ea05fcd7..1dc317f8f7 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -21,6 +21,26 @@ nextflow_function { } } + test("Test Function checkProfileProvided") { + + function "checkProfileProvided" + + when { + function { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + test("Test Function workflowCitation") { function "workflowCitation" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap index afb9ab4dca..10f948e629 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -1,4 +1,8 @@ { + "Test Function checkProfileProvided": { + "content": null, + "timestamp": "2024-02-09T15:43:55.145717" + }, "Test Function checkConfigProvided": { "content": [ true diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test index c5f7776a67..8940d32d1e 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test @@ -11,6 +11,14 @@ nextflow_workflow { test("Should run without failures") { + when { + workflow { + """ + input[0] = [] + """ + } + } + then { assertAll( { assert workflow.success }, From 91b15295719cded218565aef1fbfa939c90ca736 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Sat, 10 Feb 2024 10:57:53 +0100 Subject: [PATCH 072/178] add multiqc report to completion email --- nf_core/pipeline-template/main.nf | 10 +++++--- .../utils_nfcore_pipeline_pipeline/main.nf | 7 +++--- .../nf-core/utils_nfcore_pipeline/main.nf | 23 +++++++++++++++++-- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index c56dc5d443..c31927f236 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -55,10 +55,13 @@ workflow {{ prefix_nodash|upper }}_{{ short_name|upper }} { // // WORKFLOW: Run pipeline // - {{ short_name|upper }} ( - samplesheet + {{ short_name|upper }} ( + samplesheet ) + emit: + multiqc_report = {{ short_name|upper }}.out.multiqc_report // channel: /path/to/multiqc_report.html + } /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -99,7 +102,8 @@ workflow { params.plaintext_email, params.outdir, params.monochrome_logs, - params.hook_url + params.hook_url, + {{ prefix_nodash|upper }}_{{ short_name|upper }}.out.multiqc_report ) } diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index d33aae5330..b1046ed9db 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -40,7 +40,7 @@ workflow PIPELINE_INITIALISATION { main: ch_versions = Channel.empty() - + // // Print version and exit if required and dump pipeline parameters to JSON file // @@ -109,6 +109,7 @@ workflow PIPELINE_COMPLETION { outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output hook_url // string: hook URL for notifications + multiqc_report // string: Path to MultiQC report main: @@ -119,7 +120,7 @@ workflow PIPELINE_COMPLETION { // workflow.onComplete { if (email || email_on_fail) { - completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs) + completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs, multiqc_report.toList()) } completionSummary(monochrome_logs) @@ -177,4 +178,4 @@ def genomeExistsError() { "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" error(error_string) } -} \ No newline at end of file +} diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index c1aecf6e42..54769d5b58 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -249,7 +249,7 @@ def logColours(monochrome_logs=true) { // // Construct and send completion email // -def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true) { +def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=[]) { // Set up the e-mail variables def subject = "[$workflow.manifest.name] Successful: $workflow.runName" @@ -287,6 +287,24 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi email_fields['projectDir'] = workflow.projectDir email_fields['summary'] = summary << misc_fields + // On success try attach the multiqc report + def mqc_report = null + try { + if (workflow.success) { + mqc_report = multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { + if (mqc_report.size() > 1) { + log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" + } + mqc_report = mqc_report[0] + } + } + } catch (all) { + if (multiqc_report) { + log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + } + } + // Check if we are only sending emails on failure def email_address = email if (!email && email_on_fail && !workflow.success) { @@ -305,7 +323,8 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi def email_html = html_template.toString() // Render the sendmail template - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}" ] + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) def sendmail_html = sendmail_template.toString() From 48c7026f18c92d267998341ba952eba9c8d58f08 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Feb 2024 13:50:54 +0100 Subject: [PATCH 073/178] add methodsDescriptionText to multiqc report --- .../utils_nfcore_pipeline_pipeline/main.nf | 56 +++++++++++++++++++ .../pipeline-template/workflows/pipeline.nf | 18 +++--- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index b1046ed9db..3a76444eef 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -179,3 +179,59 @@ def genomeExistsError() { error(error_string) } } + +// +// Generate methods description for MultiQC +// +def toolCitationText() { + // TODO nf-core: Optionally add in-text citation tools to this list. + // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "Tool (Foo et al. 2023)" : "", + // Uncomment function in methodsDescriptionText to render in MultiQC report + def citation_text = [ + "Tools used in the workflow included:", + "FastQC (Andrews 2010),", + "MultiQC (Ewels et al. 2016)", + "." + ].join(' ').trim() + + return citation_text +} + +def toolBibliographyText() { + // TODO nf-core: Optionally add bibliographic entries to this list. + // Can use ternary operators to dynamically construct based conditions, e.g. params["run_xyz"] ? "
  • Author (2023) Pub name, Journal, DOI
    • " : "", + // Uncomment function in methodsDescriptionText to render in MultiQC report + def reference_text = [ + "
  • Andrews S, (2010) FastQC, URL: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).
    • ", + "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: /10.1093/bioinformatics/btw354
    • " + ].join(' ').trim() + + return reference_text +} + +def methodsDescriptionText(mqc_methods_yaml) { + // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file + def meta = [:] + meta.workflow = workflow.toMap() + meta["manifest_map"] = workflow.manifest.toMap() + + // Pipeline DOI + meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" + meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + + // Tool references + meta["tool_citations"] = "" + meta["tool_bibliography"] = "" + + // TODO nf-core: Only uncomment below if logic in toolCitationText/toolBibliographyText has been filled! + meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") + meta["tool_bibliography"] = toolBibliographyText() + + + def methods_text = mqc_methods_yaml.text + + def engine = new groovy.text.SimpleTemplateEngine() + def description_html = engine.createTemplate(methods_text).make(meta) + + return description_html +} diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 85c9dd22a1..30b0085514 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -9,6 +9,7 @@ include { MULTIQC } from '../modules/nf-core/multiqc/main' include { paramsSummaryMap } from 'plugin/nf-validation' include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_pipeline_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -45,13 +46,16 @@ workflow {{ short_name|upper }} { // // MODULE: MultiQC // - ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) - ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() - ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") - ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) - ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() + ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) + ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) MULTIQC ( ch_multiqc_files.collect(), From bbf3fac4d9f60931096e8c978eb8939b0f7a0bfc Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Feb 2024 15:44:39 +0100 Subject: [PATCH 074/178] update template subworkflows --- nf_core/pipeline-template/modules.json | 2 +- .../nf-core/utils_nfcore_pipeline/main.nf | 42 +++++++++++-------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index d1e54dbbf7..96cb7942e9 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -26,7 +26,7 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "db321a792c88d145afa0ae214fc7da70fcd7d591", + "git_sha": "262b17ed2aad591039f914951659177e6c39a8d8", "installed_by": ["subworkflows"] }, "utils_nfvalidation_plugin": { diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index 54769d5b58..a8b55d6fe1 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -246,10 +246,33 @@ def logColours(monochrome_logs=true) { return colorcodes } +// +// Attach the multiqc report to email +// +def attachMultiqcReport(multiqc_report) { + def mqc_report = null + try { + if (workflow.success) { + mqc_report = multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { + if (mqc_report.size() > 1) { + log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" + } + mqc_report = mqc_report[0] + } + } + } catch (all) { + if (multiqc_report) { + log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + } + } + return mqc_report +} + // // Construct and send completion email // -def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=[]) { +def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=null) { // Set up the e-mail variables def subject = "[$workflow.manifest.name] Successful: $workflow.runName" @@ -288,22 +311,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi email_fields['summary'] = summary << misc_fields // On success try attach the multiqc report - def mqc_report = null - try { - if (workflow.success) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" - } - mqc_report = mqc_report[0] - } - } - } catch (all) { - if (multiqc_report) { - log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" - } - } + def mqc_report = attachMultiqcReport(multiqc_report) // Check if we are only sending emails on failure def email_address = email From 125cc59f1e2f84182773ca2158479047487dff03 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Feb 2024 17:40:42 +0100 Subject: [PATCH 075/178] remove warn exceeding resources --- .../local/utils_nfcore_pipeline_pipeline/main.nf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 3a76444eef..8f71247cb4 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -129,16 +129,6 @@ workflow PIPELINE_COMPLETION { imNotification(summary_params, hook_url) } } - - // - // Log warning if exceeding resource requests - // - workflow.onError { - if (workflow.errorReport.contains("Process requirement exceeds available memory")) { - log.warning "🛑 Default resources exceed availability 🛑 " - log.warning "💡 See here on how to configure pipeline: https://nf-co.re/docs/usage/configuration#tuning-workflow-resources 💡" - } - } } /* From c64ed821225b974fc90006f595ea74d11418ad44 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Feb 2024 18:40:41 +0100 Subject: [PATCH 076/178] fix SimpleTemplateEngine error by writing to the file --- .../local/utils_nfcore_pipeline_pipeline/main.nf | 4 ++-- nf_core/pipeline-template/workflows/pipeline.nf | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 8f71247cb4..488306e538 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -214,8 +214,8 @@ def methodsDescriptionText(mqc_methods_yaml) { meta["tool_bibliography"] = "" // TODO nf-core: Only uncomment below if logic in toolCitationText/toolBibliographyText has been filled! - meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") - meta["tool_bibliography"] = toolBibliographyText() + // meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") + // meta["tool_bibliography"] = toolBibliographyText() def methods_text = mqc_methods_yaml.text diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 30b0085514..6c016168c5 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -53,9 +53,13 @@ workflow {{ short_name|upper }} { ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) + methods_description_file = new File('methods_description_mqc.yaml') + ch_methods_description.map{ + methods_description_file << it + } ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) - ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(Channel.fromPath(methods_description_file, checkIfExists: true)) MULTIQC ( ch_multiqc_files.collect(), From 77aa1cbf9fb65feed9a4665f18ef57f01626853a Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 13 Feb 2024 10:37:54 +0100 Subject: [PATCH 077/178] check if we are in a modules repo before we force checkout --- nf_core/synced_repo.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index c668be28d2..5c31e96911 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -211,7 +211,11 @@ def checkout_branch(self): try: self.repo.git.checkout(self.branch) except GitCommandError as e: - if "Your local changes to the following files would be overwritten by checkout" in str(e): + if ( + self.fullname + and "modules" in self.fullname + and "Your local changes to the following files would be overwritten by checkout" in str(e) + ): log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") self.repo.git.checkout(self.branch, force=True) else: @@ -227,7 +231,11 @@ def checkout(self, commit): try: self.repo.git.checkout(commit) except GitCommandError as e: - if "Your local changes to the following files would be overwritten by checkout" in str(e): + if ( + self.fullname + and "modules" in self.fullname + and "Your local changes to the following files would be overwritten by checkout" in str(e) + ): log.debug(f"Overwriting local changes in '{self.local_repo_dir}'") self.repo.git.checkout(self.branch, force=True) else: From e56999f3a8928188d4dbcd9f2b493322c4830372 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 13 Feb 2024 11:32:53 +0100 Subject: [PATCH 078/178] use nf-validation to create input channel --- .../assets/schema_input.json | 21 +- .../bin/check_samplesheet.py | 259 ------------------ nf_core/pipeline-template/conf/modules.config | 8 - .../modules/local/samplesheet_check.nf | 31 --- .../pipeline-template/nextflow_schema.json | 1 + .../subworkflows/local/input_check.nf | 44 --- .../utils_nfcore_pipeline_pipeline/main.nf | 48 +++- 7 files changed, 48 insertions(+), 364 deletions(-) delete mode 100755 nf_core/pipeline-template/bin/check_samplesheet.py delete mode 100644 nf_core/pipeline-template/modules/local/samplesheet_check.nf delete mode 100644 nf_core/pipeline-template/subworkflows/local/input_check.nf diff --git a/nf_core/pipeline-template/assets/schema_input.json b/nf_core/pipeline-template/assets/schema_input.json index 509048bd8a..e76b95fa99 100644 --- a/nf_core/pipeline-template/assets/schema_input.json +++ b/nf_core/pipeline-template/assets/schema_input.json @@ -10,25 +10,22 @@ "sample": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Sample name must be provided and cannot contain spaces" + "errorMessage": "Sample name must be provided and cannot contain spaces", + "meta": ["id"] }, "fastq_1": { "type": "string", + "format": "file-path", + "exists": true, "pattern": "^\\S+\\.f(ast)?q\\.gz$", "errorMessage": "FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" }, "fastq_2": { - "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", - "anyOf": [ - { - "type": "string", - "pattern": "^\\S+\\.f(ast)?q\\.gz$" - }, - { - "type": "string", - "maxLength": 0 - } - ] + "type": "string", + "format": "file-path", + "exists": true, + "pattern": "^\\S+\\.f(ast)?q\\.gz$", + "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" } }, "required": ["sample", "fastq_1"] diff --git a/nf_core/pipeline-template/bin/check_samplesheet.py b/nf_core/pipeline-template/bin/check_samplesheet.py deleted file mode 100755 index 4a758fe003..0000000000 --- a/nf_core/pipeline-template/bin/check_samplesheet.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python - - -"""Provide a command line tool to validate and transform tabular samplesheets.""" - - -import argparse -import csv -import logging -import sys -from collections import Counter -from pathlib import Path - -logger = logging.getLogger() - - -class RowChecker: - """ - Define a service that can validate and transform each given row. - - Attributes: - modified (list): A list of dicts, where each dict corresponds to a previously - validated and transformed row. The order of rows is maintained. - - """ - - VALID_FORMATS = ( - ".fq.gz", - ".fastq.gz", - ) - - def __init__( - self, - sample_col="sample", - first_col="fastq_1", - second_col="fastq_2", - single_col="single_end", - **kwargs, - ): - """ - Initialize the row checker with the expected column names. - - Args: - sample_col (str): The name of the column that contains the sample name - (default "sample"). - first_col (str): The name of the column that contains the first (or only) - FASTQ file path (default "fastq_1"). - second_col (str): The name of the column that contains the second (if any) - FASTQ file path (default "fastq_2"). - single_col (str): The name of the new column that will be inserted and - records whether the sample contains single- or paired-end sequencing - reads (default "single_end"). - - """ - super().__init__(**kwargs) - self._sample_col = sample_col - self._first_col = first_col - self._second_col = second_col - self._single_col = single_col - self._seen = set() - self.modified = [] - - def validate_and_transform(self, row): - """ - Perform all validations on the given row and insert the read pairing status. - - Args: - row (dict): A mapping from column headers (keys) to elements of that row - (values). - - """ - self._validate_sample(row) - self._validate_first(row) - self._validate_second(row) - self._validate_pair(row) - self._seen.add((row[self._sample_col], row[self._first_col])) - self.modified.append(row) - - def _validate_sample(self, row): - """Assert that the sample name exists and convert spaces to underscores.""" - if len(row[self._sample_col]) <= 0: - raise AssertionError("Sample input is required.") - # Sanitize samples slightly. - row[self._sample_col] = row[self._sample_col].replace(" ", "_") - - def _validate_first(self, row): - """Assert that the first FASTQ entry is non-empty and has the right format.""" - if len(row[self._first_col]) <= 0: - raise AssertionError("At least the first FASTQ file is required.") - self._validate_fastq_format(row[self._first_col]) - - def _validate_second(self, row): - """Assert that the second FASTQ entry has the right format if it exists.""" - if len(row[self._second_col]) > 0: - self._validate_fastq_format(row[self._second_col]) - - def _validate_pair(self, row): - """Assert that read pairs have the same file extension. Report pair status.""" - if row[self._first_col] and row[self._second_col]: - row[self._single_col] = False - first_col_suffix = Path(row[self._first_col]).suffixes[-2:] - second_col_suffix = Path(row[self._second_col]).suffixes[-2:] - if first_col_suffix != second_col_suffix: - raise AssertionError("FASTQ pairs must have the same file extensions.") - else: - row[self._single_col] = True - - def _validate_fastq_format(self, filename): - """Assert that a given filename has one of the expected FASTQ extensions.""" - if not any(filename.endswith(extension) for extension in self.VALID_FORMATS): - raise AssertionError( - f"The FASTQ file has an unrecognized extension: {filename}\n" - f"It should be one of: {', '.join(self.VALID_FORMATS)}" - ) - - def validate_unique_samples(self): - """ - Assert that the combination of sample name and FASTQ filename is unique. - - In addition to the validation, also rename all samples to have a suffix of _T{n}, where n is the - number of times the same sample exist, but with different FASTQ files, e.g., multiple runs per experiment. - - """ - if len(self._seen) != len(self.modified): - raise AssertionError("The pair of sample name and FASTQ must be unique.") - seen = Counter() - for row in self.modified: - sample = row[self._sample_col] - seen[sample] += 1 - row[self._sample_col] = f"{sample}_T{seen[sample]}" - - -def read_head(handle, num_lines=10): - """Read the specified number of lines from the current position in the file.""" - lines = [] - for idx, line in enumerate(handle): - if idx == num_lines: - break - lines.append(line) - return "".join(lines) - - -def sniff_format(handle): - """ - Detect the tabular format. - - Args: - handle (text file): A handle to a `text file`_ object. The read position is - expected to be at the beginning (index 0). - - Returns: - csv.Dialect: The detected tabular format. - - .. _text file: - https://docs.python.org/3/glossary.html#term-text-file - - """ - peek = read_head(handle) - handle.seek(0) - sniffer = csv.Sniffer() - dialect = sniffer.sniff(peek) - return dialect - - -def check_samplesheet(file_in, file_out): - """ - Check that the tabular samplesheet has the structure expected by nf-core pipelines. - - Validate the general shape of the table, expected columns, and each row. Also add - an additional column which records whether one or two FASTQ reads were found. - - Args: - file_in (pathlib.Path): The given tabular samplesheet. The format can be either - CSV, TSV, or any other format automatically recognized by ``csv.Sniffer``. - file_out (pathlib.Path): Where the validated and transformed samplesheet should - be created; always in CSV format. - - Example: - This function checks that the samplesheet follows the following structure, - see also the `viral recon samplesheet`_:: - - sample,fastq_1,fastq_2 - SAMPLE_PE,SAMPLE_PE_RUN1_1.fastq.gz,SAMPLE_PE_RUN1_2.fastq.gz - SAMPLE_PE,SAMPLE_PE_RUN2_1.fastq.gz,SAMPLE_PE_RUN2_2.fastq.gz - SAMPLE_SE,SAMPLE_SE_RUN1_1.fastq.gz, - - .. _viral recon samplesheet: - https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv - - """ - required_columns = {"sample", "fastq_1", "fastq_2"} - # See https://docs.python.org/3.9/library/csv.html#id3 to read up on `newline=""`. - with file_in.open(newline="") as in_handle: - reader = csv.DictReader(in_handle, dialect=sniff_format(in_handle)) - # Validate the existence of the expected header columns. - if not required_columns.issubset(reader.fieldnames): - req_cols = ", ".join(required_columns) - logger.critical(f"The sample sheet **must** contain these column headers: {req_cols}.") - sys.exit(1) - # Validate each row. - checker = RowChecker() - for i, row in enumerate(reader): - try: - checker.validate_and_transform(row) - except AssertionError as error: - logger.critical(f"{str(error)} On line {i + 2}.") - sys.exit(1) - checker.validate_unique_samples() - header = list(reader.fieldnames) - header.insert(1, "single_end") - # See https://docs.python.org/3.9/library/csv.html#id3 to read up on `newline=""`. - with file_out.open(mode="w", newline="") as out_handle: - writer = csv.DictWriter(out_handle, header, delimiter=",") - writer.writeheader() - for row in checker.modified: - writer.writerow(row) - - -def parse_args(argv=None): - """Define and immediately parse command line arguments.""" - parser = argparse.ArgumentParser( - description="Validate and transform a tabular samplesheet.", - epilog="Example: python check_samplesheet.py samplesheet.csv samplesheet.valid.csv", - ) - parser.add_argument( - "file_in", - metavar="FILE_IN", - type=Path, - help="Tabular input samplesheet in CSV or TSV format.", - ) - parser.add_argument( - "file_out", - metavar="FILE_OUT", - type=Path, - help="Transformed output samplesheet in CSV format.", - ) - parser.add_argument( - "-l", - "--log-level", - help="The desired log level (default WARNING).", - choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"), - default="WARNING", - ) - return parser.parse_args(argv) - - -def main(argv=None): - """Coordinate argument parsing and program execution.""" - args = parse_args(argv) - logging.basicConfig(level=args.log_level, format="[%(levelname)s] %(message)s") - if not args.file_in.is_file(): - logger.error(f"The given input file {args.file_in} was not found!") - sys.exit(2) - args.file_out.parent.mkdir(parents=True, exist_ok=True) - check_samplesheet(args.file_in, args.file_out) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index d91c6aba0b..e3ea8fa6c4 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -18,14 +18,6 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] - withName: SAMPLESHEET_CHECK { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - withName: FASTQC { ext.args = '--quiet' } diff --git a/nf_core/pipeline-template/modules/local/samplesheet_check.nf b/nf_core/pipeline-template/modules/local/samplesheet_check.nf deleted file mode 100644 index 77be6dfff4..0000000000 --- a/nf_core/pipeline-template/modules/local/samplesheet_check.nf +++ /dev/null @@ -1,31 +0,0 @@ -process SAMPLESHEET_CHECK { - tag "$samplesheet" - label 'process_single' - - conda "conda-forge::python=3.8.3" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/python:3.8.3' : - 'biocontainers/python:3.8.3' }" - - input: - path samplesheet - - output: - path '*.csv' , emit: csv - path "versions.yml", emit: versions - - when: - task.ext.when == null || task.ext.when - - script: // This script is bundled with the pipeline, in {{ name }}/bin/ - """ - check_samplesheet.py \\ - $samplesheet \\ - samplesheet.valid.csv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - python: \$(python --version | sed 's/Python //g') - END_VERSIONS - """ -} diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 080797b4eb..1b603dcf85 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -16,6 +16,7 @@ "type": "string", "format": "file-path", "exists": true, + "schema": "assets/schema_input.json", "mimetype": "text/csv", "pattern": "^\\S+\\.csv$", "description": "Path to comma-separated file containing information about the samples in the experiment.", diff --git a/nf_core/pipeline-template/subworkflows/local/input_check.nf b/nf_core/pipeline-template/subworkflows/local/input_check.nf deleted file mode 100644 index 562f44cc69..0000000000 --- a/nf_core/pipeline-template/subworkflows/local/input_check.nf +++ /dev/null @@ -1,44 +0,0 @@ -// -// Check input samplesheet and get read channels -// - -include { SAMPLESHEET_CHECK } from '../../modules/local/samplesheet_check' - -workflow INPUT_CHECK { - take: - samplesheet // file: /path/to/samplesheet.csv - - main: - SAMPLESHEET_CHECK ( samplesheet ) - .csv - .splitCsv ( header:true, sep:',' ) - .map { create_fastq_channel(it) } - .set { samplesheet } - - emit: - samplesheet // channel: [ val(meta), [ reads ] ] - versions = SAMPLESHEET_CHECK.out.versions // channel: [ versions.yml ] -} - -// Function to get list of [ meta, [ fastq_1, fastq_2 ] ] -def create_fastq_channel(LinkedHashMap row) { - // create meta map - def meta = [:] - meta.id = row.sample - meta.single_end = row.single_end.toBoolean() - - // add path(s) of the fastq file(s) to the meta map - def fastq_meta = [] - if (!file(row.fastq_1).exists()) { - exit 1, "ERROR: Please check input samplesheet -> Read 1 FastQ file does not exist!\n${row.fastq_1}" - } - if (meta.single_end) { - fastq_meta = [ meta, [ file(row.fastq_1) ] ] - } else { - if (!file(row.fastq_2).exists()) { - exit 1, "ERROR: Please check input samplesheet -> Read 2 FastQ file does not exist!\n${row.fastq_2}" - } - fastq_meta = [ meta, [ file(row.fastq_1), file(row.fastq_2) ] ] - } - return fastq_meta -} diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 488306e538..5dd87e8f47 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -10,6 +10,7 @@ include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' include { paramsSummaryMap } from 'plugin/nf-validation' +include { fromSamplesheet } from 'plugin/nf-validation' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' @@ -18,7 +19,6 @@ include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' include { imNotification } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' -include { INPUT_CHECK } from '../../local/input_check' /* ======================================================================================== @@ -79,18 +79,31 @@ workflow PIPELINE_INITIALISATION { validateInputParameters() // - // SUBWORKFLOW: Read in samplesheet, validate and stage input files + // Create channel from input file provided through params.input // - INPUT_CHECK ( - file(input) - ) - ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) - // TODO: OPTIONAL, you can use nf-validation plugin to create an input channel from the samplesheet with Channel.fromSamplesheet("input") - // See the documentation https://nextflow-io.github.io/nf-validation/samplesheets/fromSamplesheet/ - // ! There is currently no tooling to help you write a sample sheet schema + Channel + .fromSamplesheet("input") + .map { + meta, fastq_1, fastq_2 -> + if (!fastq_2) { + return [ meta.id, meta + [ single_end:true ], [ fastq_1 ] ] + } else { + return [ meta.id, meta + [ single_end:false ], [ fastq_1, fastq_2 ] ] + } + } + .groupTuple() + .map { + validateInputSamplesheet(it) + } + .map { + meta, fastqs -> + return [ meta, fastqs.flatten() ] + } + .set { ch_samplesheet } + ch_samplesheet.view() emit: - samplesheet = INPUT_CHECK.out.samplesheet + samplesheet = ch_samplesheet versions = ch_versions } @@ -144,6 +157,21 @@ def validateInputParameters() { genomeExistsError() } +// +// Validate channels from input samplesheet +// +def validateInputSamplesheet(input) { + def (metas, fastqs) = input[1..2] + + // Check that multiple runs of the same sample are of the same datatype i.e. single-end / paired-end + def endedness_ok = metas.collect{ it.single_end }.unique().size == 1 + if (!endedness_ok) { + error("Please check input samplesheet -> Multiple runs of a sample must be of the same datatype i.e. single-end or paired-end: ${metas[0].id}") + } + + return [ metas[0], fastqs ] +} + // // Get attribute from genome config file e.g. fasta // From 75f833f28c19215031bd5e12d66ecb8f251a30ad Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 13 Feb 2024 11:44:55 +0100 Subject: [PATCH 079/178] linting: warn if groovy files in lib exist --- nf_core/lint/files_exist.py | 14 +++++++------- nf_core/lint/files_unchanged.py | 2 -- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index ecd7bc4bbd..ab29c4c39e 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -52,9 +52,6 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: docs/output.md docs/README.md docs/usage.md - lib/NfcoreTemplate.groovy - lib/Utils.groovy - lib/WorkflowMain.groovy nextflow_schema.json nextflow.config README.md @@ -165,9 +162,6 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: [Path("docs", "README.md")], [Path("docs", "README.md")], [Path("docs", "usage.md")], - [Path("lib", "NfcoreTemplate.groovy")], - [Path("lib", "Utils.groovy")], - [Path("lib", "WorkflowMain.groovy")], ] files_warn = [ @@ -200,7 +194,13 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: Path("lib", "Completion.groovy"), Path("lib", "Workflow.groovy"), ] - files_warn_ifexists = [Path(".travis.yml")] + files_warn_ifexists = [ + Path(".travis.yml"), + Path("lib", "NfcoreTemplate.groovy"), + Path("lib", "Utils.groovy"), + Path("lib", "WorkflowMain.groovy"), + Path("lib", "NfcoreTemplate.groovy"), + ] files_fail_ifinconfig: List[Tuple[Path, Dict[str, str]]] = [ (Path("lib", "nfcore_external_java_deps.jar"), {"plugins": "nf-validation"}), ] diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 7f63973f16..3038c8af51 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -40,7 +40,6 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: docs/images/nf-core-PIPELINE_logo_light.png docs/images/nf-core-PIPELINE_logo_dark.png docs/README.md' - lib/NfcoreTemplate.groovy ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling Files that can have additional content but must include the template contents:: @@ -104,7 +103,6 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: [Path("docs", "images", f"nf-core-{short_name}_logo_light.png")], [Path("docs", "images", f"nf-core-{short_name}_logo_dark.png")], [Path("docs", "README.md")], - [Path("lib", "NfcoreTemplate.groovy")], ] files_partial = [ [Path(".gitignore"), Path(".prettierignore"), Path("pyproject.toml")], From a62c307b7e13df2008d20f9247c03b31d59802b3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 13 Feb 2024 15:56:02 +0100 Subject: [PATCH 080/178] fix pytests --- tests/lint/files_exist.py | 8 ++++++-- tests/modules/lint.py | 2 -- tests/modules/update.py | 24 ++++++++++++------------ tests/subworkflows/lint.py | 3 +++ tests/subworkflows/remove.py | 12 ++++++++---- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/tests/lint/files_exist.py b/tests/lint/files_exist.py index d3a6a25e82..5081522899 100644 --- a/tests/lint/files_exist.py +++ b/tests/lint/files_exist.py @@ -61,7 +61,9 @@ def test_files_exist_pass_conditional(self): lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() lint_obj.nf_config["plugins"] = [] - Path(new_pipeline, "lib/nfcore_external_java_deps.jar").touch() + lib_dir = Path(new_pipeline, "lib") + lib_dir.mkdir() + (lib_dir / "nfcore_external_java_deps.jar").touch() results = lint_obj.files_exist() assert results["failed"] == [] assert results["ignored"] == [] @@ -71,7 +73,9 @@ def test_files_exist_fail_conditional(self): new_pipeline = self._make_pipeline_copy() lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() - Path(new_pipeline, "lib/nfcore_external_java_deps.jar").touch() + lib_dir = Path(new_pipeline, "lib") + lib_dir.mkdir() + (lib_dir / "nfcore_external_java_deps.jar").touch() results = lint_obj.files_exist() assert results["failed"] == ["File must be removed: `lib/nfcore_external_java_deps.jar`"] assert results["ignored"] == [] diff --git a/tests/modules/lint.py b/tests/modules/lint.py index a5d8567b76..9bd280ddd8 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -40,7 +40,6 @@ def test_modules_lint_empty(self): """Test linting a pipeline with no modules installed""" self.mods_remove.remove("fastqc", force=True) self.mods_remove.remove("multiqc", force=True) - self.mods_remove.remove("custom/dumpsoftwareversions", force=True) with pytest.raises(LookupError): nf_core.modules.ModuleLint(dir=self.pipeline_dir) @@ -58,7 +57,6 @@ def test_modules_lint_no_gitlab(self): """Test linting a pipeline with no modules installed""" self.mods_remove.remove("fastqc", force=True) self.mods_remove.remove("multiqc", force=True) - self.mods_remove.remove("custom/dumpsoftwareversions", force=True) with pytest.raises(LookupError): nf_core.modules.ModuleLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) diff --git a/tests/modules/update.py b/tests/modules/update.py index 5208070fa5..5cf24b56f0 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -317,13 +317,13 @@ def test_update_only_show_differences(self, mock_prompt): mod_json = modules_json.get_modules_json() # Loop through all modules and check that they are NOT updated (according to the modules.json file) - # Modules that can be updated but shouldn't are custom/dumpsoftwareversions and fastqc + # A module that can be updated but shouldn't is fastqc # Module multiqc is already up to date so don't check - for mod in ["custom/dumpsoftwareversions", "fastqc"]: - correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] - current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] - assert correct_git_sha != current_git_sha - assert cmp_module(Path(tmpdir, mod), Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, mod)) is True + mod = "fastqc" + correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] + current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] + assert correct_git_sha != current_git_sha + assert cmp_module(Path(tmpdir, mod), Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, mod)) is True # Mock questionary answer: do not update module, only show diffs @@ -357,19 +357,19 @@ def test_update_only_show_differences_when_patch(self, mock_prompt): patch_obj = ModulePatch(self.pipeline_dir) patch_obj.patch("fastqc") # Check that a patch file with the correct name has been created - assert set(os.listdir(module_path)) == {"main.nf", "meta.yml", "fastqc.diff"} + assert "fastqc.diff" in set(os.listdir(module_path)) # Update all modules assert update_obj.update() is True mod_json = modules_json.get_modules_json() # Loop through all modules and check that they are NOT updated (according to the modules.json file) - # Modules that can be updated but shouldn't are custom/dumpsoftwareversions and fastqc + # A module that can be updated but shouldn't is fastqc # Module multiqc is already up to date so don't check - for mod in ["custom/dumpsoftwareversions", "fastqc"]: - correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] - current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] - assert correct_git_sha != current_git_sha + mod = "fastqc" + correct_git_sha = list(update_obj.modules_repo.get_component_git_log(mod, "modules", depth=1))[0]["git_sha"] + current_git_sha = mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME][mod]["git_sha"] + assert correct_git_sha != current_git_sha def cmp_module(dir1, dir2): diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index b53fef7f0e..8804f8bf6f 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -19,6 +19,9 @@ def test_subworkflows_lint(self): def test_subworkflows_lint_empty(self): """Test linting a pipeline with no subworkflows installed""" + self.subworkflow_remove.remove("utils_nextflow_pipeline", force=True) + self.subworkflow_remove.remove("utils_nfcore_pipeline", force=True) + self.subworkflow_remove.remove("utils_nfvalidation_plugin", force=True) with pytest.raises(LookupError): nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) diff --git a/tests/subworkflows/remove.py b/tests/subworkflows/remove.py index dec67875bd..c6a3b98454 100644 --- a/tests/subworkflows/remove.py +++ b/tests/subworkflows/remove.py @@ -20,13 +20,15 @@ def test_subworkflows_remove_subworkflow(self): mod_json_before = ModulesJson(self.pipeline_dir).get_modules_json() assert self.subworkflow_remove.remove("bam_sort_stats_samtools") mod_json_after = ModulesJson(self.pipeline_dir).get_modules_json() - assert Path.exists(subworkflow_path) is False assert Path.exists(bam_sort_stats_samtools_path) is False assert Path.exists(bam_stats_samtools_path) is False assert Path.exists(samtools_index_path) is False assert mod_json_before != mod_json_after # assert subworkflows key is removed from modules.json - assert "subworkflows" not in mod_json_after["repos"]["https://github.com/nf-core/modules.git"].keys() + assert ( + "bam_sort_stats_samtools" + not in mod_json_after["repos"]["https://github.com/nf-core/modules.git"]["subworkflows"].keys() + ) assert "samtools/index" not in mod_json_after["repos"]["https://github.com/nf-core/modules.git"]["modules"].keys() @@ -44,13 +46,15 @@ def test_subworkflows_remove_subworkflow_keep_installed_module(self): assert self.subworkflow_remove.remove("bam_sort_stats_samtools") mod_json_after = ModulesJson(self.pipeline_dir).get_modules_json() - assert Path.exists(subworkflow_path) is False assert Path.exists(bam_sort_stats_samtools_path) is False assert Path.exists(bam_stats_samtools_path) is False assert Path.exists(samtools_index_path) is True assert mod_json_before != mod_json_after # assert subworkflows key is removed from modules.json - assert "subworkflows" not in mod_json_after["repos"]["https://github.com/nf-core/modules.git"].keys() + assert ( + "bam_sort_stats_samtools" + not in mod_json_after["repos"]["https://github.com/nf-core/modules.git"]["subworkflows"].keys() + ) assert ( "samtools/index" in mod_json_after["repos"]["https://github.com/nf-core/modules.git"]["modules"]["nf-core"].keys() From 0dd064af5d426b607f7ff2bffe427f0104744e16 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 13 Feb 2024 16:03:55 +0100 Subject: [PATCH 081/178] rename local subworkflow from the template --- nf_core/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/create.py b/nf_core/create.py index 8038a995c5..c094d33a22 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -292,7 +292,7 @@ def render_template(self): short_name = self.template_params["short_name"] rename_files = { "workflows/pipeline.nf": f"workflows/{short_name}.nf", - "lib/WorkflowPipeline.groovy": f"lib/Workflow{short_name[0].upper()}{short_name[1:]}.groovy", + "subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf": f"subworkflows/local/utils_nfcore_{short_name}_pipeline/main.nf", } # Set the paths to skip according to customization From 5b5b30aa5ca0de2805935ab94baeff2fc9e87840 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 13 Feb 2024 16:09:36 +0100 Subject: [PATCH 082/178] fix editorconfig errors in the template --- nf_core/pipeline-template/.editorconfig | 7 +++++++ nf_core/pipeline-template/nextflow.config | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.editorconfig b/nf_core/pipeline-template/.editorconfig index dd991a0e95..cf8bcff952 100644 --- a/nf_core/pipeline-template/.editorconfig +++ b/nf_core/pipeline-template/.editorconfig @@ -19,6 +19,13 @@ insert_final_newline = unset trim_trailing_whitespace = unset indent_style = unset indent_size = unset +[/subworkflows/nf-core/**] +charset = unset +end_of_line = unset +insert_final_newline = unset +trim_trailing_whitespace = unset +indent_style = unset +indent_size = unset [/assets/email*] indent_size = unset diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 3153ff70d6..0f840fc60b 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -18,9 +18,9 @@ params { genome = null igenomes_base = 's3://ngi-igenomes/igenomes/' igenomes_ignore = false - {% else %} + {% else -%} fasta = null - {%- endif %} + {%- endif -%} // MultiQC options multiqc_config = null @@ -49,7 +49,7 @@ params { custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" config_profile_contact = null config_profile_url = null - {% endif %} + {%- endif %} // Max resource options // Defaults only, expecting to be overwritten From 4b51a2795e70d83d60adaedbd1fed3fa9c7126cc Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:28:51 +0000 Subject: [PATCH 083/178] chore(deps): update gitpod/workspace-base docker digest to 45e7617 --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 27ab60127f..b8adb53827 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:7a130ed80741c3f838a7607f88296d26ac477e713c12b638ab93874d3169f7f1 +FROM gitpod/workspace-base@sha256:45e7617f50af7df5006dce82a65d0d59fb4076a613f906f43b5f44941dd9f70d USER root From 17ffafbb7b739e5bf20331b0f47c6f437608210b Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 13 Feb 2024 15:39:45 +0000 Subject: [PATCH 084/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c37bfc12af..bfa62cacf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Update pre-commit hook astral-sh/ruff-pre-commit to v0.2.1 ([#2730](https://github.com/nf-core/tools/pull/2730)) - Update python:3.11-slim Docker digest to 2a746e2 ([#2743](https://github.com/nf-core/tools/pull/2743)) - Update actions/setup-python action to v5 ([#2739](https://github.com/nf-core/tools/pull/2739)) +- Update gitpod/workspace-base Docker digest to 45e7617 ([#2747](https://github.com/nf-core/tools/pull/2747)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From 71a0eec0a631d3ff15ca2af94378602fc214f0ca Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 14 Feb 2024 11:06:01 +0100 Subject: [PATCH 085/178] rename local subworkflow with jinja --- nf_core/pipeline-template/main.nf | 6 +++--- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index c31927f236..2590f7467b 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -20,10 +20,10 @@ nextflow.enable.dsl = 2 */ include { {{ short_name|upper }} } from './workflows/{{ short_name }}' -include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_pipeline_pipeline' -include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_pipeline_pipeline' +include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' +include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' {% if igenomes %} -include { getGenomeAttribute } from './subworkflows/local/utils_nfcore_pipeline_pipeline' +include { getGenomeAttribute } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 6c016168c5..81fdf9f27d 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -9,7 +9,7 @@ include { MULTIQC } from '../modules/nf-core/multiqc/main' include { paramsSummaryMap } from 'plugin/nf-validation' include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' -include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_pipeline_pipeline' +include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_{{ short_name }}_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From ff69ee3893b356d0cd0c5ca3512a8261a0ccbae7 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 14 Feb 2024 12:12:27 +0100 Subject: [PATCH 086/178] fix more pytests --- tests/subworkflows/update.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/subworkflows/update.py b/tests/subworkflows/update.py index 32a69ba180..9ddc9bec0c 100644 --- a/tests/subworkflows/update.py +++ b/tests/subworkflows/update.py @@ -171,9 +171,9 @@ def test_update_with_config_fix_all(self): with open(Path(self.pipeline_dir, config_fn), "w") as f: yaml.dump(tools_config, f) - # Update all subworkflows in the pipeline - update_obj = SubworkflowUpdate(self.pipeline_dir, update_all=True, show_diff=False) - assert update_obj.update() is True + # Update fastq_align_bowtie2 + update_obj = SubworkflowUpdate(self.pipeline_dir, update_all=False, update_deps=True, show_diff=False) + assert update_obj.update("fastq_align_bowtie2") is True # Check that the git sha for fastq_align_bowtie2 is correctly downgraded mod_json = ModulesJson(self.pipeline_dir).get_modules_json() From b3259449641cfbf3740d9e3589e09275210fb322 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 14 Feb 2024 14:42:18 +0100 Subject: [PATCH 087/178] fix creating a template without igenomes --- .prettierignore | 1 + nf_core/create.py | 1 + nf_core/pipeline-template/nextflow.config | 1 - nf_core/pipeline-template/nextflow_schema.json | 6 ++++-- .../local/utils_nfcore_pipeline_pipeline/main.nf | 7 ++++++- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.prettierignore b/.prettierignore index a55074abfb..b923532bd7 100644 --- a/.prettierignore +++ b/.prettierignore @@ -6,6 +6,7 @@ testing nf_core/module-template/meta.yml nf_core/module-template/tests/tags.yml nf_core/subworkflow-template/tests/tags.yml +nf_core/pipeline-template/nextflow_schema.json # don't run on things handled by ruff *.py *.pyc diff --git a/nf_core/create.py b/nf_core/create.py index c094d33a22..fa900cdb06 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -497,6 +497,7 @@ def fix_linting(self): # Add the lint content to the preexisting nf-core config config_fn, nf_core_yml = nf_core.utils.load_tools_config(self.outdir) nf_core_yml["lint"] = lint_config + print("added linting config") with open(self.outdir / config_fn, "w") as fh: yaml.dump(nf_core_yml, fh, default_flow_style=False, sort_keys=False) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 0f840fc60b..17e75f18a4 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -18,7 +18,6 @@ params { genome = null igenomes_base = 's3://ngi-igenomes/igenomes/' igenomes_ignore = false - {% else -%} fasta = null {%- endif -%} diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 1b603dcf85..de841550e3 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -43,6 +43,7 @@ } } }, + {%- if igenomes %} "reference_genome_options": { "title": "Reference genome options", "type": "object", @@ -74,6 +75,7 @@ } } }, + {%- endif %} "institutional_config_options": { "title": "Institutional config options", "type": "object", @@ -273,9 +275,9 @@ { "$ref": "#/definitions/input_output_options" }, - { + {%- if igenomes %}{ "$ref": "#/definitions/reference_genome_options" - }, + },{% endif %} { "$ref": "#/definitions/institutional_config_options" }, diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 5dd87e8f47..8b911d1667 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -73,10 +73,12 @@ workflow PIPELINE_INITIALISATION { nextflow_cli_args ) + {%- if igenomes %} // // Custom validation for pipeline parameters // validateInputParameters() + {%- endif %} // // Create channel from input file provided through params.input @@ -100,7 +102,6 @@ workflow PIPELINE_INITIALISATION { return [ meta, fastqs.flatten() ] } .set { ch_samplesheet } - ch_samplesheet.view() emit: samplesheet = ch_samplesheet @@ -150,12 +151,14 @@ workflow PIPELINE_COMPLETION { ======================================================================================== */ +{%- if igenomes %} // // Check and validate pipeline parameters // def validateInputParameters() { genomeExistsError() } +{%- endif -%} // // Validate channels from input samplesheet @@ -172,6 +175,7 @@ def validateInputSamplesheet(input) { return [ metas[0], fastqs ] } +{%- if igenomes %} // // Get attribute from genome config file e.g. fasta // @@ -197,6 +201,7 @@ def genomeExistsError() { error(error_string) } } +{%- endif -%} // // Generate methods description for MultiQC From 3bd4cfe42e26ba1ee6c436687973299ea131537e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 14 Feb 2024 15:52:25 +0100 Subject: [PATCH 088/178] return multiqc description as string --- .../local/utils_nfcore_pipeline_pipeline/main.nf | 2 +- nf_core/pipeline-template/workflows/pipeline.nf | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 8b911d1667..24d4c2d7e1 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -256,5 +256,5 @@ def methodsDescriptionText(mqc_methods_yaml) { def engine = new groovy.text.SimpleTemplateEngine() def description_html = engine.createTemplate(methods_text).make(meta) - return description_html + return description_html.toString() } diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 81fdf9f27d..f7de301ce5 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -53,13 +53,9 @@ workflow {{ short_name|upper }} { ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) - methods_description_file = new File('methods_description_mqc.yaml') - ch_methods_description.map{ - methods_description_file << it - } ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) - ch_multiqc_files = ch_multiqc_files.mix(Channel.fromPath(methods_description_file, checkIfExists: true)) + ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false)) MULTIQC ( ch_multiqc_files.collect(), From f6375ddc9d90deb79e7e261548e2edc72d444d02 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 14 Feb 2024 15:55:06 +0100 Subject: [PATCH 089/178] fix docstrings --- nf_core/lint/merge_markers.py | 10 ++++++++-- nf_core/lint/modules_structure.py | 6 ++++++ nf_core/lint/schema_description.py | 7 ++++--- nf_core/modules/lint/meta_yml.py | 3 ++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/nf_core/lint/merge_markers.py b/nf_core/lint/merge_markers.py index 80cd655066..d57b63fd19 100644 --- a/nf_core/lint/merge_markers.py +++ b/nf_core/lint/merge_markers.py @@ -11,19 +11,25 @@ def merge_markers(self): """Check for remaining merge markers. This test looks for remaining merge markers in the code, e.g.: - >>>>>>> or <<<<<<< + ``>>>>>>>`` or ``<<<<<<<`` - .. tip:: You can choose to ignore this lint tests by editing the file called + .. note:: You can choose to ignore this lint tests by editing the file called ``.nf-core.yml`` in the root of your pipeline and setting the test to false: + .. code-block:: yaml + lint: merge_markers: False + To disable this test only for specific files, you can specify a list of file paths to ignore. For example, to ignore a pdf you added to the docs: + .. code-block:: yaml + lint: merge_markers: - docs/my_pdf.pdf + """ passed = [] failed = [] diff --git a/nf_core/lint/modules_structure.py b/nf_core/lint/modules_structure.py index f0e13e0346..9d9b4c9fc0 100644 --- a/nf_core/lint/modules_structure.py +++ b/nf_core/lint/modules_structure.py @@ -8,9 +8,15 @@ def modules_structure(self): """ Check that the structure of the modules directory in a pipeline is the correct one: + + .. code-block:: bash + modules/nf-core/TOOL/SUBTOOL Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: + + .. code-block:: bash + modules/nf-core/modules/TOOL/SUBTOOL """ wrong_location_modules = [] diff --git a/nf_core/lint/schema_description.py b/nf_core/lint/schema_description.py index ca22f266ab..90735f609a 100644 --- a/nf_core/lint/schema_description.py +++ b/nf_core/lint/schema_description.py @@ -2,10 +2,11 @@ def schema_description(self): - """Check that every parameter in the schema has a description + """Check that every parameter in the schema has a description. - The ``nextflow_schema.json`` pipeline schema should describe every flat parameter - Furthermore warns about parameters outside of groups + The ``nextflow_schema.json`` pipeline schema should describe every flat parameter. + + Furthermore warns about parameters outside of groups. * Warning: Parameters in ``nextflow_schema.json`` without a description * Warning: Parameters in ``nextflow_schema.json`` that are defined outside of a group diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 551a978f4d..481d50b3ef 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -25,7 +25,8 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None If the module has inputs or outputs, they are expected to be formatted as: - ..code-block:: + .. code-block:: groovy + tuple val(foo) path(bar) val foo path foo From cf62ce0176e62865c7fefafc51380d3619df9b87 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 14 Feb 2024 16:04:03 +0100 Subject: [PATCH 090/178] fix more docstrings --- nf_core/download.py | 13 +++++++++++++ nf_core/params_file.py | 4 +--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 4c0bc97f42..bb7b2ae473 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -759,6 +759,9 @@ def rectify_raw_container_matches(self, raw_findings): Example syntax: Early DSL2: + + .. code-block:: groovy + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { container "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0" } else { @@ -766,11 +769,17 @@ def rectify_raw_container_matches(self, raw_findings): } Later DSL2: + + .. code-block:: groovy + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : 'biocontainers/fastqc:0.11.9--0' }" Later DSL2, variable is being used: + + .. code-block:: groovy + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? "https://depot.galaxyproject.org/singularity/${container_id}" : "quay.io/biocontainers/${container_id}" }" @@ -778,7 +787,11 @@ def rectify_raw_container_matches(self, raw_findings): container_id = 'mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0' DSL1 / Special case DSL2: + + .. code-block:: groovy + container "nfcore/cellranger:6.0.2" + """ cleaned_matches = [] diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 51986821b5..267fe7086a 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -180,9 +180,7 @@ def format_param(self, name, properties, required_properties=(), show_hidden=Fal Returns: str: Section of a params-file.yml for given parameter - None: - If the parameter is skipped because it is hidden and - show_hidden is not set + None: If the parameter is skipped because it is hidden and show_hidden is not set """ out = "" hidden = properties.get("hidden", False) From 9c89874c563e07d3a17536b1b0017b7def9e5363 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 14 Feb 2024 16:31:26 +0100 Subject: [PATCH 091/178] fix prettier inconsistencies in the pipeline template --- nf_core/pipeline-template/.editorconfig | 2 -- nf_core/pipeline-template/nextflow_schema.json | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.editorconfig b/nf_core/pipeline-template/.editorconfig index cf8bcff952..dd9ffa5387 100644 --- a/nf_core/pipeline-template/.editorconfig +++ b/nf_core/pipeline-template/.editorconfig @@ -18,14 +18,12 @@ end_of_line = unset insert_final_newline = unset trim_trailing_whitespace = unset indent_style = unset -indent_size = unset [/subworkflows/nf-core/**] charset = unset end_of_line = unset insert_final_newline = unset trim_trailing_whitespace = unset indent_style = unset -indent_size = unset [/assets/email*] indent_size = unset diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index de841550e3..77b1a7a070 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -275,7 +275,7 @@ { "$ref": "#/definitions/input_output_options" }, - {%- if igenomes %}{ + {% if igenomes %}{ "$ref": "#/definitions/reference_genome_options" },{% endif %} { From 7f7e920dcbeea7317b92b1000dcc112a52bcb91c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 14 Feb 2024 16:43:28 +0100 Subject: [PATCH 092/178] fix more linting tests --- nf_core/lint/files_exist.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index ab29c4c39e..3e98526266 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -66,7 +66,6 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: conf/igenomes.config .github/workflows/awstest.yml .github/workflows/awsfulltest.yml - lib/WorkflowPIPELINE.groovy pyproject.toml Files that *must not* be present, due to being renamed or removed in the template: @@ -88,6 +87,11 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: lib/Checks.groovy lib/Completion.groovy lib/Workflow.groovy + lib/WorkflowPIPELINE.groovy + lib/NfcoreTemplate.groovy + lib/Utils.groovy + lib/WorkflowMain.groovy + Files that *should not* be present: @@ -171,7 +175,6 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: [Path("conf", "igenomes.config")], [Path(".github", "workflows", "awstest.yml")], [Path(".github", "workflows", "awsfulltest.yml")], - [Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")], [Path("modules.json")], [Path("pyproject.toml")], ] @@ -200,6 +203,7 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: Path("lib", "Utils.groovy"), Path("lib", "WorkflowMain.groovy"), Path("lib", "NfcoreTemplate.groovy"), + Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy"), ] files_fail_ifinconfig: List[Tuple[Path, Dict[str, str]]] = [ (Path("lib", "nfcore_external_java_deps.jar"), {"plugins": "nf-validation"}), From 5b8dca6ea1a78f597cb4f3e2ef9384fb8f8ec9ca Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 21:41:52 +0000 Subject: [PATCH 093/178] Update python:3.11-slim Docker digest to ce81dc5 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3e561d151d..62431be140 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim@sha256:2a746e2b9dfd9c155e1218ee5bcaad64c3c8816258c0ee7d25f3893ed2252a1e +FROM python:3.11-slim@sha256:ce81dc539f0aedc9114cae640f8352fad83d37461c24a3615b01f081d0c0583a LABEL authors="phil.ewels@scilifelab.se,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for the nfcore tools" From 068e24ae5a8bffab4820773787eeb99b3475ba1b Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 13:05:18 +0100 Subject: [PATCH 094/178] add working generate-api-doc.sh script --- docs/api/generate-api-docs.sh | 75 +++++++++++++++++++++++++++++++++ nf_core/lint/multiqc_config.py | 1 + nf_core/lint/nextflow_config.py | 1 + 3 files changed, 77 insertions(+) create mode 100644 docs/api/generate-api-docs.sh diff --git a/docs/api/generate-api-docs.sh b/docs/api/generate-api-docs.sh new file mode 100644 index 0000000000..2d68b21204 --- /dev/null +++ b/docs/api/generate-api-docs.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Set the output directory +output_dir="${HOME}/nf-core.astro/src/content/tools/docs" + +# allow --force option and also a --release option (which takes a release name, or "all") +while [ "$1" != "" ]; do + case $1 in + -f | --force ) force=true + ;; + -r | --release ) shift + releases=$1 + ;; + * ) echo "Invalid argument: $1" + exit 1 + esac + shift +done + +# if no release is specified, use all releases +if [ -z "$releases" ]; then + releases=$(git tag) + # add 'dev' to the list of releases + releases="$releases dev" +fi + +# Loop through each release +for release in $releases +do + # Checkout the release + git checkout "$release" + echo "_________________________" + echo "Generating docs for release: $release" + echo "_________________________" + git checkout docs/api + pip install -r docs/api/requirements.txt --quiet + # add the napoleon extension to the sphinx conf.py + gsed -i 's/^extensions = \[/extensions = \[\n "sphinx_markdown_builder",/' docs/api/_src/conf.py + + find nf_core -name "*.py" | while IFS= read -r file; do + # echo "Processing $file" + + # replace ..tip:: with note in the python docstrings due to missing directive in the markdown builder + gsed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' $file + + done + + # fix syntax in lint/merge_markers.py + gsed -i 's/>>>>>>> or <<<<<<>>>>>>`` or ``<<<<<<<``/g' nf_core/lint/merge_markers.py + # remove markdown files if --force is set + if [ "$force" = true ]; then + echo -e "\n\e[31mRemoving $output_dir/$release because of '--force'\e[0m" + rm -rf "$output_dir/$release" + fi + sphinx-build -b markdown docs/api/_src "$output_dir/$release" + # replace :::{seealso} with :::tip in the markdown files + find "$output_dir/$release" -name "*.md" -exec gsed -i 's/:::{seealso}/:::tip/g' {} \; + i=1 + sp="/-\|" # spinner + find "$output_dir/$release" -name "*.md" | while IFS= read -r file; do + # echo "Processing $file" + printf "\b${sp:i++%${#sp}:1}" + node remark.mjs $file + done + # remove empty files + find "$output_dir/$release" -name "*.md" -size 0 -delete + # remove `.doctrees` directory + rm -rf "$output_dir/$release/.doctrees" + # run pre-commit to fix any formatting issues on the generated markdown files + pre-commit run --files "$output_dir/$release" + # undo all changes + git restore . + # lazydocs --output-path "$output_dir/$release" \ + # --src-base-url="https://github.com/nf-core/tools/blob/$release/" \ + # --overview-file="index.md" nf_core +done diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index 2ad786690d..9b9c80c44e 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -22,6 +22,7 @@ def multiqc_config(self) -> Dict[str, List[str]]: export_plots: true """ + passed: List[str] = [] failed: List[str] = [] diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 68750cd859..d3e29d2363 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -129,6 +129,7 @@ def nextflow_config(self): - config_defaults: - params.input """ + passed = [] warned = [] failed = [] From 28c0196477d8aa21f6ea82d1906163b3e519e773 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 15 Feb 2024 13:09:08 +0100 Subject: [PATCH 095/178] apply suggestions from code review --- nf_core/create.py | 1 - nf_core/lint/files_exist.py | 5 +---- nf_core/pipeline-template/workflows/pipeline.nf | 4 ++-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/nf_core/create.py b/nf_core/create.py index fa900cdb06..c094d33a22 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -497,7 +497,6 @@ def fix_linting(self): # Add the lint content to the preexisting nf-core config config_fn, nf_core_yml = nf_core.utils.load_tools_config(self.outdir) nf_core_yml["lint"] = lint_config - print("added linting config") with open(self.outdir / config_fn, "w") as fh: yaml.dump(nf_core_yml, fh, default_flow_style=False, sort_keys=False) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 3e98526266..5d62a23bf8 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -196,15 +196,12 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: Path("lib", "Checks.groovy"), Path("lib", "Completion.groovy"), Path("lib", "Workflow.groovy"), - ] - files_warn_ifexists = [ - Path(".travis.yml"), - Path("lib", "NfcoreTemplate.groovy"), Path("lib", "Utils.groovy"), Path("lib", "WorkflowMain.groovy"), Path("lib", "NfcoreTemplate.groovy"), Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy"), ] + files_warn_ifexists = [Path(".travis.yml")] files_fail_ifinconfig: List[Tuple[Path, Dict[str, str]]] = [ (Path("lib", "nfcore_external_java_deps.jar"), {"plugins": "nf-validation"}), ] diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index f7de301ce5..68adbaa328 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -65,8 +65,8 @@ workflow {{ short_name|upper }} { ) emit: - multiqc_report = MULTIQC.out.report // channel: /path/to/multiqc_report.html - versions = ch_versions // channel: [ path(versions.yml) ] + multiqc_report = MULTIQC.out.report.toList() // channel: /path/to/multiqc_report.html + versions = ch_versions // channel: [ path(versions.yml) ] } /* From 8b612074e4ff5d3f838787ec8376dbc6b7e37065 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 14:11:54 +0100 Subject: [PATCH 096/178] optimize script --- docs/api/generate-api-docs.sh | 47 +++++++++++++++++++---------------- nf_core/lint/__init__.py | 14 +++++------ 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/docs/api/generate-api-docs.sh b/docs/api/generate-api-docs.sh index 2d68b21204..73121966b9 100644 --- a/docs/api/generate-api-docs.sh +++ b/docs/api/generate-api-docs.sh @@ -3,29 +3,35 @@ output_dir="${HOME}/nf-core.astro/src/content/tools/docs" # allow --force option and also a --release option (which takes a release name, or "all") -while [ "$1" != "" ]; do +force=false +releases=() + +while [[ $# -gt 0 ]]; do case $1 in - -f | --force ) force=true - ;; - -r | --release ) shift - releases=$1 - ;; - * ) echo "Invalid argument: $1" - exit 1 + -f | --force ) + force=true + ;; + -r | --release ) + shift + releases+=("$1") + ;; + * ) + echo "Invalid argument: $1" + exit 1 + ;; esac shift done # if no release is specified, use all releases -if [ -z "$releases" ]; then - releases=$(git tag) +if [[ ${#releases[@]} -eq 0 ]]; then + releases=($(git tag)) # add 'dev' to the list of releases - releases="$releases dev" + releases+=("dev") fi # Loop through each release -for release in $releases -do +for release in "${releases[@]}"; do # Checkout the release git checkout "$release" echo "_________________________" @@ -34,32 +40,32 @@ do git checkout docs/api pip install -r docs/api/requirements.txt --quiet # add the napoleon extension to the sphinx conf.py - gsed -i 's/^extensions = \[/extensions = \[\n "sphinx_markdown_builder",/' docs/api/_src/conf.py + sed -i 's/^extensions = \[/extensions = \[\n "sphinx_markdown_builder",/' docs/api/_src/conf.py find nf_core -name "*.py" | while IFS= read -r file; do # echo "Processing $file" # replace ..tip:: with note in the python docstrings due to missing directive in the markdown builder - gsed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' $file + sed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' "$file" done # fix syntax in lint/merge_markers.py - gsed -i 's/>>>>>>> or <<<<<<>>>>>>`` or ``<<<<<<<``/g' nf_core/lint/merge_markers.py + sed -i 's/>>>>>>> or <<<<<<>>>>>>`` or ``<<<<<<<``/g' nf_core/lint/merge_markers.py # remove markdown files if --force is set - if [ "$force" = true ]; then + if [[ "$force" = true ]]; then echo -e "\n\e[31mRemoving $output_dir/$release because of '--force'\e[0m" rm -rf "$output_dir/$release" fi sphinx-build -b markdown docs/api/_src "$output_dir/$release" # replace :::{seealso} with :::tip in the markdown files - find "$output_dir/$release" -name "*.md" -exec gsed -i 's/:::{seealso}/:::tip/g' {} \; + find "$output_dir/$release" -name "*.md" -exec sed -i 's/:::{seealso}/:::tip/g' {} \; i=1 sp="/-\|" # spinner find "$output_dir/$release" -name "*.md" | while IFS= read -r file; do # echo "Processing $file" printf "\b${sp:i++%${#sp}:1}" - node remark.mjs $file + node remark.mjs "$file" done # remove empty files find "$output_dir/$release" -name "*.md" -size 0 -delete @@ -69,7 +75,4 @@ do pre-commit run --files "$output_dir/$release" # undo all changes git restore . - # lazydocs --output-path "$output_dir/$release" \ - # --src-base-url="https://github.com/nf-core/tools/blob/$release/" \ - # --overview-file="index.md" nf_core done diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index d39ce5d7e9..be9ac183a6 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -404,9 +404,7 @@ def format_result(test_results): if "dev" in __version__: tools_version = "latest" for eid, msg in test_results: - yield Markdown( - f"[{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html): {msg}" - ) + yield Markdown(f"[{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}): {msg}") # Table of passed tests if len(self.passed) > 0 and show_passed: @@ -507,7 +505,7 @@ def _get_results_md(self): test_failures = "### :x: Test failures:\n\n{}\n\n".format( "\n".join( [ - f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html) - " + f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}) - " f"{strip_ansi_codes(msg, '`')}" for eid, msg in self.failed ] @@ -521,7 +519,7 @@ def _get_results_md(self): test_ignored = "### :grey_question: Tests ignored:\n\n{}\n\n".format( "\n".join( [ - f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html) - " + f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}) - " f"{strip_ansi_codes(msg, '`')}" for eid, msg in self.ignored ] @@ -535,7 +533,7 @@ def _get_results_md(self): test_fixed = "### :grey_question: Tests fixed:\n\n{}\n\n".format( "\n".join( [ - f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html) - " + f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}) - " f"{strip_ansi_codes(msg, '`')}" for eid, msg in self.fixed ] @@ -549,7 +547,7 @@ def _get_results_md(self): test_warnings = "### :heavy_exclamation_mark: Test warnings:\n\n{}\n\n".format( "\n".join( [ - f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html) - " + f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}) - " f"{strip_ansi_codes(msg, '`')}" for eid, msg in self.warned ] @@ -564,7 +562,7 @@ def _get_results_md(self): "\n".join( [ ( - f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid}.html)" + f"* [{eid}](https://nf-co.re/tools/docs/{tools_version}/pipeline_lint_tests/{eid})" f" - {strip_ansi_codes(msg, '`')}" ) for eid, msg in self.passed From bba8f93eb9dec28914240752e1738ce89c7ab0de Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 15 Feb 2024 13:50:35 +0000 Subject: [PATCH 097/178] Add fix for disk space issues on github actions runners --- CHANGELOG.md | 1 + nf_core/pipeline-template/.github/workflows/ci.yml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfa62cacf3..dcfcdc51fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Add empty line in README.md to fix badges. ([#2729](https://github.com/nf-core/tools/pull/2729)) - Replace automatic branch detection in `nf-core download` CI test with hardcoded `dev` and input. ([#2727](https://github.com/nf-core/tools/pull/2727)) +- Add Github Action to automatically cleanup ubuntu-latest runners to fix runner running out of diskspace errors([#2754](https://github.com/nf-core/tools/issues/2754)) ### Linting diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 1290bfafc0..1eaefa24b8 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -35,6 +35,9 @@ jobs: with: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" + - name: Cleanup Runner + uses: jlumbroso/free-disk-space@v1.3.1 + - name: Run pipeline with test data # TODO nf-core: You can customise CI pipeline run tests as required # For example: adding multiple test runs with different parameters From e3514603c7f7dbb4d131ec13cc9fcf525bb14b5f Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 15 Feb 2024 13:51:51 +0000 Subject: [PATCH 098/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcfcdc51fc..d98bb594c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Update python:3.11-slim Docker digest to 2a746e2 ([#2743](https://github.com/nf-core/tools/pull/2743)) - Update actions/setup-python action to v5 ([#2739](https://github.com/nf-core/tools/pull/2739)) - Update gitpod/workspace-base Docker digest to 45e7617 ([#2747](https://github.com/nf-core/tools/pull/2747)) +- Fix for #2754, disk space on gha runners ([#2755](https://github.com/nf-core/tools/pull/2755)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From b20c7348bcdfb22bb9941532e74fabbebddd43e7 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 15 Feb 2024 14:55:15 +0100 Subject: [PATCH 099/178] Apply suggestions from code review --- CHANGELOG.md | 1 - nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d98bb594c1..dcfcdc51fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,6 @@ - Update python:3.11-slim Docker digest to 2a746e2 ([#2743](https://github.com/nf-core/tools/pull/2743)) - Update actions/setup-python action to v5 ([#2739](https://github.com/nf-core/tools/pull/2739)) - Update gitpod/workspace-base Docker digest to 45e7617 ([#2747](https://github.com/nf-core/tools/pull/2747)) -- Fix for #2754, disk space on gha runners ([#2755](https://github.com/nf-core/tools/pull/2755)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 1eaefa24b8..0575946844 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: with: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" - - name: Cleanup Runner + - name: Disk space cleanup uses: jlumbroso/free-disk-space@v1.3.1 - name: Run pipeline with test data From 23bd8da2172627861d147b55a79260f6048bc8dd Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 15 Feb 2024 13:57:43 +0000 Subject: [PATCH 100/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcfcdc51fc..dd5ebe402e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Update python:3.11-slim Docker digest to 2a746e2 ([#2743](https://github.com/nf-core/tools/pull/2743)) - Update actions/setup-python action to v5 ([#2739](https://github.com/nf-core/tools/pull/2739)) - Update gitpod/workspace-base Docker digest to 45e7617 ([#2747](https://github.com/nf-core/tools/pull/2747)) +- Add Github Action to automatically cleanup ubuntu-latest runners to fix runner running out of diskspace errors ([#2755](https://github.com/nf-core/tools/pull/2755)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From e28367608445016885ebcb1a9f00b262ec7dd778 Mon Sep 17 00:00:00 2001 From: Raquel Manzano <36073691+RaqManzano@users.noreply.github.com> Date: Thu, 15 Feb 2024 13:59:15 +0000 Subject: [PATCH 101/178] Update README.md Fixed GitHub Actions CI and Linting badges links. --- nf_core/pipeline-template/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 5160ccb04d..72b80e2035 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -8,8 +8,8 @@ {% endif -%} {% if github_badges -%} -[![GitHub Actions CI Status](https://github.com/{{ name }}/workflows/nf-core%20CI/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/{{ name }}/workflows/nf-core%20linting/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22){% endif -%} +[![GitHub Actions CI Status](https://github.com/{{ name }}/actions/workflows/ci.yml/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) +[![GitHub Actions Linting Status](https://github.com/{{ name }}/actions/workflows/linting_comment.yml/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22){% endif -%} {% if branded -%}[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} {%- if github_badges -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) From 54f836d4c0435a6c36655c71ec23595a3656dd5f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 15 Feb 2024 15:00:08 +0100 Subject: [PATCH 102/178] Remove duplicate changelog entry --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd5ebe402e..dcfcdc51fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,6 @@ - Update python:3.11-slim Docker digest to 2a746e2 ([#2743](https://github.com/nf-core/tools/pull/2743)) - Update actions/setup-python action to v5 ([#2739](https://github.com/nf-core/tools/pull/2739)) - Update gitpod/workspace-base Docker digest to 45e7617 ([#2747](https://github.com/nf-core/tools/pull/2747)) -- Add Github Action to automatically cleanup ubuntu-latest runners to fix runner running out of diskspace errors ([#2755](https://github.com/nf-core/tools/pull/2755)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From b1c14109390c69969c0bed84f3eff543e188c697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 15 Feb 2024 15:00:45 +0100 Subject: [PATCH 103/178] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcfcdc51fc..86a879a23e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Add empty line in README.md to fix badges. ([#2729](https://github.com/nf-core/tools/pull/2729)) - Replace automatic branch detection in `nf-core download` CI test with hardcoded `dev` and input. ([#2727](https://github.com/nf-core/tools/pull/2727)) -- Add Github Action to automatically cleanup ubuntu-latest runners to fix runner running out of diskspace errors([#2754](https://github.com/nf-core/tools/issues/2754)) +- Add Github Action to automatically cleanup ubuntu-latest runners to fix runner running out of diskspace errors([#2755](https://github.com/nf-core/tools/issues/2755)) ### Linting From 4149598bd6320ae078cb2958312ecf41c401e343 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 14:13:51 +0000 Subject: [PATCH 104/178] chore(deps): pin jlumbroso/free-disk-space action to 54081f1 --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 0575946844..631862d9ec 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" - name: Disk space cleanup - uses: jlumbroso/free-disk-space@v1.3.1 + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Run pipeline with test data # TODO nf-core: You can customise CI pipeline run tests as required From a69512f3d1bd060f53afdfc21719c8134003d9b6 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 15 Feb 2024 14:14:38 +0000 Subject: [PATCH 105/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a879a23e..08786347fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Update python:3.11-slim Docker digest to 2a746e2 ([#2743](https://github.com/nf-core/tools/pull/2743)) - Update actions/setup-python action to v5 ([#2739](https://github.com/nf-core/tools/pull/2739)) - Update gitpod/workspace-base Docker digest to 45e7617 ([#2747](https://github.com/nf-core/tools/pull/2747)) +- chore(deps): pin jlumbroso/free-disk-space action to 54081f1 ([#2756](https://github.com/nf-core/tools/pull/2756)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From d214ed619aa5f158c66f77696faeff4ee11d5eb9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 15 Feb 2024 15:23:24 +0100 Subject: [PATCH 106/178] Fix badge links --- nf_core/pipeline-template/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 14ca64b9a7..037d3fd81a 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -9,8 +9,8 @@ {% endif -%} {% if github_badges -%} -[![GitHub Actions CI Status](https://github.com/{{ name }}/actions/workflows/ci.yml/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/{{ name }}/actions/workflows/linting_comment.yml/badge.svg)](https://github.com/{{ name }}/actions?query=workflow%3A%22nf-core+linting%22){% endif -%} +[![GitHub Actions CI Status](https://github.com/{{ name }}/actions/workflows/ci.yml/badge.svg)](https://github.com/{{ name }}/actions/workflows/ci.yml) +[![GitHub Actions Linting Status](https://github.com/{{ name }}/actions/workflows/linting.yml/badge.svg)](https://github.com/{{ name }}/actions/workflows/linting.yml){% endif -%} {% if branded -%}[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/{{ short_name }}/results){% endif -%} {%- if github_badges -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) From a32f6d3f2e8c36282afd4e9ea5bca9c0c17b75be Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 15 Feb 2024 14:25:05 +0000 Subject: [PATCH 107/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a879a23e..e055d41e2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733)) - add looser comparison when pipeline logos ([#2744](https://github.com/nf-core/tools/pull/2744)) +- Template: Fix GitHub Actions CI and badges links ([#2757](https://github.com/nf-core/tools/pull/2757)) ### Modules From 1a57b3a0122d88a71d85527774cabc3c21e8ce0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 15 Feb 2024 14:36:18 +0000 Subject: [PATCH 108/178] fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e055d41e2d..2976528f14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,12 @@ - Add empty line in README.md to fix badges. ([#2729](https://github.com/nf-core/tools/pull/2729)) - Replace automatic branch detection in `nf-core download` CI test with hardcoded `dev` and input. ([#2727](https://github.com/nf-core/tools/pull/2727)) - Add Github Action to automatically cleanup ubuntu-latest runners to fix runner running out of diskspace errors([#2755](https://github.com/nf-core/tools/issues/2755)) +- Fix GitHub Actions CI and Linting badges links ([#2757](https://github.com/nf-core/tools/pull/2757)) ### Linting - make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733)) - add looser comparison when pipeline logos ([#2744](https://github.com/nf-core/tools/pull/2744)) -- Template: Fix GitHub Actions CI and badges links ([#2757](https://github.com/nf-core/tools/pull/2757)) ### Modules From 59214f5eb04e0aea3bfb544f4365c6890c9201e9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 15:44:39 +0100 Subject: [PATCH 109/178] merge api docs actions to one and trigger action on website repo --- .github/workflows/tools-api-docs-dev.yml | 54 -------------------- .github/workflows/tools-api-docs-release.yml | 48 ----------------- .github/workflows/tools-api-docs.yml | 41 +++++++++++++++ docs/api/generate-api-docs.sh | 2 +- 4 files changed, 42 insertions(+), 103 deletions(-) delete mode 100644 .github/workflows/tools-api-docs-dev.yml delete mode 100644 .github/workflows/tools-api-docs-release.yml create mode 100644 .github/workflows/tools-api-docs.yml diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml deleted file mode 100644 index 19a1621092..0000000000 --- a/.github/workflows/tools-api-docs-dev.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: nf-core/tools dev API docs -# Run on push and PR to test that docs build -on: - push: - branches: - - dev - paths-ignore: - - "CHANGELOG.md" - pull_request: - paths-ignore: - - "CHANGELOG.md" - release: - types: [published] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - api-docs: - name: Build & push Sphinx API docs - runs-on: ubuntu-latest - - steps: - - name: Check out source-code repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 - with: - python-version: 3.11 - - - name: Install python dependencies - run: | - pip install --upgrade pip - pip install -r ./docs/api/requirements.txt - pip install . - - - name: Build HTML docs - run: make --directory ./docs/api html - - - name: Sync dev docs - # Only sync with the website if it was a push from nf-core/tools dev branch - if: github.repository == 'nf-core/tools' && github.event_name == 'push' && github.event.ref == 'refs/heads/dev' - uses: SamKirkland/FTP-Deploy-Action@8a24039354ee91000cb948cb4a1dbdf1a1b94a3c # v4.3.4 - with: - server: ${{ secrets.ftp_server }} - username: ${{ secrets.ftp_username}} - password: ${{ secrets.ftp_password }} - local-dir: "./docs/api/_build/html/" - server-dir: ${{ secrets.ftp_server_old_site_dir }}/dev/ - protocol: ${{ secrets.ftp_protocol }} - port: ${{ secrets.ftp_port }} diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml deleted file mode 100644 index 45ce237f13..0000000000 --- a/.github/workflows/tools-api-docs-release.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: nf-core/tools release API docs -on: - release: - types: [published] - -# Cancel if a newer run is started -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - api-docs: - name: Build & push Sphinx API docs - runs-on: ubuntu-latest - strategy: - matrix: - dir: - - latest - - ${{ github.event.release.tag_name }} - steps: - - name: Check out source-code repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 - with: - python-version: 3.11 - - - name: Install python dependencies - run: | - pip install --upgrade pip - pip install -r ./docs/api/requirements.txt - pip install . - - - name: Build HTML docs - run: make --directory ./docs/api html - - - name: Sync release docs - if: github.repository == 'nf-core/tools' - uses: SamKirkland/FTP-Deploy-Action@8a24039354ee91000cb948cb4a1dbdf1a1b94a3c # v4.3.4 - with: - server: ${{ secrets.ftp_server }} - username: ${{ secrets.ftp_username}} - password: ${{ secrets.ftp_password }} - local-dir: "./docs/api/_build/html/" - server-dir: ${{ secrets.ftp_server_old_site_dir }}/${{ matrix.dir }}/ - protocol: ${{ secrets.ftp_protocol }} - port: ${{ secrets.ftp_port }} diff --git a/.github/workflows/tools-api-docs.yml b/.github/workflows/tools-api-docs.yml new file mode 100644 index 0000000000..60045512d4 --- /dev/null +++ b/.github/workflows/tools-api-docs.yml @@ -0,0 +1,41 @@ +name: nf-core/tools dev API docs +# Run on push and PR to test that docs build +on: + push: + branches: + - dev + paths: + - nf_core/**/*.py + release: + types: [published] + workflow_dispatch: + inputs: + ref_name: + description: "The branch or tag to build the API docs for" + required: true + default: "dev" + +# Cancel if a newer run is started +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + api-docs: + name: trigger API docs build on website repo + runs-on: ubuntu-latest + steps: + - name: trigger API docs build + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.nf_core_bot_auth_token }} + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: 'nf-core', + repo: 'website', + workflow_id: 'add-tools-api-docs.yml', + ref: 'main' + inputs: { + "ref_name": ${{ inputs.ref_name || github.ref_name }} + } + }) diff --git a/docs/api/generate-api-docs.sh b/docs/api/generate-api-docs.sh index 73121966b9..1c1c5650c4 100644 --- a/docs/api/generate-api-docs.sh +++ b/docs/api/generate-api-docs.sh @@ -1,6 +1,6 @@ #!/bin/bash # Set the output directory -output_dir="${HOME}/nf-core.astro/src/content/tools/docs" +output_dir="../src/content/tools/docs" # allow --force option and also a --release option (which takes a release name, or "all") force=false From 018ce822e02abee86f31b65ff379f5fbc18e7ad8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 15:45:01 +0100 Subject: [PATCH 110/178] add missing remark script --- docs/api/remark.mjs | 71 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 docs/api/remark.mjs diff --git a/docs/api/remark.mjs b/docs/api/remark.mjs new file mode 100644 index 0000000000..d743b42314 --- /dev/null +++ b/docs/api/remark.mjs @@ -0,0 +1,71 @@ +import fs from "fs"; +import { remark } from "remark"; +import { visit } from "unist-util-visit"; + +function remarkDirectives() { + return transformer; + + function transformer(tree) { + visit(tree, "heading", visitor); + visit(tree, "link", visitor); + } + + function visitor(node, index, parent) { + if (node.depth === 4) { + if (["note", "warning"].includes(node.children[0].value?.toLowerCase())) { + const type = node.children[0].value.toLowerCase(); + parent.children.splice(index, 1); + parent.children[index].children[0].value = `:::${type}\n${parent.children[index].children[0].value}`; + // if second to list parent.children[index].children ends with ":", check if the next node is a code block, if so, add the code block as a child to the current node + if (parent.children[index].children.slice(-1)[0]?.value?.trim().endsWith(":")) { + if (parent.children[index + 1].type === "code") { + parent.children[index].children.slice(-1)[0].value += "\n"; + parent.children[index].children.push(parent.children[index + 1]); + parent.children.splice(index + 1, 1); + } + } + parent.children[index].children.push({ type: "text", value: "\n:::" }); + } else if (node.children[0].type !== "inlineCode") { + node.children[0] = { + type: "inlineCode", + value: node.children[0].value?.trim() + "{:python}", + }; + } + } else if (node.depth === 3) { + node.children.map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + if (child.type === "link") { + child.children.map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + }); + } + if (child.type === "emphasis") { + child.children.map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + }); + } + }); + } + if (node.type === "link") { + node.url = node.url.replace(".md", ""); + } + } +} + +let markdown = fs.readFileSync(process.argv[2]); + +remark() + .use(remarkDirectives) + .process(markdown, function (err, file) { + if (err) throw err; + fs.writeFileSync(process.argv[2], String(file)); + }); From 3ef89908cad272f4e0ecc31169b3e12f329980db Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 15 Feb 2024 14:47:19 +0000 Subject: [PATCH 111/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfa62cacf3..e28ef9404e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733)) - add looser comparison when pipeline logos ([#2744](https://github.com/nf-core/tools/pull/2744)) +- Switch to markdown based API and error docs ([#2758](https://github.com/nf-core/tools/pull/2758)) ### Modules From 5543bc4e019f7176ee64f8f6a0aa8a892e50824b Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 15:57:11 +0100 Subject: [PATCH 112/178] add outdir option to generate script --- .../_src/module_lint_tests/environment_yml.md | 5 +++++ docs/api/generate-api-docs.sh | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 docs/api/_src/module_lint_tests/environment_yml.md diff --git a/docs/api/_src/module_lint_tests/environment_yml.md b/docs/api/_src/module_lint_tests/environment_yml.md new file mode 100644 index 0000000000..e931f9df05 --- /dev/null +++ b/docs/api/_src/module_lint_tests/environment_yml.md @@ -0,0 +1,5 @@ +# environment_yml + +```{eval-rst} +.. automethod:: nf_core.modules.lint.ModuleLint.environment_yml +``` diff --git a/docs/api/generate-api-docs.sh b/docs/api/generate-api-docs.sh index 1c1c5650c4..30208bc9b3 100644 --- a/docs/api/generate-api-docs.sh +++ b/docs/api/generate-api-docs.sh @@ -1,6 +1,4 @@ #!/bin/bash -# Set the output directory -output_dir="../src/content/tools/docs" # allow --force option and also a --release option (which takes a release name, or "all") force=false @@ -15,6 +13,10 @@ while [[ $# -gt 0 ]]; do shift releases+=("$1") ;; + -o | --output ) + shift + output_dir="$1" + ;; * ) echo "Invalid argument: $1" exit 1 @@ -23,6 +25,12 @@ while [[ $# -gt 0 ]]; do shift done + +# Set the output directory if not set +if [[ -z "$output_dir" ]]; then + output_dir="../src/content/tools/docs" +fi + # if no release is specified, use all releases if [[ ${#releases[@]} -eq 0 ]]; then releases=($(git tag)) @@ -42,6 +50,11 @@ for release in "${releases[@]}"; do # add the napoleon extension to the sphinx conf.py sed -i 's/^extensions = \[/extensions = \[\n "sphinx_markdown_builder",/' docs/api/_src/conf.py + # run docs/api/make_lint_md.py if it exists + if [[ -f "docs/api/make_lint_md.py" ]]; then + python docs/api/make_lint_md.py + fi + find nf_core -name "*.py" | while IFS= read -r file; do # echo "Processing $file" From baef8a06bff5db6588d545d66b96ebd4f95fc69a Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 16:01:56 +0100 Subject: [PATCH 113/178] fix index pages --- docs/api/_src/index.md | 2 -- docs/api/_src/module_lint_tests/index.md | 3 +-- docs/api/_src/pipeline_lint_tests/index.md | 3 +-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/api/_src/index.md b/docs/api/_src/index.md index cb455986d7..17bf2407f5 100644 --- a/docs/api/_src/index.md +++ b/docs/api/_src/index.md @@ -20,5 +20,3 @@ This documentation is for the `nf-core/tools` package. - [Module code lint tests](module_lint_tests/index.md) (run by `nf-core modules lint`) - [Subworkflow code lint tests](subworkflow_lint_tests/index.md) (run by `nf-core subworkflows lint`) - [nf-core/tools Python package API reference](api/index.md) -- {ref}`genindex` -- {ref}`modindex` diff --git a/docs/api/_src/module_lint_tests/index.md b/docs/api/_src/module_lint_tests/index.md index f889abf73c..dee84d06d8 100644 --- a/docs/api/_src/module_lint_tests/index.md +++ b/docs/api/_src/module_lint_tests/index.md @@ -1,9 +1,8 @@ # Module lint tests ```{toctree} -:caption: 'Tests:' :glob: true -:maxdepth: 2 +:maxdepth: 1 * ``` diff --git a/docs/api/_src/pipeline_lint_tests/index.md b/docs/api/_src/pipeline_lint_tests/index.md index 0cf9bc1d21..c631610d64 100644 --- a/docs/api/_src/pipeline_lint_tests/index.md +++ b/docs/api/_src/pipeline_lint_tests/index.md @@ -1,9 +1,8 @@ # Pipeline lint tests ```{toctree} -:caption: 'Tests:' :glob: true -:maxdepth: 2 +:maxdepth: 1 * ``` From 7350e32b7cef907990c50931287a76789c4a371c Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 16:19:17 +0100 Subject: [PATCH 114/178] update path for remark script --- docs/api/generate-api-docs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api/generate-api-docs.sh b/docs/api/generate-api-docs.sh index 30208bc9b3..70e3994bf5 100644 --- a/docs/api/generate-api-docs.sh +++ b/docs/api/generate-api-docs.sh @@ -48,7 +48,7 @@ for release in "${releases[@]}"; do git checkout docs/api pip install -r docs/api/requirements.txt --quiet # add the napoleon extension to the sphinx conf.py - sed -i 's/^extensions = \[/extensions = \[\n "sphinx_markdown_builder",/' docs/api/_src/conf.py + gsed -i 's/^extensions = \[/extensions = \[\n "sphinx_markdown_builder",/' docs/api/_src/conf.py # run docs/api/make_lint_md.py if it exists if [[ -f "docs/api/make_lint_md.py" ]]; then @@ -59,12 +59,12 @@ for release in "${releases[@]}"; do # echo "Processing $file" # replace ..tip:: with note in the python docstrings due to missing directive in the markdown builder - sed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' "$file" + gsed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' "$file"ba done # fix syntax in lint/merge_markers.py - sed -i 's/>>>>>>> or <<<<<<>>>>>>`` or ``<<<<<<<``/g' nf_core/lint/merge_markers.py + gsed -i 's/>>>>>>> or <<<<<<>>>>>>`` or ``<<<<<<<``/g' nf_core/lint/merge_markers.py # remove markdown files if --force is set if [[ "$force" = true ]]; then echo -e "\n\e[31mRemoving $output_dir/$release because of '--force'\e[0m" @@ -78,7 +78,7 @@ for release in "${releases[@]}"; do find "$output_dir/$release" -name "*.md" | while IFS= read -r file; do # echo "Processing $file" printf "\b${sp:i++%${#sp}:1}" - node remark.mjs "$file" + node docs/api/remark.mjs "$file" done # remove empty files find "$output_dir/$release" -name "*.md" -size 0 -delete From e275dff71019862996c7493b5d8b61b4a7304f98 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 16:21:34 +0100 Subject: [PATCH 115/178] don't update members in api docs --- docs/api/generate-api-docs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/generate-api-docs.sh b/docs/api/generate-api-docs.sh index 70e3994bf5..194c2b6c5e 100644 --- a/docs/api/generate-api-docs.sh +++ b/docs/api/generate-api-docs.sh @@ -51,9 +51,9 @@ for release in "${releases[@]}"; do gsed -i 's/^extensions = \[/extensions = \[\n "sphinx_markdown_builder",/' docs/api/_src/conf.py # run docs/api/make_lint_md.py if it exists - if [[ -f "docs/api/make_lint_md.py" ]]; then - python docs/api/make_lint_md.py - fi + # if [[ -f "docs/api/make_lint_md.py" ]]; then + # python docs/api/make_lint_md.py + # fi find nf_core -name "*.py" | while IFS= read -r file; do # echo "Processing $file" From 3711b74981ac97cdd95a191ce98e57b25480333c Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 16:22:42 +0100 Subject: [PATCH 116/178] remove typo --- docs/api/generate-api-docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/generate-api-docs.sh b/docs/api/generate-api-docs.sh index 194c2b6c5e..0585e4c7fe 100644 --- a/docs/api/generate-api-docs.sh +++ b/docs/api/generate-api-docs.sh @@ -59,7 +59,7 @@ for release in "${releases[@]}"; do # echo "Processing $file" # replace ..tip:: with note in the python docstrings due to missing directive in the markdown builder - gsed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' "$file"ba + gsed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' "$file" done From 4186bcbd9e473f4baae8498af2c4e09593443707 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 16:27:51 +0100 Subject: [PATCH 117/178] just copy files over and then go back to current branch --- docs/api/generate-api-docs.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/api/generate-api-docs.sh b/docs/api/generate-api-docs.sh index 0585e4c7fe..f30c7f0665 100644 --- a/docs/api/generate-api-docs.sh +++ b/docs/api/generate-api-docs.sh @@ -71,8 +71,13 @@ for release in "${releases[@]}"; do rm -rf "$output_dir/$release" fi sphinx-build -b markdown docs/api/_src "$output_dir/$release" + + # undo all changes + git restore . + + git checkout - # replace :::{seealso} with :::tip in the markdown files - find "$output_dir/$release" -name "*.md" -exec sed -i 's/:::{seealso}/:::tip/g' {} \; + find "$output_dir/$release" -name "*.md" -exec gsed -i 's/:::{seealso}/:::tip/g' {} \; i=1 sp="/-\|" # spinner find "$output_dir/$release" -name "*.md" | while IFS= read -r file; do @@ -86,6 +91,4 @@ for release in "${releases[@]}"; do rm -rf "$output_dir/$release/.doctrees" # run pre-commit to fix any formatting issues on the generated markdown files pre-commit run --files "$output_dir/$release" - # undo all changes - git restore . done From 808dc688fc17602fa6d80e357bdcdebefcd17624 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 17:10:27 +0100 Subject: [PATCH 118/178] fix for lower level emphasis terms --- docs/api/remark.mjs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/api/remark.mjs b/docs/api/remark.mjs index d743b42314..93192c786c 100644 --- a/docs/api/remark.mjs +++ b/docs/api/remark.mjs @@ -30,6 +30,13 @@ function remarkDirectives() { type: "inlineCode", value: node.children[0].value?.trim() + "{:python}", }; + } else if (node.children[0].type === "emphasis") { + child.children.map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + }); } } else if (node.depth === 3) { node.children.map((child) => { From afa0cf06e54358500b5a1b4b0e0f4c5599da0281 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 17:13:26 +0100 Subject: [PATCH 119/178] add debugging --- docs/api/remark.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api/remark.mjs b/docs/api/remark.mjs index 93192c786c..998232f12d 100644 --- a/docs/api/remark.mjs +++ b/docs/api/remark.mjs @@ -31,6 +31,7 @@ function remarkDirectives() { value: node.children[0].value?.trim() + "{:python}", }; } else if (node.children[0].type === "emphasis") { + console.log(node.children); child.children.map((child) => { if (child.type === "text") { child.type = "inlineCode"; From c0b6e19282c3cb3beac0f751bab1d3bdb2fa6234 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 17:15:12 +0100 Subject: [PATCH 120/178] handle emphasis before the rest --- docs/api/remark.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/api/remark.mjs b/docs/api/remark.mjs index 998232f12d..eef3b683b1 100644 --- a/docs/api/remark.mjs +++ b/docs/api/remark.mjs @@ -25,19 +25,19 @@ function remarkDirectives() { } } parent.children[index].children.push({ type: "text", value: "\n:::" }); - } else if (node.children[0].type !== "inlineCode") { - node.children[0] = { - type: "inlineCode", - value: node.children[0].value?.trim() + "{:python}", - }; } else if (node.children[0].type === "emphasis") { console.log(node.children); - child.children.map((child) => { + node.children[0].children.map((child) => { if (child.type === "text") { child.type = "inlineCode"; child.value = child.value?.trim() + "{:python}"; } }); + } else if (node.children[0].type !== "inlineCode") { + node.children[0] = { + type: "inlineCode", + value: node.children[0].value?.trim() + "{:python}", + }; } } else if (node.depth === 3) { node.children.map((child) => { From 8f40d4acf9c43938ad799051b58b82697ce0bc2b Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 17:16:08 +0100 Subject: [PATCH 121/178] remove debugging --- docs/api/remark.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/api/remark.mjs b/docs/api/remark.mjs index eef3b683b1..02fa6b6846 100644 --- a/docs/api/remark.mjs +++ b/docs/api/remark.mjs @@ -26,7 +26,6 @@ function remarkDirectives() { } parent.children[index].children.push({ type: "text", value: "\n:::" }); } else if (node.children[0].type === "emphasis") { - console.log(node.children); node.children[0].children.map((child) => { if (child.type === "text") { child.type = "inlineCode"; From 0f33118b16faed913426710ee3c6e40b8e813b3f Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 17:23:54 +0100 Subject: [PATCH 122/178] convert the whole heading to code --- docs/api/remark.mjs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/api/remark.mjs b/docs/api/remark.mjs index 02fa6b6846..9274321de6 100644 --- a/docs/api/remark.mjs +++ b/docs/api/remark.mjs @@ -32,6 +32,21 @@ function remarkDirectives() { child.value = child.value?.trim() + "{:python}"; } }); + // convert the rest of the heading to inline code + node.children.slice(1).map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + if (child.type === "link") { + child.children.map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + }); + } + }); } else if (node.children[0].type !== "inlineCode") { node.children[0] = { type: "inlineCode", From 714a1aab7da2fbbe5eb070c9e50e7d54d0448e77 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 17:29:02 +0100 Subject: [PATCH 123/178] tidy up remark code --- docs/api/remark.mjs | 121 +++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 59 deletions(-) diff --git a/docs/api/remark.mjs b/docs/api/remark.mjs index 9274321de6..77c7a035d3 100644 --- a/docs/api/remark.mjs +++ b/docs/api/remark.mjs @@ -2,7 +2,7 @@ import fs from "fs"; import { remark } from "remark"; import { visit } from "unist-util-visit"; -function remarkDirectives() { +const remarkDirectives = () => { return transformer; function transformer(tree) { @@ -12,66 +12,23 @@ function remarkDirectives() { function visitor(node, index, parent) { if (node.depth === 4) { - if (["note", "warning"].includes(node.children[0].value?.toLowerCase())) { - const type = node.children[0].value.toLowerCase(); - parent.children.splice(index, 1); - parent.children[index].children[0].value = `:::${type}\n${parent.children[index].children[0].value}`; - // if second to list parent.children[index].children ends with ":", check if the next node is a code block, if so, add the code block as a child to the current node - if (parent.children[index].children.slice(-1)[0]?.value?.trim().endsWith(":")) { - if (parent.children[index + 1].type === "code") { - parent.children[index].children.slice(-1)[0].value += "\n"; - parent.children[index].children.push(parent.children[index + 1]); - parent.children.splice(index + 1, 1); - } - } - parent.children[index].children.push({ type: "text", value: "\n:::" }); - } else if (node.children[0].type === "emphasis") { - node.children[0].children.map((child) => { - if (child.type === "text") { - child.type = "inlineCode"; - child.value = child.value?.trim() + "{:python}"; - } - }); - // convert the rest of the heading to inline code - node.children.slice(1).map((child) => { - if (child.type === "text") { - child.type = "inlineCode"; - child.value = child.value?.trim() + "{:python}"; - } - if (child.type === "link") { - child.children.map((child) => { - if (child.type === "text") { - child.type = "inlineCode"; - child.value = child.value?.trim() + "{:python}"; - } - }); - } - }); - } else if (node.children[0].type !== "inlineCode") { - node.children[0] = { - type: "inlineCode", - value: node.children[0].value?.trim() + "{:python}", - }; + const firstChild = node.children[0]; + if (["note", "warning"].includes(firstChild?.value?.toLowerCase())) { + handleNoteOrWarning(node, index, parent, firstChild); + } else if (firstChild?.type === "emphasis") { + handleEmphasis(node, firstChild); + } else if (firstChild?.type !== "inlineCode") { + handleInlineCode(node, firstChild); } } else if (node.depth === 3) { - node.children.map((child) => { - if (child.type === "text") { - child.type = "inlineCode"; - child.value = child.value?.trim() + "{:python}"; + node.children.forEach((child) => { + if (child.type === "text" || child.type === "emphasis") { + handleInlineCode(node, child); } if (child.type === "link") { - child.children.map((child) => { - if (child.type === "text") { - child.type = "inlineCode"; - child.value = child.value?.trim() + "{:python}"; - } - }); - } - if (child.type === "emphasis") { - child.children.map((child) => { + child.children.forEach((child) => { if (child.type === "text") { - child.type = "inlineCode"; - child.value = child.value?.trim() + "{:python}"; + handleInlineCode(node, child); } }); } @@ -81,13 +38,59 @@ function remarkDirectives() { node.url = node.url.replace(".md", ""); } } -} -let markdown = fs.readFileSync(process.argv[2]); + function handleNoteOrWarning(node, index, parent, firstChild) { + const type = firstChild.value.toLowerCase(); + parent.children.splice(index, 1); + parent.children[index].children[0].value = `:::${type}\n${parent.children[index].children[0].value}`; + + const lastChildValue = parent.children[index].children.slice(-1)[0]?.value?.trim(); + if (lastChildValue?.endsWith(":") && parent.children[index + 1]?.type === "code") { + parent.children[index].children.slice(-1)[0].value += "\n"; + parent.children[index].children.push(parent.children[index + 1]); + parent.children.splice(index + 1, 1); + } + + parent.children[index].children.push({ type: "text", value: "\n:::" }); + } + + function handleEmphasis(node, firstChild) { + firstChild.children.forEach((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = `${child.value?.trim()}{:python}`; + } + }); + + node.children.slice(1).forEach((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = `${child.value?.trim()}{:python}`; + } + if (child.type === "link") { + child.children.forEach((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = `${child.value?.trim()}{:python}`; + } + }); + } + }); + } + + function handleInlineCode(node, firstChild) { + node.children[0] = { + type: "inlineCode", + value: `${firstChild.value?.trim()}{:python}`, + }; + } +}; + +const markdown = fs.readFileSync(process.argv[2]); remark() .use(remarkDirectives) - .process(markdown, function (err, file) { + .process(markdown, (err, file) => { if (err) throw err; fs.writeFileSync(process.argv[2], String(file)); }); From 5dd018d66ecb1c8881b0059be9e638bc2ea168c8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 17:29:51 +0100 Subject: [PATCH 124/178] fix one more index page --- docs/api/_src/subworkflow_lint_tests/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/api/_src/subworkflow_lint_tests/index.md b/docs/api/_src/subworkflow_lint_tests/index.md index 6eed715c52..0ecf590c0d 100644 --- a/docs/api/_src/subworkflow_lint_tests/index.md +++ b/docs/api/_src/subworkflow_lint_tests/index.md @@ -1,9 +1,8 @@ # Subworkflow lint tests ```{toctree} -:caption: 'Tests:' :glob: true -:maxdepth: 2 +:maxdepth: 1 * ``` From fac45e72258199c4a52fc25e187cdffd7b549fa2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 15 Feb 2024 17:35:27 +0100 Subject: [PATCH 125/178] use different remark logic --- docs/api/remark.mjs | 121 +++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 62 deletions(-) diff --git a/docs/api/remark.mjs b/docs/api/remark.mjs index 77c7a035d3..9274321de6 100644 --- a/docs/api/remark.mjs +++ b/docs/api/remark.mjs @@ -2,7 +2,7 @@ import fs from "fs"; import { remark } from "remark"; import { visit } from "unist-util-visit"; -const remarkDirectives = () => { +function remarkDirectives() { return transformer; function transformer(tree) { @@ -12,23 +12,66 @@ const remarkDirectives = () => { function visitor(node, index, parent) { if (node.depth === 4) { - const firstChild = node.children[0]; - if (["note", "warning"].includes(firstChild?.value?.toLowerCase())) { - handleNoteOrWarning(node, index, parent, firstChild); - } else if (firstChild?.type === "emphasis") { - handleEmphasis(node, firstChild); - } else if (firstChild?.type !== "inlineCode") { - handleInlineCode(node, firstChild); + if (["note", "warning"].includes(node.children[0].value?.toLowerCase())) { + const type = node.children[0].value.toLowerCase(); + parent.children.splice(index, 1); + parent.children[index].children[0].value = `:::${type}\n${parent.children[index].children[0].value}`; + // if second to list parent.children[index].children ends with ":", check if the next node is a code block, if so, add the code block as a child to the current node + if (parent.children[index].children.slice(-1)[0]?.value?.trim().endsWith(":")) { + if (parent.children[index + 1].type === "code") { + parent.children[index].children.slice(-1)[0].value += "\n"; + parent.children[index].children.push(parent.children[index + 1]); + parent.children.splice(index + 1, 1); + } + } + parent.children[index].children.push({ type: "text", value: "\n:::" }); + } else if (node.children[0].type === "emphasis") { + node.children[0].children.map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + }); + // convert the rest of the heading to inline code + node.children.slice(1).map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + if (child.type === "link") { + child.children.map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + }); + } + }); + } else if (node.children[0].type !== "inlineCode") { + node.children[0] = { + type: "inlineCode", + value: node.children[0].value?.trim() + "{:python}", + }; } } else if (node.depth === 3) { - node.children.forEach((child) => { - if (child.type === "text" || child.type === "emphasis") { - handleInlineCode(node, child); + node.children.map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; } if (child.type === "link") { - child.children.forEach((child) => { + child.children.map((child) => { + if (child.type === "text") { + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; + } + }); + } + if (child.type === "emphasis") { + child.children.map((child) => { if (child.type === "text") { - handleInlineCode(node, child); + child.type = "inlineCode"; + child.value = child.value?.trim() + "{:python}"; } }); } @@ -38,59 +81,13 @@ const remarkDirectives = () => { node.url = node.url.replace(".md", ""); } } +} - function handleNoteOrWarning(node, index, parent, firstChild) { - const type = firstChild.value.toLowerCase(); - parent.children.splice(index, 1); - parent.children[index].children[0].value = `:::${type}\n${parent.children[index].children[0].value}`; - - const lastChildValue = parent.children[index].children.slice(-1)[0]?.value?.trim(); - if (lastChildValue?.endsWith(":") && parent.children[index + 1]?.type === "code") { - parent.children[index].children.slice(-1)[0].value += "\n"; - parent.children[index].children.push(parent.children[index + 1]); - parent.children.splice(index + 1, 1); - } - - parent.children[index].children.push({ type: "text", value: "\n:::" }); - } - - function handleEmphasis(node, firstChild) { - firstChild.children.forEach((child) => { - if (child.type === "text") { - child.type = "inlineCode"; - child.value = `${child.value?.trim()}{:python}`; - } - }); - - node.children.slice(1).forEach((child) => { - if (child.type === "text") { - child.type = "inlineCode"; - child.value = `${child.value?.trim()}{:python}`; - } - if (child.type === "link") { - child.children.forEach((child) => { - if (child.type === "text") { - child.type = "inlineCode"; - child.value = `${child.value?.trim()}{:python}`; - } - }); - } - }); - } - - function handleInlineCode(node, firstChild) { - node.children[0] = { - type: "inlineCode", - value: `${firstChild.value?.trim()}{:python}`, - }; - } -}; - -const markdown = fs.readFileSync(process.argv[2]); +let markdown = fs.readFileSync(process.argv[2]); remark() .use(remarkDirectives) - .process(markdown, (err, file) => { + .process(markdown, function (err, file) { if (err) throw err; fs.writeFileSync(process.argv[2], String(file)); }); From 2c11e4b5e752ecf33a896b2ccc8b91d540b89e1e Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 11:05:59 +0100 Subject: [PATCH 126/178] fix api index page --- docs/api/_src/api/index.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/api/_src/api/index.md b/docs/api/_src/api/index.md index 9050280fd3..a1863f7e39 100644 --- a/docs/api/_src/api/index.md +++ b/docs/api/_src/api/index.md @@ -1,9 +1,8 @@ # API Reference ```{toctree} -:caption: 'Tests:' :glob: true -:maxdepth: 2 +:maxdepth: 1 * ``` From c393ed4bb383e3118a8f5786f542b0b7cc48a1df Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 12:53:24 +0100 Subject: [PATCH 127/178] template: add hashtags to release announcement on mastodon --- .../.github/workflows/release-announcements.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index f293032771..c360ca41e3 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -9,6 +9,11 @@ jobs: toot: runs-on: ubuntu-latest steps: + - name: get topics and convert to hashtags + id: get_topics + run: | + curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' > $GITHUB_OUTPUT + - uses: rzr/fediverse-action@master with: access-token: ${{ secrets.MASTODON_ACCESS_TOKEN }} @@ -20,6 +25,8 @@ jobs: Please see the changelog: ${{ github.event.release.html_url }} + ${{ steps.get_topics.outputs.GITHUB_OUTPUT }} #nfcore #openscience + send-tweet: runs-on: ubuntu-latest From 8011af45619ae2b7f1ca8d735dad96b628210d54 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 16 Feb 2024 11:54:59 +0000 Subject: [PATCH 128/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cdb73d30b..bed74d45da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Add empty line in README.md to fix badges. ([#2729](https://github.com/nf-core/tools/pull/2729)) - Replace automatic branch detection in `nf-core download` CI test with hardcoded `dev` and input. ([#2727](https://github.com/nf-core/tools/pull/2727)) +- add hashtags to release announcement on mastodon ([#2761](https://github.com/nf-core/tools/pull/2761)) ### Linting From c1f06eaaecc2dd1b2721f3b9b7fc07381f8b1878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 16 Feb 2024 13:02:51 +0100 Subject: [PATCH 129/178] Apply suggestions from code review Co-authored-by: James A. Fellows Yates --- CHANGELOG.md | 2 +- .../.github/workflows/release-announcements.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95c9adba3d..500f709c23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ - Replace automatic branch detection in `nf-core download` CI test with hardcoded `dev` and input. ([#2727](https://github.com/nf-core/tools/pull/2727)) - Add Github Action to automatically cleanup ubuntu-latest runners to fix runner running out of diskspace errors([#2755](https://github.com/nf-core/tools/issues/2755)) - Fix GitHub Actions CI and Linting badges links ([#2757](https://github.com/nf-core/tools/pull/2757)) -- add hashtags to release announcement on mastodon ([#2761](https://github.com/nf-core/tools/pull/2761)) +- Add hashtags to release announcement on mastodon ([#2761](https://github.com/nf-core/tools/pull/2761)) ### Linting diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index c360ca41e3..6d11280ae8 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -25,7 +25,7 @@ jobs: Please see the changelog: ${{ github.event.release.html_url }} - ${{ steps.get_topics.outputs.GITHUB_OUTPUT }} #nfcore #openscience + ${{ steps.get_topics.outputs.GITHUB_OUTPUT }} #nfcore #openscience #nextflow #bioinformatics send-tweet: runs-on: ubuntu-latest From d9f8443eac2421042144d3a7d1fe7d953a7cd284 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 13:03:13 +0100 Subject: [PATCH 130/178] add profile parameter to nf-test command fixes #2753 --- nf_core/__main__.py | 20 ++++++++++++++++++-- nf_core/components/components_test.py | 7 ++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index a39c3cf732..9387236b9a 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1134,7 +1134,14 @@ def create_module( default=False, help="Run tests only once. Don't check snapshot stability", ) -def test_module(ctx, tool, dir, no_prompts, update, once): +@click.option( + "--profile", + type=str, + default=None, + help="Run tests with a specific profile", + options=["docker", "singularity", "conda"], +) +def test_module(ctx, tool, dir, no_prompts, update, once, profile): """ Run nf-test for a module. @@ -1153,6 +1160,7 @@ def test_module(ctx, tool, dir, no_prompts, update, once): remote_url=ctx.obj["modules_repo_url"], branch=ctx.obj["modules_repo_branch"], verbose=ctx.obj["verbose"], + profile=profile, ) module_tester.run() except (UserWarning, LookupError) as e: @@ -1398,7 +1406,14 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): default=False, help="Run tests only once. Don't check snapshot stability", ) -def test_subworkflow(ctx, subworkflow, dir, no_prompts, update, once): +@click.option( + "--profile", + type=str, + default=None, + help="Run tests with a specific profile", + options=["docker", "singularity", "conda"], +) +def test_subworkflow(ctx, subworkflow, dir, no_prompts, update, once, profile): """ Run nf-test for a subworkflow. @@ -1417,6 +1432,7 @@ def test_subworkflow(ctx, subworkflow, dir, no_prompts, update, once): remote_url=ctx.obj["modules_repo_url"], branch=ctx.obj["modules_repo_branch"], verbose=ctx.obj["verbose"], + profile=profile, ) sw_tester.run() except (UserWarning, LookupError) as e: diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 3294c2878b..a94441ae8d 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -48,6 +48,8 @@ class ComponentsTest(ComponentCommand): # type: ignore[misc] flag indicating if the existing snapshot should be updated once : bool flag indicating if the test should be run only once + profile : str + container software to use (docker, singularity or conda) Methods ------- @@ -72,6 +74,7 @@ def __init__( verbose: bool = False, update: bool = False, once: bool = False, + profile: Optional[str] = None, ): super().__init__(component_type, directory, remote_url, branch, no_prompts=no_prompts) self.component_name = component_name @@ -82,6 +85,7 @@ def __init__( self.obsolete_snapshots: bool = False self.update = update self.once = once + self.profile = profile def run(self) -> None: """Run build steps""" @@ -190,10 +194,11 @@ def generate_snapshot(self) -> bool: update = "--update-snapshot" if self.update else "" self.update = False # reset self.update to False to test if the new snapshot is stable tag = f"subworkflows/{self.component_name}" if self.component_type == "subworkflows" else self.component_name + profile = self.profile if self.profile else os.environ["PROFILE"] result = nf_core.utils.run_cmd( "nf-test", - f"test --tag {tag} --profile {os.environ['PROFILE']} {verbose} {update}", + f"test --tag {tag} --profile {profile} {verbose} {update}", ) if result is not None: nftest_out, nftest_err = result From 997d2741c2ec1af7dc16bd9b318a2ccb98abcb39 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 14:47:04 +0100 Subject: [PATCH 131/178] handle multiple module imports in linting closes #2726 --- nf_core/subworkflows/lint/main_nf.py | 24 +++++--- tests/subworkflows/lint.py | 85 ++++++++++++++++++++++++++++ tests/test_subworkflows.py | 3 + 3 files changed, 104 insertions(+), 8 deletions(-) diff --git a/nf_core/subworkflows/lint/main_nf.py b/nf_core/subworkflows/lint/main_nf.py index f59e1e4279..c73559502c 100644 --- a/nf_core/subworkflows/lint/main_nf.py +++ b/nf_core/subworkflows/lint/main_nf.py @@ -4,6 +4,7 @@ import logging import re +from typing import List log = logging.getLogger(__name__) @@ -120,7 +121,7 @@ def check_main_section(self, lines, included_components): # Check that all included components are used # Check that all included component versions are used - if included_components is not None: + if included_components: for component in included_components: if component in script: self.passed.append( @@ -152,7 +153,7 @@ def check_main_section(self, lines, included_components): ) -def check_subworkflow_section(self, lines): +def check_subworkflow_section(self, lines: List[str]) -> List[str]: """Lint the section of a subworkflow before the workflow definition Specifically checks if the subworkflow includes at least two modules or subworkflows @@ -160,7 +161,7 @@ def check_subworkflow_section(self, lines): lines (List[str]): Content of subworkflow. Returns: - List: List of included component names. If subworkflow doesn't contain any lines, return None. + List[str]: List of included components. """ # Check that we have subworkflow content if len(lines) == 0: @@ -171,7 +172,7 @@ def check_subworkflow_section(self, lines): self.main_nf, ) ) - return + return [] self.passed.append( ("subworkflow_include", "Subworkflow does include modules before the workflow definition", self.main_nf) ) @@ -179,10 +180,17 @@ def check_subworkflow_section(self, lines): includes = [] for line in lines: if line.strip().startswith("include"): - component_name = line.split("{")[1].split("}")[0].strip() - if " as " in component_name: - component_name = component_name.split(" as ")[1].strip() - includes.append(component_name) + component_name = [line.split("{")[1].split("}")[0].strip()] + # check if multiple components are included + if ";" in component_name[0]: + component_name = component_name[0].split(";") + for comp in component_name: + if " as " in comp: + comp = comp.split(" as ")[1].strip() + includes.append(comp) + continue + # remove duplicated components + includes = list(set(includes)) if len(includes) >= 2: self.passed.append(("main_nf_include", "Subworkflow includes two or more modules", self.main_nf)) else: diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 8804f8bf6f..8c72870756 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -104,3 +104,88 @@ def test_subworkflows_lint_snapshot_file_not_needed(self): assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 + + +def test_subworkflows_lint_less_than_two_modules_warning(self): + """Test linting a subworkflow with less than two modules""" + self.subworkflow_install.install("bam_stats_samtools") + # Remove two modules + with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf")) as fh: + content = fh.read() + new_content = content.replace( + "include { SAMTOOLS_IDXSTATS } from '../../../modules/nf-core/samtools/idxstats/main'", "" + ) + new_content = new_content.replace( + "include { SAMTOOLS_FLAGSTAT } from '../../../modules/nf-core/samtools/flagstat/main'", "" + ) + with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf"), "w") as fh: + fh.write(new_content) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, subworkflow="bam_stats_samtools") + assert len(subworkflow_lint.failed) >= 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) > 0 + assert subworkflow_lint.warned[0].lint_test == "main_nf_include" + # cleanup + self.subworkflow_remove.remove("bam_stats_samtools", force=True) + + +def test_subworkflows_lint_include_multiple_alias(self): + """Test linting a subworkflow with multiple include methods""" + self.subworkflow_install.install("bam_stats_samtools") + with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf")) as fh: + content = fh.read() + new_content = content.replace( + "include { SAMTOOLS_STATS }", + "include { SAMTOOLS_STATS as SAMTOOLS_STATS_1; SAMTOOLS_STATS as SAMTOOLS_STATS_2 }", + ) + with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf"), "w") as fh: + fh.write(new_content) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, subworkflow="bam_stats_samtools") + assert len(subworkflow_lint.failed) >= 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) == 0 + # assert that one message mentions SAMTOOLS_STATS but not SAMTOOLS_STATS_1 or SAMTOOLS_STATS_2 + assert any( + [ + x.message == "Included component 'SAMTOOLS_STATS' versions are added in main.nf" + for x in subworkflow_lint.passed + ] + ) + assert not any( + [ + x.message == "Included component 'SAMTOOLS_STATS_1' versions are added in main.nf" + for x in subworkflow_lint.passed + ] + ) + assert not any( + [ + x.message == "Included component 'SAMTOOLS_STATS_2' versions are added in main.nf" + for x in subworkflow_lint.passed + ] + ) + + # cleanup + self.subworkflow_remove.remove("bam_stats_samtools", force=True) + + +def test_subworkflows_lint_capitalization_fail(self): + """Test linting a subworkflow with a capitalization fail""" + self.subworkflow_install.install("bam_stats_samtools") + # change workflow name to lowercase + with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf")) as fh: + content = fh.read() + new_content = content.replace("workflow BAM_STATS_SAMTOOLS {", "workflow bam_stats_samtools {") + with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf"), "w") as fh: + fh.write(new_content) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, subworkflow="bam_stats_samtools") + assert len(subworkflow_lint.failed) >= 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert any([x.lint_test == "workflow_capitals" for x in subworkflow_lint.failed]) + + # cleanup + self.subworkflow_remove.remove("bam_stats_samtools", force=True) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 19872ee168..cd0bd21146 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -121,8 +121,11 @@ def tearDown(self): ) from .subworkflows.lint import ( # type: ignore[misc] test_subworkflows_lint, + test_subworkflows_lint_capitalization_fail, test_subworkflows_lint_empty, test_subworkflows_lint_gitlab_subworkflows, + test_subworkflows_lint_include_multiple_alias, + test_subworkflows_lint_less_than_two_modules_warning, test_subworkflows_lint_multiple_remotes, test_subworkflows_lint_new_subworkflow, test_subworkflows_lint_no_gitlab, From 27da6a609f904c54fba6df35c092424ece7954b1 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 16 Feb 2024 13:50:17 +0000 Subject: [PATCH 132/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a37e6e5ff7..856fd11987 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ ### Modules - Handle dirty local module repos by force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) +- Linting: Handle multiple aliases in module imports correctly during linting ([#2762](https://github.com/nf-core/tools/pull/2762)) ### General From 243ec6437efa1bd58724f482c50a1a3d15df1b7b Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 14:51:35 +0100 Subject: [PATCH 133/178] fix changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 856fd11987..6f349101d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,13 @@ ### Linting -- make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733)) -- add looser comparison when pipeline logos ([#2744](https://github.com/nf-core/tools/pull/2744)) +- Make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733)) +- Add looser comparison when pipeline logos ([#2744](https://github.com/nf-core/tools/pull/2744)) +- Handle multiple aliases in module imports correctly during linting ([#2762](https://github.com/nf-core/tools/pull/2762)) ### Modules - Handle dirty local module repos by force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) -- Linting: Handle multiple aliases in module imports correctly during linting ([#2762](https://github.com/nf-core/tools/pull/2762)) ### General From a84079050ba28d85e078e918a9a7e4b74f44ec38 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 16:13:08 +0100 Subject: [PATCH 134/178] fix tests --- tests/subworkflows/lint.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 8c72870756..b89b7b78ce 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -135,9 +135,10 @@ def test_subworkflows_lint_include_multiple_alias(self): self.subworkflow_install.install("bam_stats_samtools") with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf")) as fh: content = fh.read() - new_content = content.replace( - "include { SAMTOOLS_STATS }", - "include { SAMTOOLS_STATS as SAMTOOLS_STATS_1; SAMTOOLS_STATS as SAMTOOLS_STATS_2 }", + new_content = content.replace("SAMTOOLS_STATS", "SAMTOOLS_STATS_1") + new_content = new_content.replace( + "include { SAMTOOLS_STATS_1 ", + "include { SAMTOOLS_STATS as SAMTOOLS_STATS_1; SAMTOOLS_STATS as SAMTOOLS_STATS_2 ", ) with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_stats_samtools", "main.nf"), "w") as fh: fh.write(new_content) @@ -146,25 +147,16 @@ def test_subworkflows_lint_include_multiple_alias(self): subworkflow_lint.lint(print_results=False, subworkflow="bam_stats_samtools") assert len(subworkflow_lint.failed) >= 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 - assert len(subworkflow_lint.warned) == 0 - # assert that one message mentions SAMTOOLS_STATS but not SAMTOOLS_STATS_1 or SAMTOOLS_STATS_2 + assert len(subworkflow_lint.warned) == 2 assert any( - [ - x.message == "Included component 'SAMTOOLS_STATS' versions are added in main.nf" - for x in subworkflow_lint.passed - ] - ) - assert not any( [ x.message == "Included component 'SAMTOOLS_STATS_1' versions are added in main.nf" for x in subworkflow_lint.passed ] ) - assert not any( - [ - x.message == "Included component 'SAMTOOLS_STATS_2' versions are added in main.nf" - for x in subworkflow_lint.passed - ] + assert any([x.message == "Included component 'SAMTOOLS_STATS_1' used in main.nf" for x in subworkflow_lint.passed]) + assert any( + [x.message == "Included component 'SAMTOOLS_STATS_2' not used in main.nf" for x in subworkflow_lint.warned] ) # cleanup From 1d04268c972873c058ca65ede5310f6be5f309c8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 17:09:24 +0100 Subject: [PATCH 135/178] add prompt for extra files --- nf_core/components/update.py | 37 ++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 077cb2b840..545fd9ebad 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -5,6 +5,7 @@ from pathlib import Path import questionary +import rich.prompt import nf_core.modules.modules_utils import nf_core.utils @@ -76,7 +77,7 @@ def _parameter_checks(self): if not self.has_valid_directory(): raise UserWarning("The command was not run in a valid pipeline directory.") - def update(self, component=None, silent=False, updated=None, check_diff_exist=True): + def update(self, component=None, silent=False, updated=None, check_diff_exist=True) -> bool: """Updates a specified module/subworkflow or all modules/subworkflows in a pipeline. If updating a subworkflow: updates all modules used in that subworkflow. @@ -188,7 +189,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr continue # Download component files - if not self.install_component_files(component, version, modules_repo, install_tmp_dir): + if not self.install_component_files(component, version, modules_repo, str(install_tmp_dir)): exit_value = False continue @@ -282,7 +283,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr if not dry_run: # Clear the component directory and move the installed files there - self.move_files_from_tmp_dir(component, install_tmp_dir, modules_repo.repo_path, version) + self.move_files_from_tmp_dir(component, str(install_tmp_dir), modules_repo.repo_path, version) # Update modules.json with newly installed component self.modules_json.update(self.component_type, modules_repo, component, version, installed_by=None) updated.append(component) @@ -727,7 +728,7 @@ def setup_diff_file(self, check_diff_exist=True): # This guarantees that the file exists after calling the function self.save_diff_fn.touch() - def move_files_from_tmp_dir(self, component, install_folder, repo_path, new_version): + def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path: str, new_version: str) -> None: """Move the files from the temporary to the installation directory. Args: @@ -736,18 +737,34 @@ def move_files_from_tmp_dir(self, component, install_folder, repo_path, new_vers repo_path (str): The name of the directory where modules/subworkflows are installed new_version (str): The version of the module/subworkflow that was installed. """ - temp_component_dir = os.path.join(install_folder, component) - files = os.listdir(temp_component_dir) - pipeline_path = os.path.join(self.dir, self.component_type, repo_path, component) + temp_component_dir = Path(install_folder, component) + files = [f.name for f in temp_component_dir.iterdir() if f.is_file()] + pipeline_path = Path(self.dir, self.component_type, repo_path, component) log.debug(f"Removing old version of {self.component_type[:-1]} '{component}'") - self.clear_component_dir(component, pipeline_path) + # check if both directories have the same file and warn if the compnent_dir has more files + if pipeline_path.exists(): + pipeline_files = [f.name for f in pipeline_path.iterdir() if f.is_file()] + if pipeline_files: + # ask if the user wants to move the file from the current directory to the new one + for file in pipeline_files: + if file not in files: + # use prompt to ask the user if they want to move the file + if rich.prompt.Confirm().ask( + f"File '{file}' exists in the current {self.component_type[:-1]} '{component}' directory. Do you want to keep it in the updated version?" + ): + shutil.move(Path(pipeline_path, file), Path(temp_component_dir, file)) + files.append(file) + + else: + log.debug(f"Creating new {self.component_type[:-1]} '{component}' in '{self.component_type}/{repo_path}'") + self.clear_component_dir(component, str(pipeline_path)) os.makedirs(pipeline_path) for file in files: - path = os.path.join(temp_component_dir, file) + path = Path(temp_component_dir, file) if os.path.exists(path): - shutil.move(path, os.path.join(pipeline_path, file)) + shutil.move(path, Path(pipeline_path, file)) log.info(f"Updating '{repo_path}/{component}'") log.debug(f"Updating {self.component_type[:-1]} '{component}' to {new_version} from {repo_path}") From 1fa125ddad8fe022e5c99ca12d18f6b9a4766d76 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 16 Feb 2024 16:21:29 +0000 Subject: [PATCH 136/178] chore(deps): update actions/github-script action to v7 --- .github/workflows/tools-api-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tools-api-docs.yml b/.github/workflows/tools-api-docs.yml index 60045512d4..7642a3f1ae 100644 --- a/.github/workflows/tools-api-docs.yml +++ b/.github/workflows/tools-api-docs.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest steps: - name: trigger API docs build - uses: actions/github-script@v6 + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 with: github-token: ${{ secrets.nf_core_bot_auth_token }} script: | From 74e6ad71f88b16aa5eaa37310d9437bc3c86f9f5 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 16 Feb 2024 16:22:36 +0000 Subject: [PATCH 137/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index faec265c93..f73292a52a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ - Update actions/setup-python action to v5 ([#2739](https://github.com/nf-core/tools/pull/2739)) - Update gitpod/workspace-base Docker digest to 45e7617 ([#2747](https://github.com/nf-core/tools/pull/2747)) - chore(deps): pin jlumbroso/free-disk-space action to 54081f1 ([#2756](https://github.com/nf-core/tools/pull/2756)) +- chore(deps): update actions/github-script action to v7 ([#2766](https://github.com/nf-core/tools/pull/2766)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From bb7e6388cea12c4d649b6f4f2c1f142d6038e87f Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 16 Feb 2024 16:30:24 +0000 Subject: [PATCH 138/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a37e6e5ff7..ba3f01e79b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ ### Modules - Handle dirty local module repos by force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) +- Add `--profile` parameter to nf-test command ([#2767](https://github.com/nf-core/tools/pull/2767)) ### General From fba4272e6c2170b0f50180a0314d9995d34589dc Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 17:54:31 +0100 Subject: [PATCH 139/178] fix tests --- nf_core/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9387236b9a..804f024136 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1136,10 +1136,9 @@ def create_module( ) @click.option( "--profile", - type=str, + type=click.Choice(["docker", "singularity", "conda"]), default=None, help="Run tests with a specific profile", - options=["docker", "singularity", "conda"], ) def test_module(ctx, tool, dir, no_prompts, update, once, profile): """ From 6a7980d65537e5d9fde90adb3dc891de2fce2e3f Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 17:56:17 +0100 Subject: [PATCH 140/178] fix all occurences --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 804f024136..9d7291694a 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1407,7 +1407,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): ) @click.option( "--profile", - type=str, + type=click.Choice(["none", "singularity"]), default=None, help="Run tests with a specific profile", options=["docker", "singularity", "conda"], From 75f6f19cef0fadcd7c23bbcf66692fd52dc04942 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 17:59:31 +0100 Subject: [PATCH 141/178] remove options --- nf_core/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9d7291694a..7d2d083fa9 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1410,7 +1410,6 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): type=click.Choice(["none", "singularity"]), default=None, help="Run tests with a specific profile", - options=["docker", "singularity", "conda"], ) def test_subworkflow(ctx, subworkflow, dir, no_prompts, update, once, profile): """ From 31dc4376948cf6cd7c7c6b4d34ae226b8788655b Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 16 Feb 2024 22:20:52 +0100 Subject: [PATCH 142/178] simplify logic and add tests --- nf_core/components/update.py | 18 ++++++------------ tests/modules/update.py | 18 ++++++++++++++++++ tests/test_modules.py | 1 + 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 545fd9ebad..841ed45659 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -5,7 +5,6 @@ from pathlib import Path import questionary -import rich.prompt import nf_core.modules.modules_utils import nf_core.utils @@ -741,23 +740,18 @@ def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path files = [f.name for f in temp_component_dir.iterdir() if f.is_file()] pipeline_path = Path(self.dir, self.component_type, repo_path, component) - log.debug(f"Removing old version of {self.component_type[:-1]} '{component}'") # check if both directories have the same file and warn if the compnent_dir has more files if pipeline_path.exists(): pipeline_files = [f.name for f in pipeline_path.iterdir() if f.is_file()] - if pipeline_files: - # ask if the user wants to move the file from the current directory to the new one - for file in pipeline_files: - if file not in files: - # use prompt to ask the user if they want to move the file - if rich.prompt.Confirm().ask( - f"File '{file}' exists in the current {self.component_type[:-1]} '{component}' directory. Do you want to keep it in the updated version?" - ): - shutil.move(Path(pipeline_path, file), Path(temp_component_dir, file)) - files.append(file) + if "nextflow.config" in pipeline_files: + log.debug(f"Moving '{component}/nextflow.config' to updated component") + shutil.move(Path(pipeline_path, "nextflow.config"), Path(temp_component_dir, "nextflow.config")) + files.append("nextflow.config") else: log.debug(f"Creating new {self.component_type[:-1]} '{component}' in '{self.component_type}/{repo_path}'") + + log.debug(f"Removing old version of {self.component_type[:-1]} '{component}'") self.clear_component_dir(component, str(pipeline_path)) os.makedirs(pipeline_path) diff --git a/tests/modules/update.py b/tests/modules/update.py index 5cf24b56f0..80cd5c2eed 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -376,3 +376,21 @@ def cmp_module(dir1, dir2): """Compare two versions of the same module""" files = ["main.nf", "meta.yml"] return all(filecmp.cmp(os.path.join(dir1, f), os.path.join(dir2, f), shallow=False) for f in files) + + +def test_update_module_with_nextflow_config(self): + """Try updating a module with a nextflow.config file""" + # Install the module + assert self.mods_install.install("trimgalore") + # Add a nextflow.config file to the module + trimgalore_path = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + Path(trimgalore_path, "nextflow.config").touch() + with open(Path(trimgalore_path, "nextflow.config"), "w") as fh: + fh.write("params.my_param = 'my_value'\n") + # Update the module + update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False) + assert update_obj.update("trimgalore") + # Check that the nextflow.config file is still there + assert Path(trimgalore_path, "nextflow.config").exists() + with open(Path(trimgalore_path, "nextflow.config")) as fh: + assert "params.my_param = 'my_value'" in fh.read() diff --git a/tests/test_modules.py b/tests/test_modules.py index f9c3b6f2a7..796b1d2538 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -259,6 +259,7 @@ def test_modulesrepo_class(self): test_update_different_branch_mix_modules_branch_test, test_update_different_branch_mixed_modules_main, test_update_different_branch_single_module, + test_update_module_with_nextflow_config, test_update_only_show_differences, test_update_only_show_differences_when_patch, test_update_with_config_dont_update, From edc1e2610205add41233911dfdf3057e7d8d94c6 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 14:13:55 +0000 Subject: [PATCH 143/178] chore(deps): update dawidd6/action-download-artifact digest to f6b0bac --- nf_core/pipeline-template/.github/workflows/linting_comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index c49fb53955..bb3eafcc2c 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d # v3 + uses: dawidd6/action-download-artifact@f6b0bace624032e30a85a8fd9c1a7f8f611f5737 # v3 with: workflow: linting.yml workflow_conclusion: completed From 9bbf4cd8df15346a017163f424f465733634001d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 18 Feb 2024 00:10:02 +0000 Subject: [PATCH 144/178] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.2.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9bb8c3df95..03fbb7bedf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.1 + rev: v0.2.2 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From df9b5e2bdf7ceb56cb40dffe92dfa3acd6d198fc Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sun, 18 Feb 2024 00:10:44 +0000 Subject: [PATCH 145/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f73292a52a..6056307dbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - Update gitpod/workspace-base Docker digest to 45e7617 ([#2747](https://github.com/nf-core/tools/pull/2747)) - chore(deps): pin jlumbroso/free-disk-space action to 54081f1 ([#2756](https://github.com/nf-core/tools/pull/2756)) - chore(deps): update actions/github-script action to v7 ([#2766](https://github.com/nf-core/tools/pull/2766)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.2.2 ([#2769](https://github.com/nf-core/tools/pull/2769)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From 1baba7cf8ddc2eb0135021cfd51eef88cd06992c Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Sun, 18 Feb 2024 10:53:26 +0100 Subject: [PATCH 146/178] handle symlink when migrating pytest --- nf_core/components/create.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 32f6d1a433..c4b477a0ab 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -462,7 +462,13 @@ def _print_and_delete_pytest_files(self): ): with open(pytest_dir / "main.nf") as fh: log.info(fh.read()) - shutil.rmtree(pytest_dir) + if pytest_dir.is_symlink(): + resolved_dir = pytest_dir.resolve() + log.debug(f"Removing symlink: {resolved_dir}") + shutil.rmtree(resolved_dir) + pytest_dir.unlink() + else: + shutil.rmtree(pytest_dir) log.info( "[yellow]Please convert the pytest tests to nf-test in 'main.nf.test'.[/]\n" "You can find more information about nf-test [link=https://nf-co.re/docs/contributing/modules#migrating-from-pytest-to-nf-test]at the nf-core web[/link]. " From ea614e1384729a9fb867ab465d162f28848ccc87 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sun, 18 Feb 2024 09:55:08 +0000 Subject: [PATCH 147/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6056307dbf..deb5131bfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ ### Modules - Handle dirty local module repos by force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) +- Handle symlinks when migrating pytest ([#2770](https://github.com/nf-core/tools/pull/2770)) ### General From 3e1ca3d52e14c1b78e3d30666b60ad8edb0f2671 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Sun, 18 Feb 2024 11:22:55 +0100 Subject: [PATCH 148/178] patch: handle file not found when it is an added file to a module --- nf_core/modules/modules_differ.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_differ.py b/nf_core/modules/modules_differ.py index a97229ff62..dc2b163dd4 100644 --- a/nf_core/modules/modules_differ.py +++ b/nf_core/modules/modules_differ.py @@ -447,8 +447,12 @@ def try_apply_patch(module, repo_path, patch_path, module_dir, reverse=False): log.debug(f"Applying patch to {file}") fn = Path(file).relative_to(module_relpath) file_path = module_dir / fn - with open(file_path) as fh: - file_lines = fh.readlines() + try: + with open(file_path) as fh: + file_lines = fh.readlines() + except FileNotFoundError: + # The file was added with the patch + file_lines = [""] patched_new_lines = ModulesDiffer.try_apply_single_patch(file_lines, patch, reverse=reverse) new_files[str(fn)] = patched_new_lines return new_files From 30b3d5169b6a3d06f53ba93e46252edb2c5f4c11 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sun, 18 Feb 2024 10:25:30 +0000 Subject: [PATCH 149/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6056307dbf..adbc9c51c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ ### Modules - Handle dirty local module repos by force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) +- Moduels: patch: handle file not found when it is an added file to a module ([#2771](https://github.com/nf-core/tools/pull/2771)) ### General From a61549efc8eff419cab213d1649bd4e51c376d31 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 08:41:32 +0100 Subject: [PATCH 150/178] add test for symlink deletion --- tests/modules/create.py | 25 +++++++++++++++++++++++++ tests/test_modules.py | 1 + 2 files changed, 26 insertions(+) diff --git a/tests/modules/create.py b/tests/modules/create.py index 460a1439cb..cf39621f07 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -138,3 +138,28 @@ def test_modules_migrate_no_delete(self, mock_rich_ask): with open(Path(self.nfcore_modules, "tests", "config", "pytest_modules.yml")) as fh: modules_yml = yaml.safe_load(fh) assert "samtools/sort" not in modules_yml.keys() + + +@mock.patch("rich.prompt.Confirm.ask") +def test_modules_migrate_symlink(self, mock_rich_ask): + """Create a module with the --migrate-pytest option to convert pytest with symlinks to nf-test. + Test that the symlink is deleted and the file is copied.""" + + pytest_dir = Path(self.nfcore_modules, "tests", "modules", "nf-core", "samtools", "sort") + module_dir = Path(self.nfcore_modules, "modules", "nf-core", "samtools", "sort") + + # Clone modules repo with pytests + shutil.rmtree(self.nfcore_modules) + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH) + + # Create a symlinked file in the pytest directory + symlink_file = pytest_dir / "symlink_file.txt" + symlink_file.symlink_to(module_dir / "main.nf") + + # Create a module with --migrate-pytest + mock_rich_ask.return_value = True + module_create = nf_core.modules.ModuleCreate(self.nfcore_modules, "samtools/sort", migrate_pytest=True) + module_create.create() + + # Check that symlink is deleted + assert not symlink_file.is_symlink() diff --git a/tests/test_modules.py b/tests/test_modules.py index f9c3b6f2a7..807645a4b9 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -159,6 +159,7 @@ def test_modulesrepo_class(self): test_modules_create_succeed, test_modules_migrate, test_modules_migrate_no_delete, + test_modules_migrate_symlink, ) from .modules.info import ( # type: ignore[misc] test_modules_info_in_modules_repo, From d9bb78ad38243c367154e06be3932c11bc25d986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Mon, 19 Feb 2024 08:42:38 +0100 Subject: [PATCH 151/178] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adbc9c51c6..9939ede081 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ ### Modules - Handle dirty local module repos by force checkout of commits and branches if needed ([#2734](https://github.com/nf-core/tools/pull/2734)) -- Moduels: patch: handle file not found when it is an added file to a module ([#2771](https://github.com/nf-core/tools/pull/2771)) +- Patch: handle file not found when it is an added file to a module ([#2771](https://github.com/nf-core/tools/pull/2771)) ### General From 6cff41c9daa66f8d13049db3c6a78b2d17b16763 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 09:43:38 +0100 Subject: [PATCH 152/178] fix github action syntax, remove now unneeded makefile [skip changelog] --- .github/workflows/tools-api-docs.yml | 2 +- docs/api/Makefile | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 docs/api/Makefile diff --git a/.github/workflows/tools-api-docs.yml b/.github/workflows/tools-api-docs.yml index 7642a3f1ae..539c41d119 100644 --- a/.github/workflows/tools-api-docs.yml +++ b/.github/workflows/tools-api-docs.yml @@ -36,6 +36,6 @@ jobs: workflow_id: 'add-tools-api-docs.yml', ref: 'main' inputs: { - "ref_name": ${{ inputs.ref_name || github.ref_name }} + "ref_name": "${{ inputs.ref_name || github.ref_name }}" } }) diff --git a/docs/api/Makefile b/docs/api/Makefile deleted file mode 100644 index ab30a5051e..0000000000 --- a/docs/api/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SOURCEDIR = _src -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) From 7dcdd2f0b1ee82fbf6227fcc220d0d6c9c64b273 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 09:48:33 +0100 Subject: [PATCH 153/178] fix action name --- .github/workflows/tools-api-docs.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tools-api-docs.yml b/.github/workflows/tools-api-docs.yml index 539c41d119..39a45cc34d 100644 --- a/.github/workflows/tools-api-docs.yml +++ b/.github/workflows/tools-api-docs.yml @@ -1,5 +1,4 @@ -name: nf-core/tools dev API docs -# Run on push and PR to test that docs build +name: generate nf-core/tools API docs on: push: branches: From f4a4c2f2e9df84e85155dbc7ed41a24ccaedb8b4 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 10:03:06 +0100 Subject: [PATCH 154/178] no need for a prompt if it profile is set as a parameter --- nf_core/components/components_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index a94441ae8d..f1a9e7c401 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -133,7 +133,7 @@ def check_inputs(self) -> None: ) # Check container software to use - if os.environ.get("PROFILE") is None: + if os.environ.get("PROFILE") is None and self.profile is None: os.environ["PROFILE"] = "" if self.no_prompts: log.info( @@ -237,16 +237,18 @@ def check_snapshot_stability(self) -> bool: log.error("nf-test snapshot is not stable") self.errors.append("nf-test snapshot is not stable") return False + else: if self.obsolete_snapshots: # ask if the user wants to remove obsolete snapshots using nf-test --clean-snapshot if self.no_prompts or Confirm.ask( "nf-test found obsolete snapshots. Do you want to remove them?", default=True ): + profile = self.profile if self.profile else os.environ["PROFILE"] log.info("Removing obsolete snapshots") nf_core.utils.run_cmd( "nf-test", - f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", + f"test --tag {self.component_name} --profile {profile} --clean-snapshot", ) else: log.debug("Obsolete snapshots not removed") From c3a3dae92110d7fd39201c64ecc5d25637e0c8fd Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 10:58:32 +0100 Subject: [PATCH 155/178] add missing comma --- .github/workflows/tools-api-docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tools-api-docs.yml b/.github/workflows/tools-api-docs.yml index 39a45cc34d..4fd99e4a6a 100644 --- a/.github/workflows/tools-api-docs.yml +++ b/.github/workflows/tools-api-docs.yml @@ -33,8 +33,8 @@ jobs: owner: 'nf-core', repo: 'website', workflow_id: 'add-tools-api-docs.yml', - ref: 'main' + ref: 'main', inputs: { "ref_name": "${{ inputs.ref_name || github.ref_name }}" - } + }, }) From 8f946530b71781466b340b97d2d3d4a98ff82dd4 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 12:33:35 +0100 Subject: [PATCH 156/178] move all possible config files during update --- nf_core/components/update.py | 13 ++++++++----- tests/modules/update.py | 16 ++++++++-------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 841ed45659..46136bc8da 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -740,13 +740,16 @@ def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path files = [f.name for f in temp_component_dir.iterdir() if f.is_file()] pipeline_path = Path(self.dir, self.component_type, repo_path, component) - # check if both directories have the same file and warn if the compnent_dir has more files if pipeline_path.exists(): pipeline_files = [f.name for f in pipeline_path.iterdir() if f.is_file()] - if "nextflow.config" in pipeline_files: - log.debug(f"Moving '{component}/nextflow.config' to updated component") - shutil.move(Path(pipeline_path, "nextflow.config"), Path(temp_component_dir, "nextflow.config")) - files.append("nextflow.config") + # check if any *.config file exists in the pipeline + if any([f.endswith(".config") for f in pipeline_files]): + # move the *.config file to the temporary directory + config_files = [f for f in files if f.endswith(".config")] + for config_file in config_files: + log.debug(f"Moving '{component}/{config_file}' to updated component") + shutil.move(Path(pipeline_path, config_file), Path(temp_component_dir, config_file)) + files.append(config_file) else: log.debug(f"Creating new {self.component_type[:-1]} '{component}' in '{self.component_type}/{repo_path}'") diff --git a/tests/modules/update.py b/tests/modules/update.py index 80cd5c2eed..81eb85716e 100644 --- a/tests/modules/update.py +++ b/tests/modules/update.py @@ -378,19 +378,19 @@ def cmp_module(dir1, dir2): return all(filecmp.cmp(os.path.join(dir1, f), os.path.join(dir2, f), shallow=False) for f in files) -def test_update_module_with_nextflow_config(self): - """Try updating a module with a nextflow.config file""" +def test_update_module_with_extra_config_file(self): + """Try updating a module with a config file""" # Install the module assert self.mods_install.install("trimgalore") - # Add a nextflow.config file to the module + # Add a nextflow_test.config file to the module trimgalore_path = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") - Path(trimgalore_path, "nextflow.config").touch() - with open(Path(trimgalore_path, "nextflow.config"), "w") as fh: + Path(trimgalore_path, "nextflow_test.config").touch() + with open(Path(trimgalore_path, "nextflow_test.config"), "w") as fh: fh.write("params.my_param = 'my_value'\n") # Update the module update_obj = ModuleUpdate(self.pipeline_dir, show_diff=False) assert update_obj.update("trimgalore") - # Check that the nextflow.config file is still there - assert Path(trimgalore_path, "nextflow.config").exists() - with open(Path(trimgalore_path, "nextflow.config")) as fh: + # Check that the nextflow_test.config file is still there + assert Path(trimgalore_path, "nextflow_test.config").exists() + with open(Path(trimgalore_path, "nextflow_test.config")) as fh: assert "params.my_param = 'my_value'" in fh.read() From 6b65a01b6b2293f0013ba511a9375e2e7df776e1 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 12:35:31 +0100 Subject: [PATCH 157/178] fix test name --- tests/test_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 796b1d2538..86639d3800 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -259,7 +259,7 @@ def test_modulesrepo_class(self): test_update_different_branch_mix_modules_branch_test, test_update_different_branch_mixed_modules_main, test_update_different_branch_single_module, - test_update_module_with_nextflow_config, + test_update_module_with_extra_config_file, test_update_only_show_differences, test_update_only_show_differences_when_patch, test_update_with_config_dont_update, From eeca1b20d86d7dd83aaa39e517f7f5eaebdd87e9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 13:47:09 +0100 Subject: [PATCH 158/178] fix generate api docs script, gsed-> sed --- docs/api/generate-api-docs.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api/generate-api-docs.sh b/docs/api/generate-api-docs.sh index f30c7f0665..6b3c3abfa3 100644 --- a/docs/api/generate-api-docs.sh +++ b/docs/api/generate-api-docs.sh @@ -48,7 +48,7 @@ for release in "${releases[@]}"; do git checkout docs/api pip install -r docs/api/requirements.txt --quiet # add the napoleon extension to the sphinx conf.py - gsed -i 's/^extensions = \[/extensions = \[\n "sphinx_markdown_builder",/' docs/api/_src/conf.py + sed -i 's/^extensions = \[/extensions = \[\n "sphinx_markdown_builder",/' docs/api/_src/conf.py # run docs/api/make_lint_md.py if it exists # if [[ -f "docs/api/make_lint_md.py" ]]; then @@ -59,12 +59,12 @@ for release in "${releases[@]}"; do # echo "Processing $file" # replace ..tip:: with note in the python docstrings due to missing directive in the markdown builder - gsed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' "$file" + sed -i 's/^\(\s*\)\.\. tip::/\1\.\. note::/g' "$file" done # fix syntax in lint/merge_markers.py - gsed -i 's/>>>>>>> or <<<<<<>>>>>>`` or ``<<<<<<<``/g' nf_core/lint/merge_markers.py + sed -i 's/>>>>>>> or <<<<<<>>>>>>`` or ``<<<<<<<``/g' nf_core/lint/merge_markers.py # remove markdown files if --force is set if [[ "$force" = true ]]; then echo -e "\n\e[31mRemoving $output_dir/$release because of '--force'\e[0m" @@ -77,7 +77,7 @@ for release in "${releases[@]}"; do git checkout - # replace :::{seealso} with :::tip in the markdown files - find "$output_dir/$release" -name "*.md" -exec gsed -i 's/:::{seealso}/:::tip/g' {} \; + find "$output_dir/$release" -name "*.md" -exec sed -i 's/:::{seealso}/:::tip/g' {} \; i=1 sp="/-\|" # spinner find "$output_dir/$release" -name "*.md" | while IFS= read -r file; do From 2240168af42bb36ac858cf42b869476ce90d14e2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 15:45:58 +0100 Subject: [PATCH 159/178] update modules in template --- nf_core/pipeline-template/modules.json | 4 +- .../modules/nf-core/fastqc/tests/main.nf.test | 212 ------------------ .../nf-core/fastqc/tests/main.nf.test.snap | 20 -- .../modules/nf-core/fastqc/tests/tags.yml | 2 - .../modules/nf-core/multiqc/environment.yml | 2 +- .../modules/nf-core/multiqc/main.nf | 4 +- .../nf-core/multiqc/tests/main.nf.test | 83 ------- .../nf-core/multiqc/tests/main.nf.test.snap | 21 -- .../modules/nf-core/multiqc/tests/tags.yml | 2 - 9 files changed, 5 insertions(+), 345 deletions(-) delete mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test delete mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap delete mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml delete mode 100644 nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test delete mode 100644 nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap delete mode 100644 nf_core/pipeline-template/modules/nf-core/multiqc/tests/tags.yml diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 96cb7942e9..97cbbe6df5 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,12 +7,12 @@ "nf-core": { "fastqc": { "branch": "master", - "git_sha": "c9488585ce7bd35ccd2a30faa2371454c8112fb9", + "git_sha": "f4ae1d942bd50c5c0b9bd2de1393ce38315ba57c", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", + "git_sha": "ccacf6f5de6df3bc6d73b665c1fd2933d8bbc290", "installed_by": ["modules"] } } diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test deleted file mode 100644 index 1f21c66469..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test +++ /dev/null @@ -1,212 +0,0 @@ -nextflow_process { - - name "Test Process FASTQC" - script "../main.nf" - process "FASTQC" - - tag "modules" - tag "modules_nfcore" - tag "fastqc" - - test("sarscov2 single-end [fastq]") { - - when { - process { - """ - input[0] = Channel.of([ - [ id: 'test', single_end:true ], - [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] - ]) - """ - } - } - - then { - assertAll ( - { assert process.success }, - - // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. - // looks like this:
    Mon 2 Oct 2023
    test.gz
    - // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 - - { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("    ") }, - - { assert snapshot(process.out.versions).match("versions") } - ) - } - } - - test("sarscov2 paired-end [fastq]") { - - when { - process { - """ - input[0] = Channel.of([ - [id: 'test', single_end: false], // meta map - [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) ] - ]) - """ - } - } - - then { - assertAll ( - { assert process.success }, - - { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, - { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, - { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, - { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, - { assert path(process.out.html[0][1][0]).text.contains("") }, - { assert path(process.out.html[0][1][1]).text.contains("") }, - - { assert snapshot(process.out.versions).match("versions") } - ) - } - } - - test("sarscov2 interleaved [fastq]") { - - when { - process { - """ - input[0] = Channel.of([ - [id: 'test', single_end: false], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz', checkIfExists: true) - ]) - """ - } - } - - then { - assertAll ( - { assert process.success }, - - { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("") }, - - { assert snapshot(process.out.versions).match("versions") } - ) - } - } - - test("sarscov2 paired-end [bam]") { - - when { - process { - """ - input[0] = Channel.of([ - [id: 'test', single_end: false], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) - ]) - """ - } - } - - then { - assertAll ( - { assert process.success }, - - { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("") }, - - { assert snapshot(process.out.versions).match("versions") } - ) - } - } - - test("sarscov2 multiple [fastq]") { - - when { - process { - """ - input[0] = Channel.of([ - [id: 'test', single_end: false], // meta map - [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_2.fastq.gz', checkIfExists: true) ] - ]) - """ - } - } - - then { - assertAll ( - { assert process.success }, - - { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, - { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, - { assert process.out.html[0][1][2] ==~ ".*/test_3_fastqc.html" }, - { assert process.out.html[0][1][3] ==~ ".*/test_4_fastqc.html" }, - { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, - { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, - { assert process.out.zip[0][1][2] ==~ ".*/test_3_fastqc.zip" }, - { assert process.out.zip[0][1][3] ==~ ".*/test_4_fastqc.zip" }, - { assert path(process.out.html[0][1][0]).text.contains("") }, - { assert path(process.out.html[0][1][1]).text.contains("") }, - { assert path(process.out.html[0][1][2]).text.contains("") }, - { assert path(process.out.html[0][1][3]).text.contains("") }, - - { assert snapshot(process.out.versions).match("versions") } - ) - } - } - - test("sarscov2 custom_prefix") { - - when { - process { - """ - input[0] = Channel.of([ - [ id:'mysample', single_end:true ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) - ]) - """ - } - } - - then { - assertAll ( - { assert process.success }, - - { assert process.out.html[0][1] ==~ ".*/mysample_fastqc.html" }, - { assert process.out.zip[0][1] ==~ ".*/mysample_fastqc.zip" }, - { assert path(process.out.html[0][1]).text.contains("") }, - - { assert snapshot(process.out.versions).match("versions") } - ) - } - } - - test("sarscov2 single-end [fastq] - stub") { - - options "-stub" - - when { - process { - """ - input[0] = Channel.of([ - [ id: 'test', single_end:true ], - [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] - ]) - """ - } - } - - then { - assertAll ( - { assert process.success }, - { assert snapshot(process.out.html.collect { file(it[1]).getName() } + - process.out.zip.collect { file(it[1]).getName() } + - process.out.versions ).match() } - ) - } - } - -} diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap deleted file mode 100644 index 5d624bb82e..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap +++ /dev/null @@ -1,20 +0,0 @@ -{ - "sarscov2 single-end [fastq] - stub": { - "content": [ - [ - "test.html", - "test.zip", - "versions.yml:md5,e1cc25ca8af856014824abd842e93978" - ] - ], - "timestamp": "2024-01-17T18:40:57.254299" - }, - "versions": { - "content": [ - [ - "versions.yml:md5,e1cc25ca8af856014824abd842e93978" - ] - ], - "timestamp": "2024-01-17T18:36:50.033627" - } -} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml deleted file mode 100644 index 7834294ba0..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -fastqc: - - modules/nf-core/fastqc/** diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml index 7625b75206..2212096af4 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.19 + - bioconda::multiqc=1.20 diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index 1b9f7c431d..354f4430f9 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -3,8 +3,8 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : - 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.20--pyhdfd78af_0' : + 'biocontainers/multiqc:1.20--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test deleted file mode 100644 index d0438eda6b..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test +++ /dev/null @@ -1,83 +0,0 @@ -nextflow_process { - - name "Test Process MULTIQC" - script "../main.nf" - process "MULTIQC" - tag "modules" - tag "modules_nfcore" - tag "multiqc" - - test("sarscov2 single-end [fastqc]") { - - when { - process { - """ - input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) - input[1] = [] - input[2] = [] - input[3] = [] - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, - { assert process.out.data[0] ==~ ".*/multiqc_data" }, - { assert snapshot(process.out.versions).match("versions") } - ) - } - - } - - test("sarscov2 single-end [fastqc] [config]") { - - when { - process { - """ - input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) - input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) - input[2] = [] - input[3] = [] - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, - { assert process.out.data[0] ==~ ".*/multiqc_data" }, - { assert snapshot(process.out.versions).match("versions") } - ) - } - } - - test("sarscov2 single-end [fastqc] - stub") { - - options "-stub" - - when { - process { - """ - input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) - input[1] = [] - input[2] = [] - input[3] = [] - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out.report.collect { file(it).getName() } + - process.out.data.collect { file(it).getName() } + - process.out.plots.collect { file(it).getName() } + - process.out.versions ).match() } - ) - } - - } -} diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap deleted file mode 100644 index d37e73040d..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap +++ /dev/null @@ -1,21 +0,0 @@ -{ - "versions": { - "content": [ - [ - "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" - ] - ], - "timestamp": "2024-01-09T23:02:49.911994" - }, - "sarscov2 single-end [fastqc] - stub": { - "content": [ - [ - "multiqc_report.html", - "multiqc_data", - "multiqc_plots", - "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" - ] - ], - "timestamp": "2024-01-09T23:03:14.524346" - } -} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/tags.yml deleted file mode 100644 index bea6c0d37f..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -multiqc: - - modules/nf-core/multiqc/** From 9f56fcb892a50ba5bc02d7308be11a5a54d1de3a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 19 Feb 2024 14:47:25 +0000 Subject: [PATCH 160/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85896d42d7..5f84115f1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Patch: handle file not found when it is an added file to a module ([#2771](https://github.com/nf-core/tools/pull/2771)) - Handle symlinks when migrating pytest ([#2770](https://github.com/nf-core/tools/pull/2770)) - Add `--profile` parameter to nf-test command ([#2767](https://github.com/nf-core/tools/pull/2767)) +- update in template ([#2776](https://github.com/nf-core/tools/pull/2776)) ### General From 2e076d135b47952c68b5ded3b3b3b322cf9a0756 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 17:01:48 +0100 Subject: [PATCH 161/178] move also subdirectories during update --- nf_core/components/update.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 46136bc8da..7972fe80f7 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -737,15 +737,18 @@ def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path new_version (str): The version of the module/subworkflow that was installed. """ temp_component_dir = Path(install_folder, component) - files = [f.name for f in temp_component_dir.iterdir() if f.is_file()] + files = [] + for file_path in Path(temp_component_dir).rglob("*"): + if file_path.is_file(): + files.append(file_path) pipeline_path = Path(self.dir, self.component_type, repo_path, component) if pipeline_path.exists(): pipeline_files = [f.name for f in pipeline_path.iterdir() if f.is_file()] # check if any *.config file exists in the pipeline - if any([f.endswith(".config") for f in pipeline_files]): + if any([str(f).endswith(".config") for f in pipeline_files]): # move the *.config file to the temporary directory - config_files = [f for f in files if f.endswith(".config")] + config_files = [f for f in files if str(f).endswith(".config")] for config_file in config_files: log.debug(f"Moving '{component}/{config_file}' to updated component") shutil.move(Path(pipeline_path, config_file), Path(temp_component_dir, config_file)) @@ -757,11 +760,16 @@ def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path log.debug(f"Removing old version of {self.component_type[:-1]} '{component}'") self.clear_component_dir(component, str(pipeline_path)) - os.makedirs(pipeline_path) + pipeline_path.mkdir(parents=True, exist_ok=True) for file in files: + file = file.relative_to(temp_component_dir) path = Path(temp_component_dir, file) - if os.path.exists(path): - shutil.move(path, Path(pipeline_path, file)) + if path.exists(): + log.debug(f"Moving '{file}' to updated component") + dest = Path(pipeline_path, file) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.move(path, dest) + log.debug(f"{os.listdir(pipeline_path)}") log.info(f"Updating '{repo_path}/{component}'") log.debug(f"Updating {self.component_type[:-1]} '{component}' to {new_version} from {repo_path}") From f12aec562ae97ca1f6db0ccffa2f90a75c8f23b1 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 17:33:41 +0100 Subject: [PATCH 162/178] bring back lost files --- .../modules/nf-core/fastqc/tests/main.nf.test | 212 ++++++++++++++++++ .../nf-core/fastqc/tests/main.nf.test.snap | 88 ++++++++ .../modules/nf-core/fastqc/tests/tags.yml | 2 + .../nf-core/multiqc/tests/main.nf.test | 84 +++++++ .../nf-core/multiqc/tests/main.nf.test.snap | 41 ++++ .../modules/nf-core/multiqc/tests/tags.yml | 2 + 6 files changed, 429 insertions(+) create mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test create mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap create mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml create mode 100644 nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test create mode 100644 nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap create mode 100644 nf_core/pipeline-template/modules/nf-core/multiqc/tests/tags.yml diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test new file mode 100644 index 0000000000..70edae4d99 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test @@ -0,0 +1,212 @@ +nextflow_process { + + name "Test Process FASTQC" + script "../main.nf" + process "FASTQC" + + tag "modules" + tag "modules_nfcore" + tag "fastqc" + + test("sarscov2 single-end [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [ id: 'test', single_end:true ], + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. + // looks like this:
    Mon 2 Oct 2023
    test.gz
    + // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("    ") }, + + { assert snapshot(process.out.versions).match("fastqc_versions_single") } + ) + } + } + + test("sarscov2 paired-end [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("") }, + { assert path(process.out.html[0][1][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("fastqc_versions_paired") } + ) + } + } + + test("sarscov2 interleaved [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("fastqc_versions_interleaved") } + ) + } + } + + test("sarscov2 paired-end [bam]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("fastqc_versions_bam") } + ) + } + } + + test("sarscov2 multiple [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.html[0][1][2] ==~ ".*/test_3_fastqc.html" }, + { assert process.out.html[0][1][3] ==~ ".*/test_4_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert process.out.zip[0][1][2] ==~ ".*/test_3_fastqc.zip" }, + { assert process.out.zip[0][1][3] ==~ ".*/test_4_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("") }, + { assert path(process.out.html[0][1][1]).text.contains("") }, + { assert path(process.out.html[0][1][2]).text.contains("") }, + { assert path(process.out.html[0][1][3]).text.contains("") }, + + { assert snapshot(process.out.versions).match("fastqc_versions_multiple") } + ) + } + } + + test("sarscov2 custom_prefix") { + + when { + process { + """ + input[0] = Channel.of([ + [ id:'mysample', single_end:true ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/mysample_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/mysample_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("fastqc_versions_custom_prefix") } + ) + } + } + + test("sarscov2 single-end [fastq] - stub") { + + options "-stub" + + when { + process { + """ + input[0] = Channel.of([ + [ id: 'test', single_end:true ], + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out.html.collect { file(it[1]).getName() } + + process.out.zip.collect { file(it[1]).getName() } + + process.out.versions ).match("fastqc_stub") } + ) + } + } + +} diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap new file mode 100644 index 0000000000..86f7c31154 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap @@ -0,0 +1,88 @@ +{ + "fastqc_versions_interleaved": { + "content": [ + [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-01-31T17:40:07.293713" + }, + "fastqc_stub": { + "content": [ + [ + "test.html", + "test.zip", + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-01-31T17:31:01.425198" + }, + "fastqc_versions_multiple": { + "content": [ + [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-01-31T17:40:55.797907" + }, + "fastqc_versions_bam": { + "content": [ + [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-01-31T17:40:26.795862" + }, + "fastqc_versions_single": { + "content": [ + [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-01-31T17:39:27.043675" + }, + "fastqc_versions_paired": { + "content": [ + [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-01-31T17:39:47.584191" + }, + "fastqc_versions_custom_prefix": { + "content": [ + [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-01-31T17:41:14.576531" + } +} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml new file mode 100644 index 0000000000..7834294ba0 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml @@ -0,0 +1,2 @@ +fastqc: + - modules/nf-core/fastqc/** diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test new file mode 100644 index 0000000000..f1c4242ef2 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test @@ -0,0 +1,84 @@ +nextflow_process { + + name "Test Process MULTIQC" + script "../main.nf" + process "MULTIQC" + + tag "modules" + tag "modules_nfcore" + tag "multiqc" + + test("sarscov2 single-end [fastqc]") { + + when { + process { + """ + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) + input[1] = [] + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("multiqc_versions_single") } + ) + } + + } + + test("sarscov2 single-end [fastqc] [config]") { + + when { + process { + """ + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) + input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("multiqc_versions_config") } + ) + } + } + + test("sarscov2 single-end [fastqc] - stub") { + + options "-stub" + + when { + process { + """ + input[0] = Channel.of(file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastqc/test_fastqc.zip', checkIfExists: true)) + input[1] = [] + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.report.collect { file(it).getName() } + + process.out.data.collect { file(it).getName() } + + process.out.plots.collect { file(it).getName() } + + process.out.versions ).match("multiqc_stub") } + ) + } + + } +} diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap new file mode 100644 index 0000000000..c204b4881e --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -0,0 +1,41 @@ +{ + "multiqc_versions_single": { + "content": [ + [ + "versions.yml:md5,d320d4c37e349c5588e07e7a31cd4186" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-14T09:28:51.744211298" + }, + "multiqc_stub": { + "content": [ + [ + "multiqc_report.html", + "multiqc_data", + "multiqc_plots", + "versions.yml:md5,d320d4c37e349c5588e07e7a31cd4186" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-14T09:29:28.847433492" + }, + "multiqc_versions_config": { + "content": [ + [ + "versions.yml:md5,d320d4c37e349c5588e07e7a31cd4186" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-14T09:29:13.223621555" + } +} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/tags.yml new file mode 100644 index 0000000000..bea6c0d37f --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/tags.yml @@ -0,0 +1,2 @@ +multiqc: + - modules/nf-core/multiqc/** From 749b2ce01f1c5e41e653180012364548823a029d Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 17:35:32 +0100 Subject: [PATCH 163/178] fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f84115f1b..01bb78e23f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Add Github Action to automatically cleanup ubuntu-latest runners to fix runner running out of diskspace errors([#2755](https://github.com/nf-core/tools/issues/2755)) - Fix GitHub Actions CI and Linting badges links ([#2757](https://github.com/nf-core/tools/pull/2757)) - Add hashtags to release announcement on mastodon ([#2761](https://github.com/nf-core/tools/pull/2761)) +- update in fastqc and multiqc in template ([#2776](https://github.com/nf-core/tools/pull/2776)) ### Linting @@ -25,7 +26,6 @@ - Patch: handle file not found when it is an added file to a module ([#2771](https://github.com/nf-core/tools/pull/2771)) - Handle symlinks when migrating pytest ([#2770](https://github.com/nf-core/tools/pull/2770)) - Add `--profile` parameter to nf-test command ([#2767](https://github.com/nf-core/tools/pull/2767)) -- update in template ([#2776](https://github.com/nf-core/tools/pull/2776)) ### General From 2d380c567f566531cf7d1ea64b335685ad4847e4 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 19 Feb 2024 17:48:08 +0100 Subject: [PATCH 164/178] bump version to dev -> 2.13 --- CHANGELOG.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85896d42d7..265dd04a94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v2.13dev +## [v2.13 - Tin Puppy](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-19] ### Template diff --git a/setup.py b/setup.py index e6e7b40762..3bfb99d4a8 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.13dev" +version = "2.13" with open("README.md") as f: readme = f.read() From e070ebbba8daa2a173f6e154588b30042d0a9f23 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 20 Feb 2024 08:56:29 +0100 Subject: [PATCH 165/178] add code review suggestions --- nf_core/components/update.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index 7972fe80f7..f6765bed69 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -3,6 +3,7 @@ import shutil import tempfile from pathlib import Path +from typing import List import questionary @@ -737,7 +738,7 @@ def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path new_version (str): The version of the module/subworkflow that was installed. """ temp_component_dir = Path(install_folder, component) - files = [] + files: List[Path] = [] for file_path in Path(temp_component_dir).rglob("*"): if file_path.is_file(): files.append(file_path) @@ -746,13 +747,11 @@ def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path if pipeline_path.exists(): pipeline_files = [f.name for f in pipeline_path.iterdir() if f.is_file()] # check if any *.config file exists in the pipeline - if any([str(f).endswith(".config") for f in pipeline_files]): - # move the *.config file to the temporary directory - config_files = [f for f in files if str(f).endswith(".config")] - for config_file in config_files: - log.debug(f"Moving '{component}/{config_file}' to updated component") - shutil.move(Path(pipeline_path, config_file), Path(temp_component_dir, config_file)) - files.append(config_file) + config_files = [f for f in pipeline_files if str(f).endswith(".config")] + for config_file in config_files: + log.debug(f"Moving '{component}/{config_file}' to updated component") + shutil.move(Path(pipeline_path, config_file), Path(temp_component_dir, config_file)) + files.append(Path(config_file)) else: log.debug(f"Creating new {self.component_type[:-1]} '{component}' in '{self.component_type}/{repo_path}'") @@ -769,7 +768,6 @@ def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path dest = Path(pipeline_path, file) dest.parent.mkdir(parents=True, exist_ok=True) shutil.move(path, dest) - log.debug(f"{os.listdir(pipeline_path)}") log.info(f"Updating '{repo_path}/{component}'") log.debug(f"Updating {self.component_type[:-1]} '{component}' to {new_version} from {repo_path}") From 656b0f8ff04f0faf375bfbdde536eabbae2673ca Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 20 Feb 2024 09:02:28 +0100 Subject: [PATCH 166/178] update pyproject toml according to ruff 0.2.0 --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9c551f3297..ffe78d3d3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,9 +15,12 @@ norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".githu [tool.ruff] line-length = 120 target-version = "py38" -select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] cache-dir = "~/.cache/ruff" +[tool.ruff.lint] +select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] + + [tool.ruff.lint.isort] known-first-party = ["nf_core"] From 69e3ececafbfc7bc835e4060e7c4e99d4e48b47e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 20 Feb 2024 09:05:50 +0100 Subject: [PATCH 167/178] format pyproject.toml file --- pyproject.toml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ffe78d3d3f..8168bd7c11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,21 @@ [build-system] build-backend = "setuptools.build_meta" -requires = [ - "setuptools>=40.6.0", - "wheel" -] +requires = ["setuptools>=40.6.0", "wheel"] [tool.pytest.ini_options] -markers = [ - "datafiles: load datafiles" -] +markers = ["datafiles: load datafiles"] testpaths = ["tests"] -norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".github", "nf_core", "docs"] +norecursedirs = [ + ".*", + "build", + "dist", + "*.egg", + "data", + "__pycache__", + ".github", + "nf_core", + "docs", +] [tool.ruff] line-length = 120 @@ -20,7 +25,6 @@ cache-dir = "~/.cache/ruff" [tool.ruff.lint] select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] - [tool.ruff.lint.isort] known-first-party = ["nf_core"] @@ -29,4 +33,3 @@ known-first-party = ["nf_core"] [tool.ruff.lint.pep8-naming] extend-ignore-names = ["mocked_*", "*allOf", "*URI*"] - From 9951f2594009d56ee092fabdae6fbf2c97dc02f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 20 Feb 2024 09:48:53 +0000 Subject: [PATCH 168/178] update pipeline template pyproject.toml according to ruff 0.2.0 --- nf_core/pipeline-template/pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 84c16f27cd..56110621e7 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -3,9 +3,11 @@ [tool.ruff] line-length = 120 target-version = "py38" -select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] cache-dir = "~/.cache/ruff" +[tool.ruff.lint] +select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] + [tool.ruff.lint.isort] known-first-party = ["nf_core"] From 89a31548888cd844392afe578484c585440e3fb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 20 Feb 2024 10:25:11 +0000 Subject: [PATCH 169/178] add full path of nextflow.config to update files --- nf_core/components/update.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nf_core/components/update.py b/nf_core/components/update.py index f6765bed69..a54c47232e 100644 --- a/nf_core/components/update.py +++ b/nf_core/components/update.py @@ -3,7 +3,6 @@ import shutil import tempfile from pathlib import Path -from typing import List import questionary @@ -738,10 +737,7 @@ def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path new_version (str): The version of the module/subworkflow that was installed. """ temp_component_dir = Path(install_folder, component) - files: List[Path] = [] - for file_path in Path(temp_component_dir).rglob("*"): - if file_path.is_file(): - files.append(file_path) + files = [file_path for file_path in temp_component_dir.rglob("*") if file_path.is_file()] pipeline_path = Path(self.dir, self.component_type, repo_path, component) if pipeline_path.exists(): @@ -750,8 +746,8 @@ def move_files_from_tmp_dir(self, component: str, install_folder: str, repo_path config_files = [f for f in pipeline_files if str(f).endswith(".config")] for config_file in config_files: log.debug(f"Moving '{component}/{config_file}' to updated component") - shutil.move(Path(pipeline_path, config_file), Path(temp_component_dir, config_file)) - files.append(Path(config_file)) + shutil.move(pipeline_path / config_file, temp_component_dir / config_file) + files.append(temp_component_dir / config_file) else: log.debug(f"Creating new {self.component_type[:-1]} '{component}' in '{self.component_type}/{repo_path}'") From faee54292e3b00a8525b377493095fde2d53650d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 20 Feb 2024 12:38:26 +0100 Subject: [PATCH 170/178] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01bb78e23f..912a0c1a99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Add Github Action to automatically cleanup ubuntu-latest runners to fix runner running out of diskspace errors([#2755](https://github.com/nf-core/tools/issues/2755)) - Fix GitHub Actions CI and Linting badges links ([#2757](https://github.com/nf-core/tools/pull/2757)) - Add hashtags to release announcement on mastodon ([#2761](https://github.com/nf-core/tools/pull/2761)) -- update in fastqc and multiqc in template ([#2776](https://github.com/nf-core/tools/pull/2776)) +- update fastqc and multiqc in template ([#2776](https://github.com/nf-core/tools/pull/2776)) ### Linting From d95729e950cd165231dffb6da89d5cff98c534d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 20 Feb 2024 12:39:56 +0100 Subject: [PATCH 171/178] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 265dd04a94..25dbb0e654 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## [v2.13 - Tin Puppy](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-19] +## [v2.13 - Tin Puppy](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-20] ### Template From 8352dc1750e2b18fdec081c18f345af1bf16ff64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 20 Feb 2024 11:50:49 +0000 Subject: [PATCH 172/178] update gitpod image to latest --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 39fb530da2..a4f1c17169 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:dev +image: nfcore/gitpod:latest tasks: - name: install current state of nf-core/tools and setup pre-commit command: | From fdda81dbea4df32fd8c990d4e270919553cc0244 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 11:11:03 +0000 Subject: [PATCH 173/178] chore(deps): update gitpod/workspace-base docker digest to 728e1fa --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index b8adb53827..c441d40df1 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:45e7617f50af7df5006dce82a65d0d59fb4076a613f906f43b5f44941dd9f70d +FROM gitpod/workspace-base@sha256:728e1fab64f6924128b987264603a6f277bd881de95feaf39129a1ffdde36e14 USER root From 04ff0f23593b35f293deeb58a7fdcc4d067c7fbc Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 20 Feb 2024 12:55:36 +0000 Subject: [PATCH 174/178] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 912a0c1a99..42ee78fd85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ - chore(deps): pin jlumbroso/free-disk-space action to 54081f1 ([#2756](https://github.com/nf-core/tools/pull/2756)) - chore(deps): update actions/github-script action to v7 ([#2766](https://github.com/nf-core/tools/pull/2766)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.2.2 ([#2769](https://github.com/nf-core/tools/pull/2769)) +- Update gitpod/workspace-base Docker digest to 728e1fa ([#2780](https://github.com/nf-core/tools/pull/2780)) ## [v2.12.1 - Aluminium Wolf - Patch](https://github.com/nf-core/tools/releases/tag/2.12.1) - [2024-02-01] From 8525502a9d6d03fd2aa91eca7db9a4f021d61160 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 20 Feb 2024 13:48:22 +0000 Subject: [PATCH 175/178] Generate new screengrabs with rich-codex --- docs/images/nf-core-bump-version.svg | 148 ++++----- docs/images/nf-core-create-logo.svg | 68 ++-- docs/images/nf-core-create.svg | 126 +++---- docs/images/nf-core-download.svg | 100 +++--- docs/images/nf-core-launch-rnaseq.svg | 82 ++--- docs/images/nf-core-licences.svg | 130 ++++---- docs/images/nf-core-lint.svg | 205 +++++------- docs/images/nf-core-list-rna.svg | 134 ++++---- docs/images/nf-core-list-stars.svg | 104 +++--- docs/images/nf-core-list.svg | 108 +++--- docs/images/nf-core-modules-bump-version.svg | 106 +++--- docs/images/nf-core-modules-create.svg | 132 ++++---- docs/images/nf-core-modules-info.svg | 206 ++++++------ docs/images/nf-core-modules-install.svg | 86 ++--- docs/images/nf-core-modules-lint.svg | 122 +++---- docs/images/nf-core-modules-list-local.svg | 135 ++++---- docs/images/nf-core-modules-list-remote.svg | 132 ++++---- docs/images/nf-core-modules-patch.svg | 156 ++++----- docs/images/nf-core-modules-remove.svg | 70 ++-- docs/images/nf-core-modules-test.svg | 70 ++-- docs/images/nf-core-modules-update.svg | 92 +++--- docs/images/nf-core-schema-build.svg | 84 ++--- docs/images/nf-core-schema-lint.svg | 78 ++--- docs/images/nf-core-schema-validate.svg | 82 ++--- docs/images/nf-core-subworkflows-create.svg | 104 +++--- docs/images/nf-core-subworkflows-info.svg | 140 ++++---- docs/images/nf-core-subworkflows-install.svg | 70 ++-- docs/images/nf-core-subworkflows-lint.svg | 308 +++++++++--------- .../nf-core-subworkflows-list-local.svg | 134 +++++--- .../nf-core-subworkflows-list-remote.svg | 132 ++++---- docs/images/nf-core-subworkflows-remove.svg | 124 ++++--- docs/images/nf-core-subworkflows-test.svg | 70 ++-- docs/images/nf-core-subworkflows-update.svg | 81 +++-- docs/images/nf-core-sync.svg | 108 +++--- 34 files changed, 2015 insertions(+), 2012 deletions(-) diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg index af75211d65..838d1cfc7f 100644 --- a/docs/images/nf-core-bump-version.svg +++ b/docs/images/nf-core-bump-version.svg @@ -19,122 +19,122 @@ font-weight: 700; } - .terminal-3212272693-matrix { + .terminal-978478039-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3212272693-title { + .terminal-978478039-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3212272693-r1 { fill: #c5c8c6 } -.terminal-3212272693-r2 { fill: #98a84b } -.terminal-3212272693-r3 { fill: #9a9b99 } -.terminal-3212272693-r4 { fill: #608ab1 } -.terminal-3212272693-r5 { fill: #d0b344 } -.terminal-3212272693-r6 { fill: #cc555a } + .terminal-978478039-r1 { fill: #c5c8c6 } +.terminal-978478039-r2 { fill: #98a84b } +.terminal-978478039-r3 { fill: #9a9b99 } +.terminal-978478039-r4 { fill: #608ab1 } +.terminal-978478039-r5 { fill: #d0b344 } +.terminal-978478039-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -146,41 +146,41 @@ - + - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1' -INFO     Updated version in 'nextflow.config' - - version         = '1.0dev' - + version         = '1.1' - - -INFO     Updated version in 'assets/multiqc_config.yml' - - This report has been generated by the <a  -href="https://github.com/nf-core/nextbigthing/tree/dev" target="_blank">nf-core/nextbigthing</a> - + This report has been generated by the <a  -href="https://github.com/nf-core/nextbigthing/releases/tag/1.1"  -target="_blank">nf-core/nextbigthing</a> - - -INFO     Updated version in 'assets/multiqc_config.yml' - - <a href="https://nf-co.re/nextbigthing/dev/docs/output"  -target="_blank">documentation</a>. - + <a href="https://nf-co.re/nextbigthing/1.1/docs/output"  -target="_blank">documentation</a>. - - + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1' +INFO     Updated version in 'nextflow.config' + - version         = '1.0dev' + + version         = '1.1' + + +INFO     Updated version in 'assets/multiqc_config.yml' + - This report has been generated by the <a  +href="https://github.com/nf-core/nextbigthing/tree/dev" target="_blank">nf-core/nextbigthing</a> + + This report has been generated by the <a  +href="https://github.com/nf-core/nextbigthing/releases/tag/1.1"  +target="_blank">nf-core/nextbigthing</a> + + +INFO     Updated version in 'assets/multiqc_config.yml' + - <a href="https://nf-co.re/nextbigthing/dev/docs/output"  +target="_blank">documentation</a>. + + <a href="https://nf-co.re/nextbigthing/1.1/docs/output"  +target="_blank">documentation</a>. + + diff --git a/docs/images/nf-core-create-logo.svg b/docs/images/nf-core-create-logo.svg index 5f8bf56128..ac5d872b94 100644 --- a/docs/images/nf-core-create-logo.svg +++ b/docs/images/nf-core-create-logo.svg @@ -19,62 +19,62 @@ font-weight: 700; } - .terminal-2549025252-matrix { + .terminal-189794694-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2549025252-title { + .terminal-189794694-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2549025252-r1 { fill: #c5c8c6 } -.terminal-2549025252-r2 { fill: #98a84b } -.terminal-2549025252-r3 { fill: #9a9b99 } -.terminal-2549025252-r4 { fill: #608ab1 } -.terminal-2549025252-r5 { fill: #d0b344 } -.terminal-2549025252-r6 { fill: #98729f } + .terminal-189794694-r1 { fill: #c5c8c6 } +.terminal-189794694-r2 { fill: #98a84b } +.terminal-189794694-r3 { fill: #9a9b99 } +.terminal-189794694-r4 { fill: #608ab1 } +.terminal-189794694-r5 { fill: #d0b344 } +.terminal-189794694-r6 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + @@ -86,21 +86,21 @@ - + - - $ nf-core create-logo nextbigthing - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -INFO     Created logo: nf-core-nextbigthing_logo_light.png + + $ nf-core create-logo nextbigthing + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +INFO     Created logo: nf-core-nextbigthing_logo_light.png diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 02d942a7f9..68cae552b3 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,104 +19,104 @@ font-weight: 700; } - .terminal-4211528375-matrix { + .terminal-2074661465-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4211528375-title { + .terminal-2074661465-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4211528375-r1 { fill: #c5c8c6 } -.terminal-4211528375-r2 { fill: #98a84b } -.terminal-4211528375-r3 { fill: #9a9b99 } -.terminal-4211528375-r4 { fill: #608ab1 } -.terminal-4211528375-r5 { fill: #d0b344 } -.terminal-4211528375-r6 { fill: #98729f } -.terminal-4211528375-r7 { fill: #ff2c7a } -.terminal-4211528375-r8 { fill: #98a84b;font-weight: bold } -.terminal-4211528375-r9 { fill: #1984e9;text-decoration: underline; } + .terminal-2074661465-r1 { fill: #c5c8c6 } +.terminal-2074661465-r2 { fill: #98a84b } +.terminal-2074661465-r3 { fill: #9a9b99 } +.terminal-2074661465-r4 { fill: #608ab1 } +.terminal-2074661465-r5 { fill: #d0b344 } +.terminal-2074661465-r6 { fill: #98729f } +.terminal-2074661465-r7 { fill: #ff2c7a } +.terminal-2074661465-r8 { fill: #98a84b;font-weight: bold } +.terminal-2074661465-r9 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -128,34 +128,34 @@ - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' -INFO     Initialising pipeline git repository                                                        -INFO     Done. Remember to add a remote and push to GitHub:                                          - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      -INFO    !!!!!! IMPORTANT !!!!!! - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' +INFO     Initialising pipeline git repository                                                        +INFO     Done. Remember to add a remote and push to GitHub:                                          + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      +INFO    !!!!!! IMPORTANT !!!!!! + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 3d306efb25..e2ed5b22b4 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,86 +19,86 @@ font-weight: 700; } - .terminal-2749558223-matrix { + .terminal-2088037745-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2749558223-title { + .terminal-2088037745-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2749558223-r1 { fill: #c5c8c6 } -.terminal-2749558223-r2 { fill: #98a84b } -.terminal-2749558223-r3 { fill: #9a9b99 } -.terminal-2749558223-r4 { fill: #608ab1 } -.terminal-2749558223-r5 { fill: #d0b344 } -.terminal-2749558223-r6 { fill: #cc555a } + .terminal-2088037745-r1 { fill: #c5c8c6 } +.terminal-2088037745-r2 { fill: #98a84b } +.terminal-2088037745-r3 { fill: #9a9b99 } +.terminal-2088037745-r4 { fill: #608ab1 } +.terminal-2088037745-r5 { fill: #d0b344 } +.terminal-2088037745-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -110,29 +110,29 @@ - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -s none -d - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -WARNING  Could not find GitHub authentication token. Some API requests may fail.                     -INFO     Saving 'nf-core/rnaseq' -          Pipeline revision: '3.8' -          Use containers: 'none' -          Container library: 'quay.io' -          Output directory: 'nf-core-rnaseq' -          Include default institutional configuration: 'True' -INFO     Downloading centralised configs from GitHub                                                 -INFO     Downloading workflow files from GitHub                                                      + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -s none -d + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +WARNING  Could not find GitHub authentication token. Some API requests may fail.                     +INFO     Saving 'nf-core/rnaseq' +          Pipeline revision: '3.8' +          Use containers: 'none' +          Container library: 'quay.io' +          Output directory: 'nf-core-rnaseq' +          Include default institutional configuration: 'True' +INFO     Downloading centralised configs from GitHub                                                 +INFO     Downloading workflow files from GitHub                                                      diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index 42ac222ed3..6bbbbaf099 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-3783306802-matrix { + .terminal-1939516884-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3783306802-title { + .terminal-1939516884-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3783306802-r1 { fill: #c5c8c6 } -.terminal-3783306802-r2 { fill: #98a84b } -.terminal-3783306802-r3 { fill: #9a9b99 } -.terminal-3783306802-r4 { fill: #608ab1 } -.terminal-3783306802-r5 { fill: #d0b344 } -.terminal-3783306802-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3783306802-r7 { fill: #68a0b3;font-weight: bold } + .terminal-1939516884-r1 { fill: #c5c8c6 } +.terminal-1939516884-r2 { fill: #98a84b } +.terminal-1939516884-r3 { fill: #9a9b99 } +.terminal-1939516884-r4 { fill: #608ab1 } +.terminal-1939516884-r5 { fill: #d0b344 } +.terminal-1939516884-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1939516884-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      -         files or profiles                                                                           - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1) + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      +         files or profiles                                                                           + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index e7971f9883..ab30be3467 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-3657550497-matrix { + .terminal-2618804803-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3657550497-title { + .terminal-2618804803-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3657550497-r1 { fill: #c5c8c6 } -.terminal-3657550497-r2 { fill: #98a84b } -.terminal-3657550497-r3 { fill: #9a9b99 } -.terminal-3657550497-r4 { fill: #608ab1 } -.terminal-3657550497-r5 { fill: #d0b344 } -.terminal-3657550497-r6 { fill: #68a0b3;font-weight: bold } -.terminal-3657550497-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-2618804803-r1 { fill: #c5c8c6 } +.terminal-2618804803-r2 { fill: #98a84b } +.terminal-2618804803-r3 { fill: #9a9b99 } +.terminal-2618804803-r4 { fill: #608ab1 } +.terminal-2618804803-r5 { fill: #d0b344 } +.terminal-2618804803-r6 { fill: #68a0b3;font-weight: bold } +.terminal-2618804803-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    -INFO     Warning: This tool only prints licence information for the software tools packaged using    -         conda.                                                                                      -INFO     The pipeline may use other software and dependencies not described here.                    -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    +INFO     Warning: This tool only prints licence information for the software tools packaged using    +         conda.                                                                                      +INFO     The pipeline may use other software and dependencies not described here.                    +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index 8bab8f6e57..74e41870d3 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Testing pipeline: . - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos                                                                  │ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ -││ -│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ -│doi (after the first release).                                                                  │ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - -╭─[!] 1 Module Test Warning──────────────────────────────────────────────────────────────────────╮ -│                              ╷                        ╷                                          │ -│Module name                 │File path             │Test message                            │ -│╶─────────────────────────────┼────────────────────────┼─────────────────────────────────────────╴│ -│fastqc                      │modules/nf-core/fastqc│New version available                   │ -│                              ╵                        ╵                                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY │ -├───────────────────────┤ -│[✔] 198 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   2 Test Warnings│ -│[✗]   0 Tests Failed│ -╰───────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Testing pipeline: . + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos                                                                  │ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ +││ +│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ +│doi (after the first release).                                                                  │ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔] 188 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index d94144af44..ff51d523e4 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,111 +19,111 @@ font-weight: 700; } - .terminal-3256931459-matrix { + .terminal-822138325-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3256931459-title { + .terminal-822138325-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3256931459-r1 { fill: #c5c8c6 } -.terminal-3256931459-r2 { fill: #98a84b } -.terminal-3256931459-r3 { fill: #9a9b99 } -.terminal-3256931459-r4 { fill: #608ab1 } -.terminal-3256931459-r5 { fill: #d0b344 } -.terminal-3256931459-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3256931459-r7 { fill: #868887 } + .terminal-822138325-r1 { fill: #c5c8c6 } +.terminal-822138325-r2 { fill: #98a84b } +.terminal-822138325-r3 { fill: #9a9b99 } +.terminal-822138325-r4 { fill: #608ab1 } +.terminal-822138325-r5 { fill: #d0b344 } +.terminal-822138325-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-822138325-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -135,37 +135,37 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnafusion            │   125 │          3.0.1 │ 2 months ago │           - │ -                   │ -│ smrnaseq             │    63 │          2.2.4 │ 3 months ago │           - │ -                   │ -│ circrna              │    33 │            dev │   2 days ago │           - │ -                   │ -│ differentialabundan… │    35 │          1.4.0 │ 2 months ago │           - │ -                   │ -│ spatialtranscriptom… │    36 │            dev │   6 days ago │           - │ -                   │ -│ scrnaseq             │   122 │          2.5.1 │  1 weeks ago │           - │ -                   │ -│ rnaseq               │   724 │         3.14.0 │  3 weeks ago │           - │ -                   │ -│ rnasplice            │    20 │          1.0.2 │  3 weeks ago │           - │ -                   │ -│ dualrnaseq           │    15 │          1.0.0 │  3 years ago │           - │ -                   │ -│ marsseq              │     5 │          1.0.3 │ 7 months ago │           - │ -                   │ -│ lncpipe              │    27 │            dev │  1 years ago │           - │ -                   │ -│ scflow               │    23 │            dev │  2 years ago │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnasplice            │    22 │          1.0.2 │ 1 months ago │           - │ -                   │ +│ circrna              │    34 │            dev │    yesterday │           - │ -                   │ +│ rnaseq               │   735 │         3.14.0 │ 1 months ago │           - │ -                   │ +│ smrnaseq             │    64 │          2.2.4 │ 4 months ago │           - │ -                   │ +│ scrnaseq             │   124 │          2.5.1 │  4 weeks ago │           - │ -                   │ +│ differentialabundan… │    38 │          1.4.0 │ 3 months ago │           - │ -                   │ +│ rnafusion            │   126 │          3.0.1 │ 3 months ago │           - │ -                   │ +│ spatialtranscriptom… │    36 │            dev │  3 weeks ago │           - │ -                   │ +│ dualrnaseq           │    16 │          1.0.0 │  3 years ago │           - │ -                   │ +│ marsseq              │     5 │          1.0.3 │ 7 months ago │           - │ -                   │ +│ lncpipe              │    28 │            dev │  1 years ago │           - │ -                   │ +│ scflow               │    24 │            dev │  2 years ago │           - │ -                   │ +└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 8f9f0d1072..4b020b0dc7 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-3218995592-matrix { + .terminal-2391931080-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3218995592-title { + .terminal-2391931080-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3218995592-r1 { fill: #c5c8c6 } -.terminal-3218995592-r2 { fill: #98a84b } -.terminal-3218995592-r3 { fill: #9a9b99 } -.terminal-3218995592-r4 { fill: #608ab1 } -.terminal-3218995592-r5 { fill: #d0b344 } -.terminal-3218995592-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3218995592-r7 { fill: #868887 } -.terminal-3218995592-r8 { fill: #868887;font-style: italic; } + .terminal-2391931080-r1 { fill: #c5c8c6 } +.terminal-2391931080-r2 { fill: #98a84b } +.terminal-2391931080-r3 { fill: #9a9b99 } +.terminal-2391931080-r4 { fill: #608ab1 } +.terminal-2391931080-r5 { fill: #d0b344 } +.terminal-2391931080-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2391931080-r7 { fill: #868887 } +.terminal-2391931080-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq              │   724 │         3.14.0 │   3 weeks ago │           - │ -                   │ -│ sarek               │   305 │          3.4.0 │  3 months ago │           - │ -                   │ -│ mag                 │   167 │          2.5.1 │  3 months ago │           - │ -                   │ -│ chipseq             │   159 │          2.0.0 │   1 years ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq              │   735 │         3.14.0 │  1 months ago │           - │ -                   │ +│ sarek               │   312 │          3.4.0 │  3 months ago │           - │ -                   │ +│ mag                 │   170 │          2.5.4 │    1 week ago │           - │ -                   │ +│ chipseq             │   160 │          2.0.0 │   1 years ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index bbcd1d7782..d34af82bb2 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-2091548930-matrix { + .terminal-1084195510-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2091548930-title { + .terminal-1084195510-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2091548930-r1 { fill: #c5c8c6 } -.terminal-2091548930-r2 { fill: #98a84b } -.terminal-2091548930-r3 { fill: #9a9b99 } -.terminal-2091548930-r4 { fill: #608ab1 } -.terminal-2091548930-r5 { fill: #d0b344 } -.terminal-2091548930-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2091548930-r7 { fill: #868887 } -.terminal-2091548930-r8 { fill: #868887;font-style: italic; } + .terminal-1084195510-r1 { fill: #c5c8c6 } +.terminal-1084195510-r2 { fill: #98a84b } +.terminal-1084195510-r3 { fill: #9a9b99 } +.terminal-1084195510-r4 { fill: #608ab1 } +.terminal-1084195510-r5 { fill: #d0b344 } +.terminal-1084195510-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1084195510-r7 { fill: #868887 } +.terminal-1084195510-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ raredisease         │    64 │          1.1.1 │  6 months ago │           - │ -                   │ -│ funcscan            │    49 │          1.1.4 │  3 months ago │           - │ -                   │ -│ circdna             │    21 │          1.0.4 │  7 months ago │           - │ -                   │ -│ mhcquant            │    29 │          2.5.0 │  4 months ago │           - │ -                   │ -│ fetchngs            │    97 │         1.11.0 │  4 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ fetchngs            │   100 │         1.11.0 │  4 months ago │           - │ -                   │ +│ phageannotator      │     7 │            dev │  20 hours ago │           - │ -                   │ +│ riboseq             │     2 │            dev │  21 hours ago │           - │ -                   │ +│ bacass              │    48 │          2.1.0 │  4 months ago │           - │ -                   │ +│ rnasplice           │    22 │          1.0.2 │  1 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 0b8f9dbeae..c886806891 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-1422915651-matrix { + .terminal-692975589-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1422915651-title { + .terminal-692975589-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1422915651-r1 { fill: #c5c8c6 } -.terminal-1422915651-r2 { fill: #98a84b } -.terminal-1422915651-r3 { fill: #9a9b99 } -.terminal-1422915651-r4 { fill: #608ab1 } -.terminal-1422915651-r5 { fill: #d0b344 } -.terminal-1422915651-r6 { fill: #98a84b;font-weight: bold } -.terminal-1422915651-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-692975589-r1 { fill: #c5c8c6 } +.terminal-692975589-r2 { fill: #98a84b } +.terminal-692975589-r3 { fill: #9a9b99 } +.terminal-692975589-r4 { fill: #608ab1 } +.terminal-692975589-r5 { fill: #d0b344 } +.terminal-692975589-r6 { fill: #98a84b;font-weight: bold } +.terminal-692975589-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index 19855f1cc4..aeb42f1255 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' -INFO     Could not find a Docker/Singularity container (Unexpected response code `503` for           -https://api.biocontainers.pro/ga4gh/trs/v2/tools/fastqc/versions/fastqc-0.12.1) -INFO     Created component template: 'fastqc' -INFO     Created following files:                                                                    -           modules/nf-core/fastqc/main.nf                                                            -           modules/nf-core/fastqc/meta.yml                                                           -           modules/nf-core/fastqc/environment.yml                                                    -           modules/nf-core/fastqc/tests/tags.yml                                                     -           modules/nf-core/fastqc/tests/main.nf.test                                                 + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' +INFO     Using Docker container: 'biocontainers/fastqc:0.12.1--hdfd78af_0' +INFO     Using Singularity container:                                                                +'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' +INFO     Created component template: 'fastqc' +INFO     Created following files:                                                                    +           modules/nf-core/fastqc/main.nf                                                            +           modules/nf-core/fastqc/meta.yml                                                           +           modules/nf-core/fastqc/environment.yml                                                    +           modules/nf-core/fastqc/tests/tags.yml                                                     +           modules/nf-core/fastqc/tests/main.nf.test                                                 diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index 13794a61df..a35ecdec75 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,163 +19,163 @@ font-weight: 700; } - .terminal-729543216-matrix { + .terminal-3092902354-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-729543216-title { + .terminal-3092902354-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-729543216-r1 { fill: #c5c8c6 } -.terminal-729543216-r2 { fill: #98a84b } -.terminal-729543216-r3 { fill: #9a9b99 } -.terminal-729543216-r4 { fill: #608ab1 } -.terminal-729543216-r5 { fill: #d0b344 } -.terminal-729543216-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-729543216-r7 { fill: #98a84b;font-weight: bold } -.terminal-729543216-r8 { fill: #868887 } -.terminal-729543216-r9 { fill: #d08442 } -.terminal-729543216-r10 { fill: #868887;font-style: italic; } -.terminal-729543216-r11 { fill: #98729f } + .terminal-3092902354-r1 { fill: #c5c8c6 } +.terminal-3092902354-r2 { fill: #98a84b } +.terminal-3092902354-r3 { fill: #9a9b99 } +.terminal-3092902354-r4 { fill: #608ab1 } +.terminal-3092902354-r5 { fill: #d0b344 } +.terminal-3092902354-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3092902354-r7 { fill: #98a84b;font-weight: bold } +.terminal-3092902354-r8 { fill: #868887 } +.terminal-3092902354-r9 { fill: #d08442 } +.terminal-3092902354-r10 { fill: #868887;font-style: italic; } +.terminal-3092902354-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -187,53 +187,53 @@ - + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (file) │List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (file) │List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index 0f7d460ad9..d0ac5b2946 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-235834510-matrix { + .terminal-2787740720-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-235834510-title { + .terminal-2787740720-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-235834510-r1 { fill: #c5c8c6 } -.terminal-235834510-r2 { fill: #98a84b } -.terminal-235834510-r3 { fill: #9a9b99 } -.terminal-235834510-r4 { fill: #608ab1 } -.terminal-235834510-r5 { fill: #d0b344 } + .terminal-2787740720-r1 { fill: #c5c8c6 } +.terminal-2787740720-r2 { fill: #98a84b } +.terminal-2787740720-r3 { fill: #9a9b99 } +.terminal-2787740720-r4 { fill: #608ab1 } +.terminal-2787740720-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Installing 'abacas' -INFO     Use the following statement to include this module:                                         - - include { ABACAS } from '../modules/nf-core/abacas/main'                                            - + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Installing 'abacas' +INFO     Use the following statement to include this module:                                         + + include { ABACAS } from '../modules/nf-core/abacas/main'                                            + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index e82458136e..631e065df7 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -19,101 +19,101 @@ font-weight: 700; } - .terminal-3818930560-matrix { + .terminal-3871949092-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3818930560-title { + .terminal-3871949092-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3818930560-r1 { fill: #c5c8c6 } -.terminal-3818930560-r2 { fill: #98a84b } -.terminal-3818930560-r3 { fill: #9a9b99 } -.terminal-3818930560-r4 { fill: #608ab1 } -.terminal-3818930560-r5 { fill: #d0b344 } -.terminal-3818930560-r6 { fill: #68a0b3;font-weight: bold } -.terminal-3818930560-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-3818930560-r8 { fill: #98a84b;font-weight: bold } -.terminal-3818930560-r9 { fill: #cc555a } + .terminal-3871949092-r1 { fill: #c5c8c6 } +.terminal-3871949092-r2 { fill: #98a84b } +.terminal-3871949092-r3 { fill: #9a9b99 } +.terminal-3871949092-r4 { fill: #608ab1 } +.terminal-3871949092-r5 { fill: #d0b344 } +.terminal-3871949092-r6 { fill: #68a0b3;font-weight: bold } +.terminal-3871949092-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3871949092-r8 { fill: #98a84b;font-weight: bold } +.terminal-3871949092-r9 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -125,33 +125,33 @@ - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' -INFO     Found 4 inputs in modules/nf-core/multiqc/main.nf                                           -INFO     Found 4 outputs in modules/nf-core/multiqc/main.nf                                          - -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  57 Tests Passed │ -│[!]   0 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' +INFO     Found 4 inputs in modules/nf-core/multiqc/main.nf                                           +INFO     Found 4 outputs in modules/nf-core/multiqc/main.nf                                          + +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  59 Tests Passed │ +│[!]   0 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index 2d4759e444..4b06362a82 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - + - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                                   - -self.repo_type='pipeline' -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name        ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ custom/dumpsoftwar… │ https://github.co… │ 8ec825f465b9c17f9d… │ Bump MultiQC       │ 2024-01-10 │ -│                     │                    │                     │ version to 1.19    │            │ -│                     │                    │                     │ (#4705)            │            │ -│ fastqc              │ https://github.co… │ c9488585ce7bd35ccd… │ CHORES: update     │ 2024-01-18 │ -│                     │                    │                     │ fasqc tests with   │            │ -│                     │                    │                     │ new data           │            │ -│                     │                    │                     │ organisation       │            │ -│                     │                    │                     │ (#4760)            │            │ -[..truncated..] + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Modules installed in '.':                                                                   + +self.repo_type='pipeline' +┏━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name┃Repository           ┃Version SHA          ┃Message              ┃Date      ┃ +┡━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ fastqc      │ https://github.com/n… │ f4ae1d942bd50c5c0b9b… │ Update FASTQC to use  │ 2024-01-31 │ +│             │                       │                       │ unique names for      │            │ +│             │                       │                       │ snapshots (#4825)     │            │ +│ multiqc     │ https://github.com/n… │ ccacf6f5de6df3bc6d73… │ Bump Multiqc to 1.20  │ 2024-02-14 │ +│             │                       │                       │ (#4910)               │            │ +└─────────────┴───────────────────────┴───────────────────────┴───────────────────────┴────────────┘ diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 6848284ec6..aca2dc7d9b 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2238208439-matrix { + .terminal-2422626649-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2238208439-title { + .terminal-2422626649-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2238208439-r1 { fill: #c5c8c6 } -.terminal-2238208439-r2 { fill: #98a84b } -.terminal-2238208439-r3 { fill: #9a9b99 } -.terminal-2238208439-r4 { fill: #608ab1 } -.terminal-2238208439-r5 { fill: #d0b344 } -.terminal-2238208439-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-2238208439-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2238208439-r8 { fill: #868887;font-style: italic; } + .terminal-2422626649-r1 { fill: #c5c8c6 } +.terminal-2422626649-r2 { fill: #98a84b } +.terminal-2422626649-r3 { fill: #9a9b99 } +.terminal-2422626649-r4 { fill: #608ab1 } +.terminal-2422626649-r5 { fill: #d0b344 } +.terminal-2422626649-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-2422626649-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2422626649-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Modules available from https://github.com/nf-core/modules.git(master):                     - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                                          ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                                │ -│ abricate/run                                          │ -│ abricate/summary                                      │ -│ abritamr/run                                          │ -│ adapterremoval                                        │ -│ adapterremovalfixprefix                               │ -│ admixture                                             │ -│ affy/justrma                                          │ -│ agat/convertspgff2gtf                                 │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):                     + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                                          ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                                │ +│ abricate/run                                          │ +│ abricate/summary                                      │ +│ abritamr/run                                          │ +│ adapterremoval                                        │ +│ adapterremovalfixprefix                               │ +│ admixture                                             │ +│ affy/justrma                                          │ +│ agat/convertspgff2gtf                                 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index ca6445e5f4..84ad24867e 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -19,127 +19,127 @@ font-weight: 700; } - .terminal-3644795976-matrix { + .terminal-3587976170-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3644795976-title { + .terminal-3587976170-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3644795976-r1 { fill: #c5c8c6 } -.terminal-3644795976-r2 { fill: #98a84b } -.terminal-3644795976-r3 { fill: #9a9b99 } -.terminal-3644795976-r4 { fill: #608ab1 } -.terminal-3644795976-r5 { fill: #d0b344 } -.terminal-3644795976-r6 { fill: #ff2627 } -.terminal-3644795976-r7 { fill: #00823d } -.terminal-3644795976-r8 { fill: #ff2c7a;font-weight: bold } + .terminal-3587976170-r1 { fill: #c5c8c6 } +.terminal-3587976170-r2 { fill: #98a84b } +.terminal-3587976170-r3 { fill: #9a9b99 } +.terminal-3587976170-r4 { fill: #608ab1 } +.terminal-3587976170-r5 { fill: #d0b344 } +.terminal-3587976170-r6 { fill: #ff2627 } +.terminal-3587976170-r7 { fill: #00823d } +.terminal-3587976170-r8 { fill: #ff2c7a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -151,42 +151,42 @@ - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Changes in module 'nf-core/fastqc' -INFO    'modules/nf-core/fastqc/meta.yml' is unchanged                                              -INFO    'modules/nf-core/fastqc/environment.yml' is unchanged                                       -INFO     Changes in 'fastqc/main.nf':                                                                - ---- modules/nf-core/fastqc/main.nf -+++ modules/nf-core/fastqc/main.nf -@@ -1,6 +1,6 @@ -process FASTQC {                                                                                   -    tag "$meta.id"                                                                                 --    label 'process_medium' -+    label 'process_low' - -    conda "${moduleDir}/environment.yml"                                                           -    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  - - -INFO    'modules/nf-core/fastqc/tests/main.nf.test' is unchanged                                    -INFO    'modules/nf-core/fastqc/tests/tags.yml' is unchanged                                        -INFO    'modules/nf-core/fastqc/tests/main.nf.test.snap' is unchanged                               -INFO     Patch file of 'modules/nf-core/fastqc' written to 'modules/nf-core/fastqc/fastqc.diff' + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Changes in module 'nf-core/fastqc' +INFO    'modules/nf-core/fastqc/environment.yml' is unchanged                                       +INFO    'modules/nf-core/fastqc/meta.yml' is unchanged                                              +INFO     Changes in 'fastqc/main.nf':                                                                + +--- modules/nf-core/fastqc/main.nf ++++ modules/nf-core/fastqc/main.nf +@@ -1,6 +1,6 @@ +process FASTQC {                                                                                   +    tag "$meta.id"                                                                                 +-    label 'process_medium' ++    label 'process_low' + +    conda "${moduleDir}/environment.yml"                                                           +    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  + + +INFO    'modules/nf-core/fastqc/tests/tags.yml' is unchanged                                        +INFO    'modules/nf-core/fastqc/tests/main.nf.test' is unchanged                                    +INFO    'modules/nf-core/fastqc/tests/main.nf.test.snap' is unchanged                               +INFO     Patch file of 'modules/nf-core/fastqc' written to 'modules/nf-core/fastqc/fastqc.diff' diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index c8ac631ad5..c78034e35c 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-2072587744-matrix { + .terminal-3332058498-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2072587744-title { + .terminal-3332058498-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2072587744-r1 { fill: #c5c8c6 } -.terminal-2072587744-r2 { fill: #98a84b } -.terminal-2072587744-r3 { fill: #9a9b99 } -.terminal-2072587744-r4 { fill: #608ab1 } -.terminal-2072587744-r5 { fill: #d0b344 } + .terminal-3332058498-r1 { fill: #c5c8c6 } +.terminal-3332058498-r2 { fill: #98a84b } +.terminal-3332058498-r3 { fill: #9a9b99 } +.terminal-3332058498-r4 { fill: #608ab1 } +.terminal-3332058498-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Removed files for 'abacas' and its dependencies 'abacas'.                                   + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Removed files for 'abacas' and its dependencies 'abacas'.                                   diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 9520b174af..daff523354 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-4272025854-matrix { + .terminal-2250961056-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4272025854-title { + .terminal-2250961056-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4272025854-r1 { fill: #c5c8c6 } -.terminal-4272025854-r2 { fill: #98a84b } -.terminal-4272025854-r3 { fill: #9a9b99 } -.terminal-4272025854-r4 { fill: #608ab1 } -.terminal-4272025854-r5 { fill: #d0b344 } + .terminal-2250961056-r1 { fill: #c5c8c6 } +.terminal-2250961056-r2 { fill: #98a84b } +.terminal-2250961056-r3 { fill: #9a9b99 } +.terminal-2250961056-r4 { fill: #608ab1 } +.terminal-2250961056-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules test fastqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Generating nf-test snapshot                                                                 + + $ nf-core modules test fastqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Generating nf-test snapshot                                                                 diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index fd13131174..ddc5facf65 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                              -INFO    'modules/nf-core/custom/dumpsoftwareversions' is already up to date                         -INFO     Updating 'nf-core/fastqc' -INFO    'modules/nf-core/multiqc' is already up to date                                             -INFO     Updates complete ✨                                                                         + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                              +INFO    'modules/nf-core/fastqc' is already up to date                                              +INFO    'modules/nf-core/multiqc' is already up to date                                             +INFO     Updates complete ✨                                                                         diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 6875c1282b..cf0ab90005 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-848808703-matrix { + .terminal-2534001313-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-848808703-title { + .terminal-2534001313-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-848808703-r1 { fill: #c5c8c6 } -.terminal-848808703-r2 { fill: #98a84b } -.terminal-848808703-r3 { fill: #9a9b99 } -.terminal-848808703-r4 { fill: #608ab1 } -.terminal-848808703-r5 { fill: #d0b344 } -.terminal-848808703-r6 { fill: #98a84b;font-weight: bold } -.terminal-848808703-r7 { fill: #868887;font-weight: bold } -.terminal-848808703-r8 { fill: #868887 } -.terminal-848808703-r9 { fill: #4e707b;font-weight: bold } -.terminal-848808703-r10 { fill: #68a0b3;font-weight: bold } + .terminal-2534001313-r1 { fill: #c5c8c6 } +.terminal-2534001313-r2 { fill: #98a84b } +.terminal-2534001313-r3 { fill: #9a9b99 } +.terminal-2534001313-r4 { fill: #608ab1 } +.terminal-2534001313-r5 { fill: #d0b344 } +.terminal-2534001313-r6 { fill: #98a84b;font-weight: bold } +.terminal-2534001313-r7 { fill: #868887;font-weight: bold } +.terminal-2534001313-r8 { fill: #868887 } +.terminal-2534001313-r9 { fill: #4e707b;font-weight: bold } +.terminal-2534001313-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 30 params) -INFO     Writing schema with 31 params: 'nextflow_schema.json' + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 30 params) +INFO     Writing schema with 31 params: 'nextflow_schema.json' diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index 751f284e59..fc3b8f5f31 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-3463596934-matrix { + .terminal-3503377192-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3463596934-title { + .terminal-3503377192-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3463596934-r1 { fill: #c5c8c6 } -.terminal-3463596934-r2 { fill: #98a84b } -.terminal-3463596934-r3 { fill: #9a9b99 } -.terminal-3463596934-r4 { fill: #608ab1 } -.terminal-3463596934-r5 { fill: #d0b344 } -.terminal-3463596934-r6 { fill: #98a84b;font-weight: bold } -.terminal-3463596934-r7 { fill: #868887;font-weight: bold } -.terminal-3463596934-r8 { fill: #868887 } -.terminal-3463596934-r9 { fill: #4e707b;font-weight: bold } + .terminal-3503377192-r1 { fill: #c5c8c6 } +.terminal-3503377192-r2 { fill: #98a84b } +.terminal-3503377192-r3 { fill: #9a9b99 } +.terminal-3503377192-r4 { fill: #608ab1 } +.terminal-3503377192-r5 { fill: #d0b344 } +.terminal-3503377192-r6 { fill: #98a84b;font-weight: bold } +.terminal-3503377192-r7 { fill: #868887;font-weight: bold } +.terminal-3503377192-r8 { fill: #868887 } +.terminal-3503377192-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 31 params) + + $ nf-core schema lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 31 params) diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 199c1cb0ea..c5bbd59a37 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,71 +19,71 @@ font-weight: 700; } - .terminal-2317352182-matrix { + .terminal-2855795864-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2317352182-title { + .terminal-2855795864-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2317352182-r1 { fill: #c5c8c6 } -.terminal-2317352182-r2 { fill: #98a84b } -.terminal-2317352182-r3 { fill: #9a9b99 } -.terminal-2317352182-r4 { fill: #608ab1 } -.terminal-2317352182-r5 { fill: #d0b344 } -.terminal-2317352182-r6 { fill: #98a84b;font-weight: bold } -.terminal-2317352182-r7 { fill: #868887;font-weight: bold } -.terminal-2317352182-r8 { fill: #868887 } -.terminal-2317352182-r9 { fill: #4e707b;font-weight: bold } + .terminal-2855795864-r1 { fill: #c5c8c6 } +.terminal-2855795864-r2 { fill: #98a84b } +.terminal-2855795864-r3 { fill: #9a9b99 } +.terminal-2855795864-r4 { fill: #608ab1 } +.terminal-2855795864-r5 { fill: #d0b344 } +.terminal-2855795864-r6 { fill: #98a84b;font-weight: bold } +.terminal-2855795864-r7 { fill: #868887;font-weight: bold } +.terminal-2855795864-r8 { fill: #868887 } +.terminal-2855795864-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,23 +95,23 @@ - + - - $ nf-core schema validate nf-core-rnaseq/3_8 nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 93 params) -INFO    [✓] Input parameters look valid + + $ nf-core schema validate nf-core-rnaseq/3_8 nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 93 params) +INFO    [✓] Input parameters look valid diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index 8ce6f0cc8d..e39d489764 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -19,89 +19,89 @@ font-weight: 700; } - .terminal-519671195-matrix { + .terminal-1669631293-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-519671195-title { + .terminal-1669631293-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-519671195-r1 { fill: #c5c8c6 } -.terminal-519671195-r2 { fill: #98a84b } -.terminal-519671195-r3 { fill: #9a9b99 } -.terminal-519671195-r4 { fill: #608ab1 } -.terminal-519671195-r5 { fill: #d0b344 } -.terminal-519671195-r6 { fill: #68a0b3;font-weight: bold } + .terminal-1669631293-r1 { fill: #c5c8c6 } +.terminal-1669631293-r2 { fill: #98a84b } +.terminal-1669631293-r3 { fill: #9a9b99 } +.terminal-1669631293-r4 { fill: #608ab1 } +.terminal-1669631293-r5 { fill: #d0b344 } +.terminal-1669631293-r6 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -113,30 +113,30 @@ - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Created component template: 'bam_stats_samtools' -INFO     Created following files:                                                                    -           subworkflows/nf-core/bam_stats_samtools/main.nf                                           -           subworkflows/nf-core/bam_stats_samtools/meta.yml                                          -           subworkflows/nf-core/bam_stats_samtools/tests/tags.yml                                    -           subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test                                + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Created component template: 'bam_stats_samtools' +INFO     Created following files:                                                                    +           subworkflows/nf-core/bam_stats_samtools/main.nf                                           +           subworkflows/nf-core/bam_stats_samtools/meta.yml                                          +           subworkflows/nf-core/bam_stats_samtools/tests/tags.yml                                    +           subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test                                diff --git a/docs/images/nf-core-subworkflows-info.svg b/docs/images/nf-core-subworkflows-info.svg index 85cfb8a2f1..5e792aaf27 100644 --- a/docs/images/nf-core-subworkflows-info.svg +++ b/docs/images/nf-core-subworkflows-info.svg @@ -19,112 +19,112 @@ font-weight: 700; } - .terminal-2433004757-matrix { + .terminal-3309876258-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2433004757-title { + .terminal-3309876258-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2433004757-r1 { fill: #c5c8c6 } -.terminal-2433004757-r2 { fill: #98a84b } -.terminal-2433004757-r3 { fill: #9a9b99 } -.terminal-2433004757-r4 { fill: #608ab1 } -.terminal-2433004757-r5 { fill: #d0b344 } -.terminal-2433004757-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2433004757-r7 { fill: #98a84b;font-weight: bold } -.terminal-2433004757-r8 { fill: #868887 } -.terminal-2433004757-r9 { fill: #868887;font-style: italic; } -.terminal-2433004757-r10 { fill: #d08442 } -.terminal-2433004757-r11 { fill: #98729f } + .terminal-3309876258-r1 { fill: #c5c8c6 } +.terminal-3309876258-r2 { fill: #98a84b } +.terminal-3309876258-r3 { fill: #9a9b99 } +.terminal-3309876258-r4 { fill: #608ab1 } +.terminal-3309876258-r5 { fill: #d0b344 } +.terminal-3309876258-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3309876258-r7 { fill: #98a84b;font-weight: bold } +.terminal-3309876258-r8 { fill: #868887 } +.terminal-3309876258-r9 { fill: #868887;font-style: italic; } +.terminal-3309876258-r10 { fill: #d08442 } +.terminal-3309876258-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -136,36 +136,36 @@ - - - - $ nf-core subworkflows info bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -╭─ Subworkflow: bam_rseqc  ────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 📖 Description: Subworkflow to run multiple commands in the RSeqC package                        │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -[..truncated..] - ch_readduplication_rscript  (file)│script to reproduce the plot        │                      *.R -╶───────────────────────────────────┼────────────────────────────────────┼─────────────────────────╴ - ch_tin  (file)                    │TXT file containing tin.py results  │                    *.txt -                                    │summary                             │ -╶───────────────────────────────────┼────────────────────────────────────┼─────────────────────────╴ - versions  (file)                  │File containing software versions   │             versions.yml -                                    ╵                                    ╵                           - - 💻  Installation command: nf-core subworkflows install bam_rseqc - + + + + $ nf-core subworkflows info bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +╭─ Subworkflow: bam_rseqc  ────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 📖 Description: Subworkflow to run multiple commands in the RSeqC package                        │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +[..truncated..] + readduplication_rscript  (file)    │script to reproduce the plot       │                      *.R +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + tin_txt  (file)                    │TXT file containing tin.py results │                    *.txt +                                     │summary                            │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + versions  (file)                   │File containing software versions  │             versions.yml +                                     ╵                                   ╵                           + + 💻  Installation command: nf-core subworkflows install bam_rseqc + diff --git a/docs/images/nf-core-subworkflows-install.svg b/docs/images/nf-core-subworkflows-install.svg index d282df2f7f..84c9fe6ac9 100644 --- a/docs/images/nf-core-subworkflows-install.svg +++ b/docs/images/nf-core-subworkflows-install.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-3559394546-matrix { + .terminal-964365460-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3559394546-title { + .terminal-964365460-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3559394546-r1 { fill: #c5c8c6 } -.terminal-3559394546-r2 { fill: #98a84b } -.terminal-3559394546-r3 { fill: #9a9b99 } -.terminal-3559394546-r4 { fill: #608ab1 } -.terminal-3559394546-r5 { fill: #d0b344 } + .terminal-964365460-r1 { fill: #c5c8c6 } +.terminal-964365460-r2 { fill: #98a84b } +.terminal-964365460-r3 { fill: #9a9b99 } +.terminal-964365460-r4 { fill: #608ab1 } +.terminal-964365460-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core subworkflows install bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Installing 'bam_rseqc' + + $ nf-core subworkflows install bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Installing 'bam_rseqc' diff --git a/docs/images/nf-core-subworkflows-lint.svg b/docs/images/nf-core-subworkflows-lint.svg index aa67f0ca26..3199189c16 100644 --- a/docs/images/nf-core-subworkflows-lint.svg +++ b/docs/images/nf-core-subworkflows-lint.svg @@ -19,239 +19,239 @@ font-weight: 700; } - .terminal-84539906-matrix { + .terminal-785643940-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-84539906-title { + .terminal-785643940-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-84539906-r1 { fill: #c5c8c6 } -.terminal-84539906-r2 { fill: #98a84b } -.terminal-84539906-r3 { fill: #9a9b99 } -.terminal-84539906-r4 { fill: #608ab1 } -.terminal-84539906-r5 { fill: #d0b344 } -.terminal-84539906-r6 { fill: #d0b344;font-weight: bold } -.terminal-84539906-r7 { fill: #8d7b39 } -.terminal-84539906-r8 { fill: #68a0b3;font-weight: bold } -.terminal-84539906-r9 { fill: #d0b344;font-style: italic; } -.terminal-84539906-r10 { fill: #c5c8c6;font-weight: bold } -.terminal-84539906-r11 { fill: #98a84b;font-weight: bold } -.terminal-84539906-r12 { fill: #cc555a } + .terminal-785643940-r1 { fill: #c5c8c6 } +.terminal-785643940-r2 { fill: #98a84b } +.terminal-785643940-r3 { fill: #9a9b99 } +.terminal-785643940-r4 { fill: #608ab1 } +.terminal-785643940-r5 { fill: #d0b344 } +.terminal-785643940-r6 { fill: #d0b344;font-weight: bold } +.terminal-785643940-r7 { fill: #8d7b39 } +.terminal-785643940-r8 { fill: #68a0b3;font-weight: bold } +.terminal-785643940-r9 { fill: #d0b344;font-style: italic; } +.terminal-785643940-r10 { fill: #c5c8c6;font-weight: bold } +.terminal-785643940-r11 { fill: #98a84b;font-weight: bold } +.terminal-785643940-r12 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -263,78 +263,78 @@ - + - - $ nf-core subworkflows lint bam_stats_samtools - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting subworkflow: 'bam_stats_samtools' - -╭─[!] 14 Subworkflow Test Warnings───────────────────────────────────────────────────────────────╮ -│                     ╷                                     ╷                                      │ -│Subworkflow name   │File path                          │Test message                        │ -│╶────────────────────┼─────────────────────────────────────┼─────────────────────────────────────╴│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: //Add │ -│││all required assertions to verify │ -│││the test output.│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Add │ -│││tags for all modules used within │ -│││this subworkflow. Example:│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Change │ -│││the test name preferably indicating │ -│││the test-data and file-format used│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Once │ -│││you have added the required tests, │ -│││please run the following command to │ -│││build this file:│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: define │ -│││inputs of the workflow here. │ -│││Example:│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: A │ -│││subworkflow SHOULD import at least │ -│││two modules│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: If in doubt │ -│││look at other nf-core/subworkflows │ -│││to see how we are doing things! :)│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit emitted│ -│││channels│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit input │ -│││(take) channels│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: substitute │ -│││modules here for the modules of your│ -│││subworkflow│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a │ -│││description of the subworkflow and │ -│││list keywords│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a list│ -│││of the modules and/or subworkflows │ -│││used in the subworkflow│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ -│││of the channels used as input with a│ -│││description and their structure│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ -│││of the channels used as output with │ -│││a descriptions and their structure│ -│                     ╵                                     ╵                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  42 Tests Passed │ -│[!]  14 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + $ nf-core subworkflows lint bam_stats_samtools + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting subworkflow: 'bam_stats_samtools' + +╭─[!] 14 Subworkflow Test Warnings───────────────────────────────────────────────────────────────╮ +│                     ╷                                     ╷                                      │ +│Subworkflow name   │File path                          │Test message                        │ +│╶────────────────────┼─────────────────────────────────────┼─────────────────────────────────────╴│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: //Add │ +│││all required assertions to verify │ +│││the test output.│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Add │ +│││tags for all modules used within │ +│││this subworkflow. Example:│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Change │ +│││the test name preferably indicating │ +│││the test-data and file-format used│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Once │ +│││you have added the required tests, │ +│││please run the following command to │ +│││build this file:│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: define │ +│││inputs of the workflow here. │ +│││Example:│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: A │ +│││subworkflow SHOULD import at least │ +│││two modules│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: If in doubt │ +│││look at other nf-core/subworkflows │ +│││to see how we are doing things! :)│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit emitted│ +│││channels│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit input │ +│││(take) channels│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: substitute │ +│││modules here for the modules of your│ +│││subworkflow│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a │ +│││description of the subworkflow and │ +│││list keywords│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a list│ +│││of the modules and/or subworkflows │ +│││used in the subworkflow│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ +│││of the channels used as input with a│ +│││description and their structure│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ +│││of the channels used as output with │ +│││a descriptions and their structure│ +│                     ╵                                     ╵                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  42 Tests Passed │ +│[!]  14 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-subworkflows-list-local.svg b/docs/images/nf-core-subworkflows-list-local.svg index 2b72e839d0..04a21b7708 100644 --- a/docs/images/nf-core-subworkflows-list-local.svg +++ b/docs/images/nf-core-subworkflows-list-local.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core subworkflows list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     No nf-core subworkflows found in '.' -self.repo_type='pipeline' - + + $ nf-core subworkflows list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Subworkflows installed in '.':                                                              + +self.repo_type='pipeline' +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Subworkflow Name   ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ utils_nextflow_pip… │ https://github.co… │ cd08c91373cd00a732… │ nicer assertions   │ 2024-01-19 │ +│                     │                    │                     │ in utils           │            │ +│                     │                    │                     │ subworkflows       │            │ +│                     │                    │                     │ (#4779)            │            │ +│ utils_nfcore_pipel… │ https://github.co… │ 262b17ed2aad591039… │ Update             │ 2024-02-12 │ +│                     │                    │                     │ utils-nfcore-pipe… │            │ +│                     │                    │                     │ add multiqc report │            │ +│                     │                    │                     │ to completion      │            │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-list-remote.svg b/docs/images/nf-core-subworkflows-list-remote.svg index 5158fb8c88..28754bea25 100644 --- a/docs/images/nf-core-subworkflows-list-remote.svg +++ b/docs/images/nf-core-subworkflows-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2156284683-matrix { + .terminal-3053865645-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2156284683-title { + .terminal-3053865645-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2156284683-r1 { fill: #c5c8c6 } -.terminal-2156284683-r2 { fill: #98a84b } -.terminal-2156284683-r3 { fill: #9a9b99 } -.terminal-2156284683-r4 { fill: #608ab1 } -.terminal-2156284683-r5 { fill: #d0b344 } -.terminal-2156284683-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-2156284683-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2156284683-r8 { fill: #868887;font-style: italic; } + .terminal-3053865645-r1 { fill: #c5c8c6 } +.terminal-3053865645-r2 { fill: #98a84b } +.terminal-3053865645-r3 { fill: #9a9b99 } +.terminal-3053865645-r4 { fill: #608ab1 } +.terminal-3053865645-r5 { fill: #d0b344 } +.terminal-3053865645-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-3053865645-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-3053865645-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core subworkflows list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Subworkflow Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ bam_cnv_wisecondorx                           │ -│ bam_create_som_pon_gatk                       │ -│ bam_dedup_stats_samtools_umitools             │ -│ bam_docounts_contamination_angsd              │ -│ bam_markduplicates_picard                     │ -│ bam_markduplicates_samtools                   │ -│ bam_ngscheckmate                              │ -│ bam_qc_picard                                 │ -│ bam_rseqc                                     │ -[..truncated..] + + $ nf-core subworkflows list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Subworkflow Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ bam_cnv_wisecondorx                           │ +│ bam_create_som_pon_gatk                       │ +│ bam_dedup_stats_samtools_umitools             │ +│ bam_docounts_contamination_angsd              │ +│ bam_markduplicates_picard                     │ +│ bam_markduplicates_samtools                   │ +│ bam_ngscheckmate                              │ +│ bam_qc_picard                                 │ +│ bam_rseqc                                     │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index fd9841a27a..94da858340 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Removed files for 'rseqc/bamstat' and its dependencies 'rseqc/bamstat'.                     -INFO     Removed files for 'rseqc/inferexperiment' and its dependencies 'rseqc/inferexperiment'.     -INFO     Removed files for 'rseqc/innerdistance' and its dependencies 'rseqc/innerdistance'.         -INFO     Removed files for 'rseqc/junctionannotation' and its dependencies                           -'rseqc/junctionannotation'.                                                                 -INFO     Removed files for 'rseqc/junctionsaturation' and its dependencies                           -'rseqc/junctionsaturation'.                                                                 -INFO     Removed files for 'rseqc/readdistribution' and its dependencies 'rseqc/readdistribution'.   -INFO     Removed files for 'rseqc/readduplication' and its dependencies 'rseqc/readduplication'.     -INFO     Removed files for 'rseqc/tin' and its dependencies 'rseqc/tin'.                             -INFO     Removed files for 'bam_rseqc' and its dependencies 'bam_rseqc, rseqc_bamstat,  -rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation,  -rseqc_junctionsaturation, rseqc_readdistribution, rseqc_readduplication, rseqc_tin'.        + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Removed files for 'rseqc/bamstat' and its dependencies 'rseqc/bamstat'.                     +INFO     Removed files for 'rseqc/inferexperiment' and its dependencies 'rseqc/inferexperiment'.     +INFO     Removed files for 'rseqc/innerdistance' and its dependencies 'rseqc/innerdistance'.         +INFO     Removed files for 'rseqc/junctionannotation' and its dependencies                           +'rseqc/junctionannotation'.                                                                 +INFO     Removed files for 'rseqc/junctionsaturation' and its dependencies                           +'rseqc/junctionsaturation'.                                                                 +INFO     Removed files for 'rseqc/readdistribution' and its dependencies 'rseqc/readdistribution'.   +INFO     Removed files for 'rseqc/readduplication' and its dependencies 'rseqc/readduplication'.     +INFO     Removed files for 'bam_rseqc' and its dependencies 'bam_rseqc, rseqc_bamstat,  +rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation,  +rseqc_junctionsaturation, rseqc_readdistribution, rseqc_readduplication'.                   diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index 1c719d84c3..f4261d37f5 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-1736045672-matrix { + .terminal-3894408202-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1736045672-title { + .terminal-3894408202-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1736045672-r1 { fill: #c5c8c6 } -.terminal-1736045672-r2 { fill: #98a84b } -.terminal-1736045672-r3 { fill: #9a9b99 } -.terminal-1736045672-r4 { fill: #608ab1 } -.terminal-1736045672-r5 { fill: #d0b344 } + .terminal-3894408202-r1 { fill: #c5c8c6 } +.terminal-3894408202-r2 { fill: #98a84b } +.terminal-3894408202-r3 { fill: #9a9b99 } +.terminal-3894408202-r4 { fill: #608ab1 } +.terminal-3894408202-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO     Generating nf-test snapshot                                                                 + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Generating nf-test snapshot                                                                 diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index db8d96c904..e5e9b2b14f 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - - -INFO    'subworkflows/nf-core/bam_rseqc' is already up to date                                      -INFO     Updates complete ✨                                                                         + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +ERROR   'rseqc/tin' diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index 344a9b04ba..1b2917e5db 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-372548199-matrix { + .terminal-1194172937-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-372548199-title { + .terminal-1194172937-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-372548199-r1 { fill: #c5c8c6 } -.terminal-372548199-r2 { fill: #98a84b } -.terminal-372548199-r3 { fill: #9a9b99 } -.terminal-372548199-r4 { fill: #608ab1 } -.terminal-372548199-r5 { fill: #d0b344 } -.terminal-372548199-r6 { fill: #cc555a } -.terminal-372548199-r7 { fill: #98729f } -.terminal-372548199-r8 { fill: #ff2c7a } + .terminal-1194172937-r1 { fill: #c5c8c6 } +.terminal-1194172937-r2 { fill: #98a84b } +.terminal-1194172937-r3 { fill: #9a9b99 } +.terminal-1194172937-r4 { fill: #608ab1 } +.terminal-1194172937-r5 { fill: #d0b344 } +.terminal-1194172937-r6 { fill: #cc555a } +.terminal-1194172937-r7 { fill: #98729f } +.terminal-1194172937-r8 { fill: #ff2c7a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.12.1 - https://nf-co.re - - -WARNING  Could not find GitHub authentication token. Some API requests may fail.                     -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master' -INFO     Deleting all files in 'TEMPLATE' branch                                                     -INFO     Making a new template pipeline using pipeline variables                                     -INFO     Committed changes to 'TEMPLATE' branch                                                      -INFO     Checking out original branch: 'master' -INFO     Now try to merge the updates in to your pipeline:                                           -           cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -           git merge TEMPLATE                                                                        + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +WARNING  Could not find GitHub authentication token. Some API requests may fail.                     +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master' +INFO     Deleting all files in 'TEMPLATE' branch                                                     +INFO     Making a new template pipeline using pipeline variables                                     +INFO     Committed changes to 'TEMPLATE' branch                                                      +INFO     Checking out original branch: 'master' +INFO     Now try to merge the updates in to your pipeline:                                           +           cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +           git merge TEMPLATE                                                                        From 275a902a0a95e813312722712781c41659f2b02e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 20 Feb 2024 14:59:03 +0100 Subject: [PATCH 176/178] increase timeout for sfw update screenshot --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0adf04d4c6..cf0b01d210 100644 --- a/README.md +++ b/README.md @@ -1142,6 +1142,7 @@ You can update subworkflows installed from a remote repository in your pipeline working_dir: tmp/nf-core-nextbigthing before_command: > echo "repository_type: pipeline" >> .nf-core.yml +timeout: 30 --> ![`nf-core subworkflows update --all --no-preview`](docs/images/nf-core-subworkflows-update.svg) From 6a6562a78a45d810d8299c84429d099595402e79 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 20 Feb 2024 14:07:28 +0000 Subject: [PATCH 177/178] Generate new screengrabs with rich-codex --- docs/images/nf-core-licences.svg | 135 ++++++-------------- docs/images/nf-core-list.svg | 108 ++++++++-------- docs/images/nf-core-modules-create.svg | 130 +++++++------------ docs/images/nf-core-modules-lint.svg | 124 ++++++------------ docs/images/nf-core-subworkflows-remove.svg | 120 +++++++++-------- docs/images/nf-core-subworkflows-update.svg | 72 +++++------ 6 files changed, 270 insertions(+), 419 deletions(-) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index ab30be3467..35962d8e4f 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    -INFO     Warning: This tool only prints licence information for the software tools packaged using    -         conda.                                                                                      -INFO     The pipeline may use other software and dependencies not described here.                    -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index d34af82bb2..aa93f830ee 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-1084195510-matrix { + .terminal-1155039927-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1084195510-title { + .terminal-1155039927-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1084195510-r1 { fill: #c5c8c6 } -.terminal-1084195510-r2 { fill: #98a84b } -.terminal-1084195510-r3 { fill: #9a9b99 } -.terminal-1084195510-r4 { fill: #608ab1 } -.terminal-1084195510-r5 { fill: #d0b344 } -.terminal-1084195510-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1084195510-r7 { fill: #868887 } -.terminal-1084195510-r8 { fill: #868887;font-style: italic; } + .terminal-1155039927-r1 { fill: #c5c8c6 } +.terminal-1155039927-r2 { fill: #98a84b } +.terminal-1155039927-r3 { fill: #9a9b99 } +.terminal-1155039927-r4 { fill: #608ab1 } +.terminal-1155039927-r5 { fill: #d0b344 } +.terminal-1155039927-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1155039927-r7 { fill: #868887 } +.terminal-1155039927-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ fetchngs            │   100 │         1.11.0 │  4 months ago │           - │ -                   │ -│ phageannotator      │     7 │            dev │  20 hours ago │           - │ -                   │ -│ riboseq             │     2 │            dev │  21 hours ago │           - │ -                   │ -│ bacass              │    48 │          2.1.0 │  4 months ago │           - │ -                   │ -│ rnasplice           │    22 │          1.0.2 │  1 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ fetchngs            │   100 │         1.11.0 │  4 months ago │           - │ -                   │ +│ phageannotator      │     7 │            dev │  20 hours ago │           - │ -                   │ +│ riboseq             │     2 │            dev │  22 hours ago │           - │ -                   │ +│ bacass              │    48 │          2.1.0 │  4 months ago │           - │ -                   │ +│ rnasplice           │    22 │          1.0.2 │  1 months ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index aeb42f1255..295289aaca 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' -INFO     Using Docker container: 'biocontainers/fastqc:0.12.1--hdfd78af_0' -INFO     Using Singularity container:                                                                -'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' -INFO     Created component template: 'fastqc' -INFO     Created following files:                                                                    -           modules/nf-core/fastqc/main.nf                                                            -           modules/nf-core/fastqc/meta.yml                                                           -           modules/nf-core/fastqc/environment.yml                                                    -           modules/nf-core/fastqc/tests/tags.yml                                                     -           modules/nf-core/fastqc/tests/main.nf.test                                                 + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 631e065df7..c53b1dfdbc 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' -INFO     Found 4 inputs in modules/nf-core/multiqc/main.nf                                           -INFO     Found 4 outputs in modules/nf-core/multiqc/main.nf                                          - -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  59 Tests Passed │ -│[!]   0 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 94da858340..91bce3c2ef 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13 - https://nf-co.re - - - -INFO     Removed files for 'rseqc/bamstat' and its dependencies 'rseqc/bamstat'.                     -INFO     Removed files for 'rseqc/inferexperiment' and its dependencies 'rseqc/inferexperiment'.     -INFO     Removed files for 'rseqc/innerdistance' and its dependencies 'rseqc/innerdistance'.         -INFO     Removed files for 'rseqc/junctionannotation' and its dependencies                           -'rseqc/junctionannotation'.                                                                 -INFO     Removed files for 'rseqc/junctionsaturation' and its dependencies                           -'rseqc/junctionsaturation'.                                                                 -INFO     Removed files for 'rseqc/readdistribution' and its dependencies 'rseqc/readdistribution'.   -INFO     Removed files for 'rseqc/readduplication' and its dependencies 'rseqc/readduplication'.     -INFO     Removed files for 'bam_rseqc' and its dependencies 'bam_rseqc, rseqc_bamstat,  -rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation,  -rseqc_junctionsaturation, rseqc_readdistribution, rseqc_readduplication'.                   + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +INFO     Removed files for 'rseqc/bamstat' and its dependencies 'rseqc/bamstat'.                     +INFO     Removed files for 'rseqc/inferexperiment' and its dependencies 'rseqc/inferexperiment'.     +INFO     Removed files for 'rseqc/innerdistance' and its dependencies 'rseqc/innerdistance'.         +INFO     Removed files for 'rseqc/junctionannotation' and its dependencies                           +'rseqc/junctionannotation'.                                                                 +INFO     Removed files for 'rseqc/junctionsaturation' and its dependencies                           +'rseqc/junctionsaturation'.                                                                 +INFO     Removed files for 'rseqc/readdistribution' and its dependencies 'rseqc/readdistribution'.   +INFO     Removed files for 'bam_rseqc' and its dependencies 'bam_rseqc, rseqc_bamstat,  +rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation,  +rseqc_junctionsaturation, rseqc_readdistribution'.                                          diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index e5e9b2b14f..64919a7e3d 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-1502220196-matrix { + .terminal-4230681361-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1502220196-title { + .terminal-4230681361-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1502220196-r1 { fill: #c5c8c6 } -.terminal-1502220196-r2 { fill: #98a84b } -.terminal-1502220196-r3 { fill: #9a9b99 } -.terminal-1502220196-r4 { fill: #608ab1 } -.terminal-1502220196-r5 { fill: #d0b344 } -.terminal-1502220196-r6 { fill: #cc555a;font-weight: bold } + .terminal-4230681361-r1 { fill: #c5c8c6 } +.terminal-4230681361-r2 { fill: #98a84b } +.terminal-4230681361-r3 { fill: #9a9b99 } +.terminal-4230681361-r4 { fill: #608ab1 } +.terminal-4230681361-r5 { fill: #d0b344 } +.terminal-4230681361-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13 - https://nf-co.re - - - -ERROR   'rseqc/tin' + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.13 - https://nf-co.re + + + +ERROR   'rseqc/readduplication' From f4871ef3b11a3ffc2503a0c1caad1b829bb395f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 20 Feb 2024 15:23:09 +0100 Subject: [PATCH 178/178] update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ac4437496..a9d83051f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,14 +10,14 @@ - Fix GitHub Actions CI and Linting badges links ([#2757](https://github.com/nf-core/tools/pull/2757)) - Add hashtags to release announcement on mastodon ([#2761](https://github.com/nf-core/tools/pull/2761)) - update fastqc and multiqc in template ([#2776](https://github.com/nf-core/tools/pull/2776)) +- template refactoring: remove the `lib` directory and use nf-core subworkflows ([#2736](https://github.com/nf-core/tools/pull/2736)) +- use nf-validation to create an input channel from a sample sheet ([#2736](https://github.com/nf-core/tools/pull/2736)) ### Linting - Make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733)) - Add looser comparison when pipeline logos ([#2744](https://github.com/nf-core/tools/pull/2744)) - Handle multiple aliases in module imports correctly during linting ([#2762](https://github.com/nf-core/tools/pull/2762)) -- make creat-lint-wf composable ([#2733](https://github.com/nf-core/tools/pull/2733)) -- add looser comparison when pipeline logos ([#2744](https://github.com/nf-core/tools/pull/2744)) - Switch to markdown based API and error docs ([#2758](https://github.com/nf-core/tools/pull/2758)) ### Modules
    Process Name \\", - " \\ Software Version
    CUSTOM_DUMPSOFTWAREVERSIONSpython3.11.7
    yaml5.4.1
    TOOL1tool10.11.9
    TOOL2tool21.9
    WorkflowNextflow
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls
    File typeConventional base calls