From 22ceda85feb8aa07bbf264b14010c0f6c990c018 Mon Sep 17 00:00:00 2001 From: mdouglas Date: Tue, 26 Sep 2023 09:53:22 -0700 Subject: [PATCH 001/270] fix ValueError when loading config with no manifest --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 8b73a10921..51666c326d 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -177,7 +177,7 @@ def _load_pipeline_config(self): """ self.nf_config = fetch_wf_config(self.wf_path) - self.pipeline_prefix, self.pipeline_name = self.nf_config.get("manifest.name", "").strip("'").split("/") + self.pipeline_prefix, self.pipeline_name = self.nf_config.get("manifest.name", "/").strip("'").split("/") nextflowVersionMatch = re.search(r"[0-9\.]+(-edge)?", self.nf_config.get("manifest.nextflowVersion", "")) if nextflowVersionMatch: From 98b7da08d9c49b1090719c543dcb90e25fcca108 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 28 Feb 2024 14:22:32 +0100 Subject: [PATCH 002/270] upgrade to python 3.12 --- .github/ISSUE_TEMPLATE/bug_report.yml | 4 ++-- .github/workflows/changelog.yml | 6 +++--- .github/workflows/create-lint-wf.yml | 4 ++-- .github/workflows/create-test-lint-wf-template.yml | 4 ++-- .github/workflows/create-test-wf.yml | 4 ++-- .github/workflows/deploy-pypi.yml | 4 ++-- .github/workflows/fix-linting.yml | 2 +- .github/workflows/lint-code.yml | 4 ++-- .github/workflows/pytest.yml | 6 +++--- .github/workflows/sync.yml | 4 ++-- Dockerfile | 2 +- README.md | 2 +- .../.github/workflows/download_pipeline.yml | 2 +- nf_core/pipeline-template/.github/workflows/fix-linting.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 6 +++--- 15 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8fdd2bd7e1..5043b37acc 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -37,5 +37,5 @@ body: * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ - * Version of nf-core/tools _(eg. 1.1, 1.5, 1.8.2)_ - * Python version _(eg. 3.10, 3.11)_ + * Version of nf-core/tools _(eg. 1.10, 1.12.1, 1.13)_ + * Python version _(eg. 3.11, 3.12)_ diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 6316d62f33..88d0be2eb1 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -36,7 +36,7 @@ jobs: - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install packages run: | @@ -62,10 +62,10 @@ jobs: git diff --exit-code ${GITHUB_WORKSPACE}/CHANGELOG.md || echo "changed=YES" >> $GITHUB_ENV echo "File changed: ${{ env.changed }}" - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index e69229c051..0162dc6ec0 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -49,10 +49,10 @@ jobs: name: Check out source-code repository # Set up nf-core/tools - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" cache: pip - name: Install python dependencies diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 6f27236462..3f1bc362f3 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -61,10 +61,10 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 0166931be7..e82faae592 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -48,10 +48,10 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index e53d2f2f5a..ca698dad1f 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 name: Check out source-code repository - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 95a03c70fe..3e6c4d4ef6 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -34,7 +34,7 @@ jobs: # Install and run pre-commit - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install pre-commit run: pip install pre-commit diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 8ed52a0582..53151a8989 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -20,10 +20,10 @@ jobs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5faeef49da..c749458dbe 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -38,7 +38,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.8", "3.12"] runner: ["ubuntu-latest"] include: - python-version: "3.8" @@ -173,12 +173,12 @@ jobs: cd pytest - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 env: AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/ with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install dependencies diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index d89e255bfb..14eb589360 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -56,10 +56,10 @@ jobs: path: nf-core/${{ matrix.pipeline }} fetch-depth: "0" - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/Dockerfile b/Dockerfile index 62431be140..6ca7e9a67f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim@sha256:ce81dc539f0aedc9114cae640f8352fad83d37461c24a3615b01f081d0c0583a +FROM python:3.12-slim LABEL authors="phil.ewels@scilifelab.se,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for the nfcore tools" diff --git a/README.md b/README.md index cf0b01d210..833fad626f 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ conda install nf-core Alternatively, you can create a new environment with both nf-core/tools and nextflow: ```bash -conda create --name nf-core python=3.11 nf-core nextflow +conda create --name nf-core python=3.12 nf-core nextflow conda activate nf-core ``` diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 20b811ab2b..4017fbdc2d 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 with: diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 28e6605b96..e976b9f09c 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -34,7 +34,7 @@ jobs: # Install and run pre-commit - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install pre-commit run: pip install pre-commit diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 3ec259b5ed..ea740d15a3 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -16,10 +16,10 @@ jobs: steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit @@ -39,7 +39,7 @@ jobs: - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - name: Install dependencies From ecc697ea8c98fddf3e0b0ab0d424916f88848a98 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 28 Feb 2024 13:27:11 +0000 Subject: [PATCH 003/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fe084d81a..7c6be5ca44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Remove obsolete editor settings in `devcontainer.json` and `gitpod.yml` ([#2795](https://github.com/nf-core/tools/pull/2795)) +- Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) ### Linting From dc558607c8334861126b7879e90877b8efe77bac Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 29 Feb 2024 15:32:05 +0000 Subject: [PATCH 004/270] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03fbb7bedf..508bf1ac24 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.2 + rev: v0.3.0 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 09041f656ae3fe5384b8d73bca24ad6f194d14db Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 29 Feb 2024 18:38:44 +0100 Subject: [PATCH 005/270] bump to dev version [skip changelog] --- .gitpod.yml | 2 +- CHANGELOG.md | 10 ++++++++++ setup.py | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index 445cb35706..b2fbb73133 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3189edaa..6d59140f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # nf-core/tools: Changelog +## v2.13.2dev + +### Template + +### Linting + +### Components + +### General + ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] ### Template diff --git a/setup.py b/setup.py index c7f3fa41c6..fc7b69ac1e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.13.1" +version = "2.13.2dev" with open("README.md") as f: readme = f.read() From e5de53f7dd45abe297ec2ca2e18418cb219a0684 Mon Sep 17 00:00:00 2001 From: "Thomas A. Christensen II" <25492070+MillironX@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:27:31 -0600 Subject: [PATCH 006/270] chore: Update templates to use nf-core/setup-nextflow v2 --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- .../pipeline-template/.github/workflows/download_pipeline.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 84c727f60d..3880b2c4dc 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index dcd7caabfc..4fdec4e243 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 59b85f95fc..612467ff6e 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -35,7 +35,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: From c69268b58ae7d832d2b5a8584c2ef29434d03aac Mon Sep 17 00:00:00 2001 From: "Thomas A. Christensen II" <25492070+MillironX@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:29:52 -0600 Subject: [PATCH 007/270] chore: Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d59140f95..42cef06a36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Update templates to use nf-core/setup-nextflow v2 + ### Linting ### Components From 49c6bed924d5543276f2392ab616394a1517732f Mon Sep 17 00:00:00 2001 From: "Thomas A. Christensen II" <25492070+MillironX@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:44:50 -0600 Subject: [PATCH 008/270] chore: Update ci to use nf-core/setup-nextflow v2 --- .github/actions/create-lint-wf/action.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/pytest.yml | 2 +- .github/workflows/rich-codex.yml | 2 +- .github/workflows/sync.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 51bf2ae5f7..1741b934f2 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -17,7 +17,7 @@ runs: # Set up Nextflow - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: ${{ matrix.NXF_VER }} diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 188cccfd75..5f5d545199 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -72,7 +72,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: latest-everything diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index cd36d4e62d..d68bfae71c 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -59,7 +59,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: ${{ matrix.NXF_VER }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index eec676a746..7481d0a155 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -119,7 +119,7 @@ jobs: run: echo "date=$(date +'%Y-%m')" >> $GITHUB_ENV - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - name: Look if nf-test is already installed and write to env variable id: check-nftest diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 8748f3d7b3..3cef06c7c3 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -15,7 +15,7 @@ jobs: cache-dependency-path: setup.py - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 - name: Cache nf-test installation id: cache-software diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index c998552ddd..19e895f452 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -67,7 +67,7 @@ jobs: pip install . - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "latest-everything" From ae4725156c7694f248cdf51a0cad41cf7c5e312c Mon Sep 17 00:00:00 2001 From: "Thomas A. Christensen II" <25492070+MillironX@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:45:43 -0600 Subject: [PATCH 009/270] chore: Update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d59140f95..9d2be810fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ ### General +- Update ci to use nf-core/setup-nextflow v2 + ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] ### Template From 8c1ec05947bc5a78a7a3bd5417865bc3faf2e6b2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 1 Mar 2024 08:29:11 +0100 Subject: [PATCH 010/270] handle also patch version before `dev` suffix --- .github/workflows/changelog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/changelog.py b/.github/workflows/changelog.py index 4835b0d0f0..7106cc7501 100644 --- a/.github/workflows/changelog.py +++ b/.github/workflows/changelog.py @@ -144,10 +144,11 @@ def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> st # Parse version from the line `## v2.12dev` or # `## [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20]` ... - if not (m := re.match(r".*(v\d+\.\d+(dev)?).*", line)): + if not (m := re.match(r".*(v\d+\.\d+.\d*(dev)?).*", line)): print(f"Cannot parse version from line {line.strip()}.", file=sys.stderr) sys.exit(1) version = m.group(1) + print(f"Found version: {version}") if not inside_version_dev: if not version.endswith("dev"): From 6e6e916992b93dc336afb3d6e31bc7b9111770db Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 1 Mar 2024 09:01:30 +0100 Subject: [PATCH 011/270] add ability to add new sections with word before colon in PR title --- .github/workflows/changelog.py | 76 +++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/.github/workflows/changelog.py b/.github/workflows/changelog.py index 7106cc7501..412912812b 100644 --- a/.github/workflows/changelog.py +++ b/.github/workflows/changelog.py @@ -71,6 +71,11 @@ def _determine_change_type(pr_title) -> tuple[str, str]: if re.sub(r"s$", "", section.lower().replace("ing", "")) in pr_title.lower(): current_section_header = section_header current_section = section + # Add new section type if PR title has a colon and the section is not in the list + if ": " in pr_title and current_section == "General": + section = pr_title.split(":")[0] + current_section_header = f"### {section}" + current_section = section print(f"Detected section: {current_section}") return current_section, current_section_header @@ -79,7 +84,7 @@ def _determine_change_type(pr_title) -> tuple[str, str]: section, section_header = _determine_change_type(pr_title) # Remove section indicator from the PR title. -pr_title = re.sub(rf"{section}[:\s]*", "", pr_title, flags=re.IGNORECASE) +pr_title = re.sub(rf"{section}:[\s]*", "", pr_title, flags=re.IGNORECASE) # Prepare the change log entry. pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))" @@ -91,8 +96,11 @@ def _determine_change_type(pr_title) -> tuple[str, str]: new_lines = [ f"- {pr_title} {pr_link}\n", ] - print(f"Adding new lines into section '{section}':\n" + "".join(new_lines)) +# Add new section if it doesn't exist +if section_header not in open(changelog_path).read(): + print(f"Adding new section '{section}'") + new_lines = [f"{section_header}\n"] + new_lines # Finally, updating the changelog. # Read the current changelog lines. We will print them back as is, except for one new @@ -179,37 +187,39 @@ def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> st print(f"Found line: {line.strip()}") print(f"inside_version_dev: {inside_version_dev}") print(f"section_header: {section_header}") - if inside_version_dev and line.lower().startswith(section_header.lower()): # Section of interest header - print(f"Found section header: {line.strip()}") - if already_added_entry: - print( - f"Already added new lines into section {section}, is the section duplicated?", - file=sys.stderr, - ) - sys.exit(1) - updated_lines.append(line) - # Collecting lines until the next section. - section_lines: List[str] = [] - while True: - line = orig_lines.pop(0) - if line.startswith("#"): - print(f"Found the next section header: {line.strip()}") - # Found the next section header, so need to put all the lines we collected. - updated_lines.append("\n") - _updated_lines = [_l for _l in section_lines + new_lines if _l.strip()] - updated_lines.extend(_updated_lines) - updated_lines.append("\n") - if new_lines: - print(f"Updated {changelog_path} section '{section}' with lines:\n" + "".join(new_lines)) - else: - print(f"Removed existing entry from {changelog_path} section '{section}'") - already_added_entry = True - # Pushing back the next section header line - orig_lines.insert(0, line) - break - # If the line already contains a link to the PR, don't add it again. - line = _skip_existing_entry_for_this_pr(line, same_section=True) - section_lines.append(line) + if inside_version_dev: + if line.lower().startswith(section_header.lower()): # Section of interest header + print(f"Found section header: {line.strip()}") + if already_added_entry: + print( + f"Already added new lines into section {section}, is the section duplicated?", + file=sys.stderr, + ) + sys.exit(1) + updated_lines.append(line) + # Collecting lines until the next section. + section_lines: List[str] = [] + while True: + line = orig_lines.pop(0) + if line.startswith("#"): + print(f"Found the next section header: {line.strip()}") + # Found the next section header, so need to put all the lines we collected. + updated_lines.append("\n") + _updated_lines = [_l for _l in section_lines + new_lines if _l.strip()] + updated_lines.extend(_updated_lines) + updated_lines.append("\n") + if new_lines: + print(f"Updated {changelog_path} section '{section}' with lines:\n" + "".join(new_lines)) + else: + print(f"Removed existing entry from {changelog_path} section '{section}'") + already_added_entry = True + # Pushing back the next section header line + orig_lines.insert(0, line) + break + # If the line already contains a link to the PR, don't add it again. + line = _skip_existing_entry_for_this_pr(line, same_section=True) + section_lines.append(line) + else: updated_lines.append(line) From 3618fd8774ee2e0e69a05879f157c2fae414bb91 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 1 Mar 2024 10:19:40 +0100 Subject: [PATCH 012/270] add `force_pr` flag to sync, to force a PR even though there are no changes committed fixes problems when template branch is already updated but we failed opening a PR --- .github/workflows/sync.yml | 4 ++++ nf_core/__main__.py | 10 ++++++++-- nf_core/sync.py | 7 +++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index c998552ddd..026bc555cf 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -15,6 +15,10 @@ on: - "ubuntu-latest" - "self-hosted" default: "self-hosted" + force_pr: + description: "Force a PR to be created" + type: boolean + default: false # Cancel if a newer run is started concurrency: diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 7d2d083fa9..570f5c4e04 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -2119,10 +2119,16 @@ def logo(logo_text, dir, name, theme, width, format, force): default=False, help="Make a GitHub pull-request with the changes.", ) +@click.option( + "--force_pr", + is_flag=True, + default=False, + help="Force the creation of a pull-request, even if there are no changes.", +) @click.option("-g", "--github-repository", type=str, help="GitHub PR: target repository.") @click.option("-u", "--username", type=str, help="GitHub PR: auth username.") @click.option("-t", "--template-yaml", help="Pass a YAML file to customize the template") -def sync(dir, from_branch, pull_request, github_repository, username, template_yaml): +def sync(dir, from_branch, pull_request, github_repository, username, template_yaml, force_pr): """ Sync a pipeline [cyan i]TEMPLATE[/] branch with the nf-core template. @@ -2142,7 +2148,7 @@ def sync(dir, from_branch, pull_request, github_repository, username, template_y is_pipeline_directory(dir) # Sync the given pipeline dir - sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml) + sync_obj = PipelineSync(dir, from_branch, pull_request, github_repository, username, template_yaml, force_pr) try: sync_obj.sync() except (SyncExceptionError, PullRequestExceptionError) as e: diff --git a/nf_core/sync.py b/nf_core/sync.py index 995baeacd2..5eb14f8348 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -45,6 +45,7 @@ class PipelineSync: gh_username (str): GitHub username gh_repo (str): GitHub repository name template_yaml_path (str): Path to template.yml file for pipeline creation settings. DEPRECATED + force_pr (bool): Force the creation of a pull request, even if there are no changes to the template Attributes: pipeline_dir (str): Path to target pipeline directory @@ -65,6 +66,7 @@ def __init__( gh_repo=None, gh_username=None, template_yaml_path=None, + force_pr=False, ): """Initialise syncing object""" @@ -77,6 +79,7 @@ def __init__( self.make_pr = make_pr self.gh_pr_returned_data = {} self.required_config_vars = ["manifest.name", "manifest.description", "manifest.version", "manifest.author"] + self.force_pr = force_pr self.gh_username = gh_username self.gh_repo = gh_repo @@ -133,6 +136,10 @@ def sync(self): self.make_template_pipeline() self.commit_template_changes() + if not self.made_changes and self.force_pr: + log.info("No changes made to TEMPLATE, but PR forced") + self.made_changes = True + # Push and make a pull request if we've been asked to if self.made_changes and self.make_pr: try: From 281cb5545934099f7f534c3aa8b30ac699668c2d Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 1 Mar 2024 11:40:58 +0100 Subject: [PATCH 013/270] Update PULL_REQUEST_TEMPLATE.md fix command --- nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index d04335275a..44a00ea920 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -20,7 +20,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. {%- endif %} - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nf-test test main.nf.test -profile test,docker`). +- [ ] Ensure the test suite passes (`nf-test test tests/ --verbose --debug --profile +docker`). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. From 6d16be384f73b9f753cb904c1dfc7c4081a7a342 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 1 Mar 2024 11:52:09 +0100 Subject: [PATCH 014/270] Update PULL_REQUEST_TEMPLATE.md --- nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index 44a00ea920..ca0687733e 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -20,7 +20,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. {%- endif %} - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nf-test test tests/ --verbose --debug --profile +docker`). +- [ ] Ensure the test suite passes (`nf-test test tests/ --verbose --profile +docker`). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. From 61880487a1c649e167b98b16d96013c06a9c3f44 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 1 Mar 2024 13:23:28 +0100 Subject: [PATCH 015/270] remove abilty to add arbitrary sections --- .github/workflows/changelog.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/changelog.py b/.github/workflows/changelog.py index 412912812b..ce5306e9ef 100644 --- a/.github/workflows/changelog.py +++ b/.github/workflows/changelog.py @@ -71,11 +71,6 @@ def _determine_change_type(pr_title) -> tuple[str, str]: if re.sub(r"s$", "", section.lower().replace("ing", "")) in pr_title.lower(): current_section_header = section_header current_section = section - # Add new section type if PR title has a colon and the section is not in the list - if ": " in pr_title and current_section == "General": - section = pr_title.split(":")[0] - current_section_header = f"### {section}" - current_section = section print(f"Detected section: {current_section}") return current_section, current_section_header @@ -97,10 +92,6 @@ def _determine_change_type(pr_title) -> tuple[str, str]: f"- {pr_title} {pr_link}\n", ] print(f"Adding new lines into section '{section}':\n" + "".join(new_lines)) -# Add new section if it doesn't exist -if section_header not in open(changelog_path).read(): - print(f"Adding new section '{section}'") - new_lines = [f"{section_header}\n"] + new_lines # Finally, updating the changelog. # Read the current changelog lines. We will print them back as is, except for one new From 582a68d86910a574f9330f3716a4544ccdcbd509 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 1 Mar 2024 13:31:50 +0100 Subject: [PATCH 016/270] fix if clause --- .github/workflows/changelog.py | 63 +++++++++++++++++----------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/.github/workflows/changelog.py b/.github/workflows/changelog.py index ce5306e9ef..471665e4b1 100644 --- a/.github/workflows/changelog.py +++ b/.github/workflows/changelog.py @@ -178,38 +178,37 @@ def _skip_existing_entry_for_this_pr(line: str, same_section: bool = True) -> st print(f"Found line: {line.strip()}") print(f"inside_version_dev: {inside_version_dev}") print(f"section_header: {section_header}") - if inside_version_dev: - if line.lower().startswith(section_header.lower()): # Section of interest header - print(f"Found section header: {line.strip()}") - if already_added_entry: - print( - f"Already added new lines into section {section}, is the section duplicated?", - file=sys.stderr, - ) - sys.exit(1) - updated_lines.append(line) - # Collecting lines until the next section. - section_lines: List[str] = [] - while True: - line = orig_lines.pop(0) - if line.startswith("#"): - print(f"Found the next section header: {line.strip()}") - # Found the next section header, so need to put all the lines we collected. - updated_lines.append("\n") - _updated_lines = [_l for _l in section_lines + new_lines if _l.strip()] - updated_lines.extend(_updated_lines) - updated_lines.append("\n") - if new_lines: - print(f"Updated {changelog_path} section '{section}' with lines:\n" + "".join(new_lines)) - else: - print(f"Removed existing entry from {changelog_path} section '{section}'") - already_added_entry = True - # Pushing back the next section header line - orig_lines.insert(0, line) - break - # If the line already contains a link to the PR, don't add it again. - line = _skip_existing_entry_for_this_pr(line, same_section=True) - section_lines.append(line) + if inside_version_dev and line.lower().startswith(section_header.lower()): # Section of interest header + print(f"Found section header: {line.strip()}") + if already_added_entry: + print( + f"Already added new lines into section {section}, is the section duplicated?", + file=sys.stderr, + ) + sys.exit(1) + updated_lines.append(line) + # Collecting lines until the next section. + section_lines: List[str] = [] + while True: + line = orig_lines.pop(0) + if line.startswith("#"): + print(f"Found the next section header: {line.strip()}") + # Found the next section header, so need to put all the lines we collected. + updated_lines.append("\n") + _updated_lines = [_l for _l in section_lines + new_lines if _l.strip()] + updated_lines.extend(_updated_lines) + updated_lines.append("\n") + if new_lines: + print(f"Updated {changelog_path} section '{section}' with lines:\n" + "".join(new_lines)) + else: + print(f"Removed existing entry from {changelog_path} section '{section}'") + already_added_entry = True + # Pushing back the next section header line + orig_lines.insert(0, line) + break + # If the line already contains a link to the PR, don't add it again. + line = _skip_existing_entry_for_this_pr(line, same_section=True) + section_lines.append(line) else: updated_lines.append(line) From f65253cc3895036fd9f1147462eb38c5d27b159f Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 1 Mar 2024 12:33:32 +0000 Subject: [PATCH 017/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d59140f95..eefa851fd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ ### General +- Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) + ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] ### Template From 78438ad2751eea307eb60d53d3b07b7f50b9ef89 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 1 Mar 2024 13:00:44 +0000 Subject: [PATCH 018/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eefa851fd1..a3ff7cd4e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ ### General - Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) +- Add `force_pr` flag to sync, to force a PR even though there are no changes committed ([#2822](https://github.com/nf-core/tools/pull/2822)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 7d480d2d0efe89992824e78a2d6c3cae5aaa7d59 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 1 Mar 2024 14:39:47 +0100 Subject: [PATCH 019/270] fix path in component update script --- .github/workflows/update_components_template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index f357bed03b..0fa3adaf20 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -23,11 +23,11 @@ jobs: - name: Update modules run: nf-core modules update --all - working-directory: nf-core/pipeline-template + working-directory: nf_core/pipeline-template - name: Update subworkflows run: nf-core subworkflows update --all - working-directory: nf-core/pipeline-template + working-directory: nf_core/pipeline-template # Commit the changes - name: Commit changes From ea0452e1619e53281df50ec3eba21e761063af59 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 1 Mar 2024 13:53:16 +0000 Subject: [PATCH 020/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a16db59e56..10b603b035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Update CI to use nf-core/setup-nextflow v2 - Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) +- Fix path in component update script ([#2823](https://github.com/nf-core/tools/pull/2823)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 56a56c880bcc6f93ab72b3daaacfe10ae0e38e22 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 1 Mar 2024 14:59:07 +0100 Subject: [PATCH 021/270] don't rerun pipeline creation tests on changelog changes --- .github/workflows/create-lint-wf.yml | 3 +++ .github/workflows/create-test-lint-wf-template.yml | 3 +++ .github/workflows/create-test-wf.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index e69229c051..7a72e18d22 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -7,6 +7,9 @@ on: - "docs/**" - "CHANGELOG.md" pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] workflow_dispatch: diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 5f5d545199..0d65c40b07 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -6,6 +6,9 @@ on: paths: - nf_core/pipeline-template/** pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] workflow_dispatch: diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index d68bfae71c..9800ac1910 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -7,6 +7,9 @@ on: - "docs/**" - "CHANGELOG.md" pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" release: types: [published] workflow_dispatch: From 2ace697120b5b1e1c002dedf03a338a8da5f69f5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 1 Mar 2024 15:30:06 +0100 Subject: [PATCH 022/270] add missing api build requirement --- docs/api/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api/requirements.txt b/docs/api/requirements.txt index fc5184b95f..abffe30740 100644 --- a/docs/api/requirements.txt +++ b/docs/api/requirements.txt @@ -1,4 +1,5 @@ Sphinx>=3.3.1 sphinxcontrib-napoleon +sphinx-markdown-builder sphinx_rtd_theme>=0.5.0 myst-parser From 0cd08ae704f156beec1222f88998a05fa9421169 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 1 Mar 2024 20:17:12 +0000 Subject: [PATCH 023/270] Update GitHub Actions --- .github/workflows/pytest.yml | 4 ++-- .github/workflows/rich-codex.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7481d0a155..461b888516 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -133,7 +133,7 @@ jobs: - name: Cache nf-test installation if: env.nftest_installed != 'true' id: cache-software - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4 + uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 with: path: | /usr/local/bin/nf-test @@ -197,7 +197,7 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85 # v4 + uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 3cef06c7c3..9d1bf7b76c 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -19,7 +19,7 @@ jobs: - name: Cache nf-test installation id: cache-software - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4 + uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 with: path: | /usr/local/bin/nf-test From 0ae1dec9e6c6ce25edce76e39509d0c03a72cad5 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 2 Mar 2024 21:57:51 +0000 Subject: [PATCH 024/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a16db59e56..14ff84e937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Update CI to use nf-core/setup-nextflow v2 - Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) +- Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From fee94786fadbb250bb00f3f21cc1fa1e4aea2387 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 5 Mar 2024 12:55:29 +0100 Subject: [PATCH 025/270] remove fasta default from nextflow.config --- nf_core/pipeline-template/nextflow.config | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 17e75f18a4..2f9b349916 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -18,7 +18,6 @@ params { genome = null igenomes_base = 's3://ngi-igenomes/igenomes/' igenomes_ignore = false - fasta = null {%- endif -%} // MultiQC options From 5369e729247a8d7f927ec135ae8a21dd71cd09fd Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 5 Mar 2024 11:57:47 +0000 Subject: [PATCH 026/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14ff84e937..5285beacf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Remove fasta default from nextflow.config ([#2828](https://github.com/nf-core/tools/pull/2828)) + ### Linting ### Components From d8e5a38d8ec6ae2abc97ed370192e9deeba953b6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 5 Mar 2024 14:20:50 +0100 Subject: [PATCH 027/270] changes added by ruff v0.3.0 --- nf_core/__init__.py | 2 +- nf_core/__main__.py | 3 +- nf_core/components/components_test.py | 7 ++- nf_core/components/create.py | 1 - nf_core/components/lint/__init__.py | 1 - nf_core/components/nfcore_component.py | 1 + nf_core/create.py | 1 + nf_core/launch.py | 3 +- nf_core/licences.py | 1 - nf_core/list.py | 1 - nf_core/modules/bump_versions.py | 1 - nf_core/modules/lint/__init__.py | 1 - nf_core/modules/lint/module_changes.py | 1 + nf_core/modules/lint/module_tests.py | 1 + nf_core/params_file.py | 3 +- nf_core/schema.py | 45 +++++++++---------- nf_core/subworkflows/lint/__init__.py | 1 - .../subworkflows/lint/subworkflow_changes.py | 1 + .../subworkflows/lint/subworkflow_tests.py | 1 + nf_core/sync.py | 3 +- nf_core/utils.py | 1 + tests/components/generate_snapshot.py | 1 + tests/components/snapshot_test.py | 1 + tests/test_bump_version.py | 4 +- tests/test_cli.py | 2 +- tests/test_components.py | 3 +- tests/test_create.py | 4 +- tests/test_download.py | 3 +- tests/test_launch.py | 3 +- tests/test_licenses.py | 3 +- tests/test_lint.py | 4 +- tests/test_list.py | 3 +- tests/test_modules.py | 3 +- tests/test_refgenie.py | 3 +- tests/test_schema.py | 3 +- tests/test_subworkflows.py | 3 +- tests/test_sync.py | 3 +- tests/test_utils.py | 3 +- 38 files changed, 58 insertions(+), 71 deletions(-) diff --git a/nf_core/__init__.py b/nf_core/__init__.py index d96be73f3d..2d4fe45a0a 100644 --- a/nf_core/__init__.py +++ b/nf_core/__init__.py @@ -1,4 +1,4 @@ -""" Main nf_core module file. +"""Main nf_core module file. Shouldn't do much, as everything is under subcommands. """ diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 7d2d083fa9..d6f6077be9 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -""" nf-core: Helper tools for use with nf-core Nextflow pipelines. """ +"""nf-core: Helper tools for use with nf-core Nextflow pipelines.""" + import logging import os import sys diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index f1a9e7c401..9b81f54f06 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -2,7 +2,6 @@ The ComponentsTest class handles the generation and testing of nf-test snapshots. """ - import logging import os import re @@ -91,9 +90,9 @@ def run(self) -> None: """Run build steps""" self.check_inputs() os.environ["NFT_DIFF"] = "pdiff" # set nf-test differ to pdiff to get a better diff output - os.environ[ - "NFT_DIFF_ARGS" - ] = "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences + os.environ["NFT_DIFF_ARGS"] = ( + "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences + ) with nf_core.utils.set_wd(Path(self.dir)): self.check_snapshot_stability() if len(self.errors) > 0: diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c4b477a0ab..d2169e3a72 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -2,7 +2,6 @@ The ComponentCreate class handles generating of module and subworkflow templates """ - import glob import json import logging diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 3c2fb9dde3..c99934bca3 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -3,7 +3,6 @@ in nf-core pipelines """ - import logging import operator import os diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 2f73afe9d3..d9731ba7c9 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -1,6 +1,7 @@ """ The NFCoreComponent class holds information and utility functions for a single module or subworkflow """ + import logging import re from pathlib import Path diff --git a/nf_core/create.py b/nf_core/create.py index c094d33a22..b420b1c86d 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -1,6 +1,7 @@ """Creates a nf-core pipeline matching the current organization's specification based on a template. """ + import configparser import logging import os diff --git a/nf_core/launch.py b/nf_core/launch.py index 25bb4c150c..bc0cd58aec 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -1,5 +1,4 @@ -""" Launch a pipeline, interactively collecting params """ - +"""Launch a pipeline, interactively collecting params""" import copy import json diff --git a/nf_core/licences.py b/nf_core/licences.py index a8a35334dd..be737280f8 100644 --- a/nf_core/licences.py +++ b/nf_core/licences.py @@ -1,6 +1,5 @@ """Lists software licences for a given workflow.""" - import json import logging import os diff --git a/nf_core/list.py b/nf_core/list.py index 67d1a76878..658f4dc6d2 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -1,6 +1,5 @@ """Lists available nf-core pipelines and versions.""" - import json import logging import os diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index b9003be974..9b54174d5a 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -3,7 +3,6 @@ or for a single module """ - import logging import os import re diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 866e6312aa..b2816dab6a 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -6,7 +6,6 @@ nf-core modules lint """ - import logging import os diff --git a/nf_core/modules/lint/module_changes.py b/nf_core/modules/lint/module_changes.py index ee8cabebe1..eb76f4b88b 100644 --- a/nf_core/modules/lint/module_changes.py +++ b/nf_core/modules/lint/module_changes.py @@ -1,6 +1,7 @@ """ Check whether the content of a module has changed compared to the original repository """ + import shutil import tempfile from pathlib import Path diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 520f8cf0a2..b1a611d70a 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -1,6 +1,7 @@ """ Lint the tests of a module in nf-core/modules """ + import json import logging from pathlib import Path diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 267fe7086a..78798b065e 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -1,5 +1,4 @@ -""" Create a YAML parameter file """ - +"""Create a YAML parameter file""" import json import logging diff --git a/nf_core/schema.py b/nf_core/schema.py index df04dc5a1e..b5816063d7 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -1,5 +1,4 @@ -""" Code to deal with pipeline JSON Schema """ - +"""Code to deal with pipeline JSON Schema""" import copy import json @@ -282,9 +281,9 @@ def validate_default_params(self): if param in self.pipeline_params: self.validate_config_default_parameter(param, group_properties[param], self.pipeline_params[param]) else: - self.invalid_nextflow_config_default_parameters[ - param - ] = "Not in pipeline parameters. Check `nextflow.config`." + self.invalid_nextflow_config_default_parameters[param] = ( + "Not in pipeline parameters. Check `nextflow.config`." + ) # Go over ungrouped params if any exist ungrouped_properties = self.schema.get("properties") @@ -297,9 +296,9 @@ def validate_default_params(self): param, ungrouped_properties[param], self.pipeline_params[param] ) else: - self.invalid_nextflow_config_default_parameters[ - param - ] = "Not in pipeline parameters. Check `nextflow.config`." + self.invalid_nextflow_config_default_parameters[param] = ( + "Not in pipeline parameters. Check `nextflow.config`." + ) def validate_config_default_parameter(self, param, schema_param, config_default): """ @@ -314,9 +313,9 @@ def validate_config_default_parameter(self, param, schema_param, config_default) ): # Check that we are not deferring the execution of this parameter in the schema default with squiggly brakcets if schema_param["type"] != "string" or "{" not in schema_param["default"]: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Schema default (`{schema_param['default']}`) does not match the config default (`{config_default}`)" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Schema default (`{schema_param['default']}`) does not match the config default (`{config_default}`)" + ) return # if default is null, we're good @@ -326,28 +325,28 @@ def validate_config_default_parameter(self, param, schema_param, config_default) # Check variable types in nextflow.config if schema_param["type"] == "string": if str(config_default) in ["false", "true", "''"]: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"String should not be set to `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"String should not be set to `{config_default}`" + ) if schema_param["type"] == "boolean": if str(config_default) not in ["false", "true"]: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Booleans should only be true or false, not `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Booleans should only be true or false, not `{config_default}`" + ) if schema_param["type"] == "integer": try: int(config_default) except ValueError: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Does not look like an integer: `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Does not look like an integer: `{config_default}`" + ) if schema_param["type"] == "number": try: float(config_default) except ValueError: - self.invalid_nextflow_config_default_parameters[ - param - ] = f"Does not look like a number (float): `{config_default}`" + self.invalid_nextflow_config_default_parameters[param] = ( + f"Does not look like a number (float): `{config_default}`" + ) def validate_schema(self, schema=None): """ diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index 3a87190422..96d2e0ab92 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -6,7 +6,6 @@ nf-core subworkflows lint """ - import logging import os diff --git a/nf_core/subworkflows/lint/subworkflow_changes.py b/nf_core/subworkflows/lint/subworkflow_changes.py index b7fa13d931..a9c9616a21 100644 --- a/nf_core/subworkflows/lint/subworkflow_changes.py +++ b/nf_core/subworkflows/lint/subworkflow_changes.py @@ -1,6 +1,7 @@ """ Check whether the content of a subworkflow has changed compared to the original repository """ + from pathlib import Path import nf_core.modules.modules_repo diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index f7284320ea..796a56d018 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -1,6 +1,7 @@ """ Lint the tests of a subworkflow in nf-core/modules """ + import json import logging from pathlib import Path diff --git a/nf_core/sync.py b/nf_core/sync.py index 995baeacd2..93a79c3d50 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -1,5 +1,4 @@ -"""Synchronise a pipeline TEMPLATE branch with the template. -""" +"""Synchronise a pipeline TEMPLATE branch with the template.""" import json import logging diff --git a/nf_core/utils.py b/nf_core/utils.py index e1778b55b3..8ea6f418aa 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1,6 +1,7 @@ """ Common utility functions for the nf-core python package. """ + import concurrent.futures import datetime import errno diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index c5067d7210..50024a8ebb 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -1,4 +1,5 @@ """Test generate a snapshot""" + import json from pathlib import Path from unittest.mock import MagicMock diff --git a/tests/components/snapshot_test.py b/tests/components/snapshot_test.py index d774618476..b3fc259770 100644 --- a/tests/components/snapshot_test.py +++ b/tests/components/snapshot_test.py @@ -1,4 +1,5 @@ """Test the 'modules test' or 'subworkflows test' command which runs nf-test test.""" + import shutil from pathlib import Path diff --git a/tests/test_bump_version.py b/tests/test_bump_version.py index 658a2339d4..c697d34287 100644 --- a/tests/test_bump_version.py +++ b/tests/test_bump_version.py @@ -1,5 +1,5 @@ -"""Some tests covering the bump_version code. -""" +"""Some tests covering the bump_version code.""" + import os import yaml diff --git a/tests/test_cli.py b/tests/test_cli.py index 1261e3a9e9..d1dff0fe7b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,4 @@ -""" Tests covering the command-line code. +"""Tests covering the command-line code. Most tests check the cli arguments are passed along and that some action is taken. diff --git a/tests/test_components.py b/tests/test_components.py index b7f67eb51d..eaf999c3c3 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -1,5 +1,4 @@ -""" Tests covering the modules commands -""" +"""Tests covering the modules commands""" import os import shutil diff --git a/tests/test_create.py b/tests/test_create.py index 1cc073cb54..e2672499cd 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -1,5 +1,5 @@ -"""Some tests covering the pipeline creation sub command. -""" +"""Some tests covering the pipeline creation sub command.""" + import os import unittest from pathlib import Path diff --git a/tests/test_download.py b/tests/test_download.py index d823040247..621379842b 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -1,5 +1,4 @@ -"""Tests for the download subcommand of nf-core tools -""" +"""Tests for the download subcommand of nf-core tools""" import os import re diff --git a/tests/test_launch.py b/tests/test_launch.py index dc8d6b147c..79dbe3fb97 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -1,5 +1,4 @@ -""" Tests covering the pipeline launch code. -""" +"""Tests covering the pipeline launch code.""" import json import os diff --git a/tests/test_licenses.py b/tests/test_licenses.py index 4fb58a107c..8023c9e891 100644 --- a/tests/test_licenses.py +++ b/tests/test_licenses.py @@ -1,5 +1,4 @@ -"""Some tests covering the pipeline creation sub command. -""" +"""Some tests covering the pipeline creation sub command.""" # import json # import os # import tempfile diff --git a/tests/test_lint.py b/tests/test_lint.py index 9839265892..8ec97d224a 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -1,5 +1,5 @@ -"""Some tests covering the linting code. -""" +"""Some tests covering the linting code.""" + import fnmatch import json import os diff --git a/tests/test_list.py b/tests/test_list.py index c1f51e03e0..c78276b41d 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -1,5 +1,4 @@ -""" Tests covering the workflow listing code. -""" +"""Tests covering the workflow listing code.""" import json import os diff --git a/tests/test_modules.py b/tests/test_modules.py index 539d3dcc57..944a09f670 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,5 +1,4 @@ -""" Tests covering the modules commands -""" +"""Tests covering the modules commands""" import os import shutil diff --git a/tests/test_refgenie.py b/tests/test_refgenie.py index 5440c1c477..23cc0dd14a 100644 --- a/tests/test_refgenie.py +++ b/tests/test_refgenie.py @@ -1,5 +1,4 @@ -""" Tests covering the refgenie integration code -""" +"""Tests covering the refgenie integration code""" import os import shlex diff --git a/tests/test_schema.py b/tests/test_schema.py index 89fcc98b66..29f4921985 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1,5 +1,4 @@ -""" Tests covering the pipeline schema code. -""" +"""Tests covering the pipeline schema code.""" import json import os diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index cd0bd21146..6163faa7a9 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -1,5 +1,4 @@ -""" Tests covering the subworkflows commands -""" +"""Tests covering the subworkflows commands""" import os import shutil diff --git a/tests/test_sync.py b/tests/test_sync.py index 51a27653ab..6f0e502e8b 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,5 +1,4 @@ -""" Tests covering the sync command -""" +"""Tests covering the sync command""" import json import os diff --git a/tests/test_utils.py b/tests/test_utils.py index 4b5ab19fce..145060450f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,4 @@ -""" Tests covering for utility functions. -""" +"""Tests covering for utility functions.""" import os import shutil From 18c964da7f42212f5ae9cb024dcd68476048fd1c Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 6 Mar 2024 11:23:48 +0100 Subject: [PATCH 028/270] use a cleaner way to force a PR --- nf_core/sync.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/sync.py b/nf_core/sync.py index 5eb14f8348..5e7b198d8d 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -1,5 +1,4 @@ -"""Synchronise a pipeline TEMPLATE branch with the template. -""" +"""Synchronise a pipeline TEMPLATE branch with the template.""" import json import logging @@ -141,7 +140,7 @@ def sync(self): self.made_changes = True # Push and make a pull request if we've been asked to - if self.made_changes and self.make_pr: + if self.made_changes and self.make_pr or self.force_pr: try: # Check that we have an API auth token if os.environ.get("GITHUB_AUTH_TOKEN", "") == "": From f23bd3065498f50529b053c4ffc460ed374e642e Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 6 Mar 2024 15:38:13 +0100 Subject: [PATCH 029/270] update prettier to 3.2.5 --- .pre-commit-config.yaml | 2 ++ nf_core/pipeline-template/.pre-commit-config.yaml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03fbb7bedf..abd92a27f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,8 @@ repos: rev: "v3.1.0" hooks: - id: prettier + additional_dependencies: + - prettier@3.2.5 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python rev: "2.7.3" diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index af57081f60..4dc0f1dcd7 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -3,6 +3,9 @@ repos: rev: "v3.1.0" hooks: - id: prettier + additional_dependencies: + - prettier@3.2.5 + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python rev: "2.7.3" hooks: From df45429cbe1df0d5e57dcd86da8b2da2c3fbac36 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 6 Mar 2024 14:45:39 +0000 Subject: [PATCH 030/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a16db59e56..0c18f1b67f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Update CI to use nf-core/setup-nextflow v2 - Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) +- update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 2b93ad7467882c6f4448eb3b59d62e77a6bc4d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 7 Mar 2024 09:59:19 +0100 Subject: [PATCH 031/270] Update .pre-commit-config.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 508bf1ac24..2df98d8392 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.0 + rev: v0.3.1 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 905d71599048220cb269d9b1442a101e8fcf024e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 7 Mar 2024 11:14:04 +0000 Subject: [PATCH 032/270] fix linting --- nf_core/lint/nextflow_config.py | 5 ++--- nf_core/schema.py | 21 +++++++-------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index d3e29d2363..2e142cde6e 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -245,10 +245,9 @@ def nextflow_config(self): raise AssertionError() except (AssertionError, IndexError): failed.append( - "Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {}".format( - manifest_homepage - ) + f"Config variable ``manifest.homePage`` did not begin with https://github.com/nf-core/:\n {manifest_homepage}" ) + else: passed.append("Config variable ``manifest.homePage`` began with https://github.com/nf-core/") diff --git a/nf_core/schema.py b/nf_core/schema.py index b5816063d7..373f8bbaa1 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -83,16 +83,13 @@ def load_lint_schema(self): self.get_schema_defaults() self.validate_default_params() if len(self.invalid_nextflow_config_default_parameters) > 0: + params = "\n --".join( + [f"{param}: {msg}" for param, msg in self.invalid_nextflow_config_default_parameters.items()] + ) log.info( - "[red][✗] Invalid default parameters found:\n --{}\n\nNOTE: Use null in config for no default.".format( - "\n --".join( - [ - f"{param}: {msg}" - for param, msg in self.invalid_nextflow_config_default_parameters.items() - ] - ) - ) + f"[red][✗] Invalid default parameters found:\n {params} \n\nNOTE: Use null in config for no default." ) + else: log.info(f"[green][✓] Pipeline schema looks valid[/] [dim](found {num_params} params)") except json.decoder.JSONDecodeError as e: @@ -646,17 +643,13 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): # Extra help for people running offline if "Could not connect" in e.args[0]: log.info( - "If you're working offline, now copy your schema ({}) and paste at https://nf-co.re/pipeline_schema_builder".format( - self.schema_filename - ) + f"If you're working offline, now copy your schema ({self.schema_filename}) and paste at https://nf-co.re/pipeline_schema_builder" ) log.info("When you're finished, you can paste the edited schema back into the same file") if self.web_schema_build_web_url: log.info( "To save your work, open {}\n" - "Click the blue 'Finished' button, copy the schema and paste into this file: {}".format( - self.web_schema_build_web_url, self.schema_filename - ) + f"Click the blue 'Finished' button, copy the schema and paste into this file: { self.web_schema_build_web_url, self.schema_filename}" ) return False From 09cf3554490e10c10792154c6693c057448f1be3 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 Mar 2024 16:52:20 +0100 Subject: [PATCH 033/270] switch to setup-nf-test --- .github/workflows/pytest.yml | 24 +----------------------- .github/workflows/rich-codex.yml | 14 +------------- 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 461b888516..5168986aea 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -121,30 +121,8 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@v2 - - name: Look if nf-test is already installed and write to env variable - id: check-nftest - run: | - if [ -f /usr/local/bin/nf-test ]; then - echo "nftest_installed=true" >> $GITHUB_ENV - else - echo "nftest_installed=false" >> $GITHUB_ENV - fi - - - name: Cache nf-test installation - if: env.nftest_installed != 'true' - id: cache-software - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 - with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-nftest-${{ env.date }} - - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' && env.nftest_installed != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ + uses: nf-core/setup-nf-test@v0.0.1 - name: move coveragerc file up run: | diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 9d1bf7b76c..443299b0a2 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -17,20 +17,8 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@v2 - - name: Cache nf-test installation - id: cache-software - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4 - with: - path: | - /usr/local/bin/nf-test - /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-nftest-${{ env.date }} - - name: Install nf-test - if: steps.cache-software.outputs.cache-hit != 'true' - run: | - wget -qO- https://code.askimed.com/install/nf-test | bash - sudo mv nf-test /usr/local/bin/ + uses: nf-core/setup-nf-test@v0.0.1 - name: Install nf-core/tools run: pip install git+https://github.com/nf-core/tools.git@dev From be2ebea3f245bba54ff111c6d78bd592c7e93f68 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 8 Mar 2024 15:55:11 +0000 Subject: [PATCH 034/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b7ff2ef6e..1a03768361 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) - update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) - Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) +- switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 9b96060abf8996586967445b13aaf021b4f9a7ea Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 Mar 2024 17:04:22 +0100 Subject: [PATCH 035/270] run pytest workflow if something changed in pytest --- .github/workflows/pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5168986aea..1bc81fbb9d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -12,7 +12,9 @@ on: paths-ignore: - "docs/**" - "CHANGELOG.md" + # ignore github workflows except for the current one - ".github/**" + - "!.github/workflows/pytest.yml" release: types: [published] workflow_dispatch: From 77d54f009bbbb4ea51d8786747db294dfe2f9ea4 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 Mar 2024 17:24:27 +0100 Subject: [PATCH 036/270] update version --- .github/workflows/pytest.yml | 2 +- .github/workflows/rich-codex.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 1bc81fbb9d..7467a37c4f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -124,7 +124,7 @@ jobs: uses: nf-core/setup-nextflow@v2 - name: Install nf-test - uses: nf-core/setup-nf-test@v0.0.1 + uses: nf-core/setup-nf-test@v0.0.2 - name: move coveragerc file up run: | diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 443299b0a2..ff0b600ab5 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -18,7 +18,7 @@ jobs: uses: nf-core/setup-nextflow@v2 - name: Install nf-test - uses: nf-core/setup-nf-test@v0.0.1 + uses: nf-core/setup-nf-test@v0.0.2 - name: Install nf-core/tools run: pip install git+https://github.com/nf-core/tools.git@dev From 809266b9020e1395db9f107137369e488919dea9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 Mar 2024 19:18:26 +0100 Subject: [PATCH 037/270] update to 0.0.3 --- .github/workflows/pytest.yml | 2 +- .github/workflows/rich-codex.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7467a37c4f..46489229d5 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -124,7 +124,7 @@ jobs: uses: nf-core/setup-nextflow@v2 - name: Install nf-test - uses: nf-core/setup-nf-test@v0.0.2 + uses: nf-core/setup-nf-test@v0.0.3 - name: move coveragerc file up run: | diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index ff0b600ab5..0b9ca061c8 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -18,7 +18,7 @@ jobs: uses: nf-core/setup-nextflow@v2 - name: Install nf-test - uses: nf-core/setup-nf-test@v0.0.2 + uses: nf-core/setup-nf-test@v0.0.3 - name: Install nf-core/tools run: pip install git+https://github.com/nf-core/tools.git@dev From 9f1ccbde5e7a0a34fa9604093ac9e87395a5b994 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 8 Mar 2024 21:47:45 +0100 Subject: [PATCH 038/270] switch to 1.0.0 --- .github/workflows/pytest.yml | 2 +- .github/workflows/rich-codex.yml | 2 +- CHANGELOG.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 46489229d5..2aed1752ff 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -124,7 +124,7 @@ jobs: uses: nf-core/setup-nextflow@v2 - name: Install nf-test - uses: nf-core/setup-nf-test@v0.0.3 + uses: nf-core/setup-nf-test@v1 - name: move coveragerc file up run: | diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 0b9ca061c8..13fd71a883 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -18,7 +18,7 @@ jobs: uses: nf-core/setup-nextflow@v2 - name: Install nf-test - uses: nf-core/setup-nf-test@v0.0.3 + uses: nf-core/setup-nf-test@v1 - name: Install nf-core/tools run: pip install git+https://github.com/nf-core/tools.git@dev diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a03768361..a5ef0c8706 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,9 @@ - Update CI to use nf-core/setup-nextflow v2 - Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) -- update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) +- Update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) - Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) -- switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) +- Switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 84333f07f2cfee320d59bf5efaee6731a2936d57 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 01:39:16 +0000 Subject: [PATCH 039/270] Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b618e00837..ec1c068e79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.1 + rev: v0.3.2 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 936623ee8c2532b165343a7bf2a9fc42040bf398 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 9 Mar 2024 01:40:04 +0000 Subject: [PATCH 040/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b7ff2ef6e..5480bbed1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) - update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) - Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.2 ([#2836](https://github.com/nf-core/tools/pull/2836)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 91ececd32e76cb45fea41231654b7a1c7f5b8b8b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 07:27:08 +0000 Subject: [PATCH 041/270] Update pre-commit hook pre-commit/mirrors-mypy to v1.9.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec1c068e79..d0d5381b9f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: alias: ec - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.8.0" + rev: "v1.9.0" hooks: - id: mypy additional_dependencies: From 385c81defab5e05493849d31085bd88928474c93 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 11 Mar 2024 09:49:48 +0100 Subject: [PATCH 042/270] try to fix pytest CI --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e4319d2352..6b5b3ab57d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ pillow pdiff pre-commit prompt_toolkit>=3.0.3 -pytest-workflow>=1.6.0 +pytest-workflow>=2.0.0 pytest>=7.0.0 pyyaml questionary>=1.8.0 From 76faed61bd689525381232a29e4e5d821d0e081b Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 11 Mar 2024 10:04:30 +0100 Subject: [PATCH 043/270] remove warning filter --- pytest.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 652bdf8e53..cf37159478 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,4 @@ [pytest] -filterwarnings = - ignore::pytest.PytestRemovedIn8Warning:_pytest.nodes:140 testpaths = tests python_files = test_*.py From 8824d239342c906c039147eea116e033e747fde6 Mon Sep 17 00:00:00 2001 From: asp8200 Date: Mon, 11 Mar 2024 10:47:25 +0000 Subject: [PATCH 044/270] Only match assignments of params --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 8ea6f418aa..9261388a39 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -289,7 +289,7 @@ def fetch_wf_config(wf_path, cache_config=True): main_nf = os.path.join(wf_path, "main.nf") with open(main_nf) as fh: for line in fh: - match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=", line) + match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line) if match: config[match.group(1)] = "null" except FileNotFoundError as e: From 2e1c81866e8fb78403a69d35c51fd91b352d0d92 Mon Sep 17 00:00:00 2001 From: asp8200 Date: Mon, 11 Mar 2024 12:00:42 +0000 Subject: [PATCH 045/270] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce479c29bb..3239824803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ ### Linting +- Only match assignments of params in `main.nf` and not references like `params.aligner == ` ([#2833](https://github.com/nf-core/tools/pull/2833)) + ### Components ### General From d25f22b86ee56d7dbdee289eef428129f8b77fdb Mon Sep 17 00:00:00 2001 From: asp8200 Date: Tue, 12 Mar 2024 10:35:12 +0000 Subject: [PATCH 046/270] pleasing ruff --- nf_core/pipeline-template/main.nf | 2 ++ tests/lint/nextflow_config.py | 30 ++++++++++++++++++++++++++++++ tests/test_lint.py | 2 ++ 3 files changed, 34 insertions(+) diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index 2590f7467b..d90f7616ce 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -86,6 +86,8 @@ workflow { params.input ) + params.max_time == 42 + // // WORKFLOW: Run main workflow // diff --git a/tests/lint/nextflow_config.py b/tests/lint/nextflow_config.py index fa85568f14..a8d343427f 100644 --- a/tests/lint/nextflow_config.py +++ b/tests/lint/nextflow_config.py @@ -100,6 +100,36 @@ def test_default_values_fail(self): ) +def test_catch_params_assignment_in_main_nf(self): + """Test linting fails if main.nf contains an assignment to a parameter from nextflow_schema.json.""" + new_pipeline = self._make_pipeline_copy() + # Change reference of params.max_time to assignment + main_nf_file = Path(new_pipeline) / "main.nf" + with open(main_nf_file) as f: + content = f.read() + fail_content = re.sub(r"params.max_time ==", "params.max_time =", content) + print(fail_content) + with open(main_nf_file, "w") as f: + f.write(fail_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 1 + assert ( + result["failed"][0] + == "Config default value incorrect: `params.max_time` is set as `240.h` in `nextflow_schema.json` but is `null` in `nextflow.config`." + ) + + +def test_allow_params_reference_in_main_nf(self): + """Test linting allows for references like `params.aligner == 'bwa'` in main.nf. The test will detect if the bug mentioned in GitHub-issue #2833 reemerges.""" + new_pipeline = self._make_pipeline_copy() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load_pipeline_config() + result = lint_obj.nextflow_config() + assert len(result["failed"]) == 0 + + def test_default_values_ignored(self): """Test ignoring linting of default values.""" new_pipeline = self._make_pipeline_copy() diff --git a/tests/test_lint.py b/tests/test_lint.py index 8ec97d224a..d10cef37e4 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -221,6 +221,8 @@ def test_sphinx_md_files(self): test_multiqc_incorrect_export_plots, ) from .lint.nextflow_config import ( # type: ignore[misc] + test_allow_params_reference_in_main_nf, + test_catch_params_assignment_in_main_nf, test_default_values_fail, test_default_values_float, test_default_values_float_fail, From 3080e58091f340a7b98ceedc3b1b3add290453a4 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 12 Mar 2024 10:40:52 +0000 Subject: [PATCH 047/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3239824803..69a6830b55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) - Switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.2 ([#2836](https://github.com/nf-core/tools/pull/2836)) +- Add tests for assignment and referencing of params in main.nf ([#2841](https://github.com/nf-core/tools/pull/2841)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From adf3948ad0a0bbfc9f48acce095acf3bca68c6b4 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Mar 2024 11:49:07 +0100 Subject: [PATCH 048/270] optimize layers in dockerfile --- nf_core/gitpod/gitpod.Dockerfile | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index f04c7dcd04..438eba5df1 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -8,7 +8,7 @@ USER root # Install util tools. # software-properties-common is needed to add ppa support for Apptainer installation RUN apt-get update --quiet && \ - apt-get install --quiet --yes \ + apt-get install --quiet --yes --no-install-recommends \ apt-transport-https \ apt-utils \ sudo \ @@ -18,18 +18,17 @@ RUN apt-get update --quiet && \ curl \ tree \ graphviz \ - software-properties-common - -# Install Apptainer (Singularity) -RUN add-apt-repository -y ppa:apptainer/ppa && \ + software-properties-common && \ + add-apt-repository -y ppa:apptainer/ppa && \ apt-get update --quiet && \ - apt install -y apptainer - -# Install Conda -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + apt-get install --quiet --yes apptainer && \ + wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda3-latest-Linux-x86_64.sh + rm Miniconda3-latest-Linux-x86_64.sh && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +# Set PATH for Conda ENV PATH="/opt/conda/bin:$PATH" # Add the nf-core source files to the image @@ -58,12 +57,10 @@ RUN conda config --add channels defaults && \ pytest-workflow && \ conda clean --all --force-pkgs-dirs --yes -# Update Nextflow -RUN nextflow self-update - -# Install nf-core -RUN python -m pip install . --no-cache-dir +# Update Nextflow and Install nf-core +RUN nextflow self-update && \ + python -m pip install . --no-cache-dir # Setup pdiff for nf-test diffs -RUN export NFT_DIFF="pdiff" && \ - export NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" +ENV NFT_DIFF="pdiff" +ENV NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" From db317caf14b20fdf1ebe89c9824ad180b23b7954 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 12 Mar 2024 10:59:11 +0000 Subject: [PATCH 049/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3239824803..7169d64a3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) - Switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.2 ([#2836](https://github.com/nf-core/tools/pull/2836)) +- optimize layers in dockerfile ([#2842](https://github.com/nf-core/tools/pull/2842)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From b0431b2f07533fe7f2e32184b49f170f3cfa6388 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 12 Mar 2024 12:05:19 +0100 Subject: [PATCH 050/270] drop mamba and nf-core conda install, add mypy --- nf_core/gitpod/gitpod.Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 438eba5df1..f0ee6054a7 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -46,13 +46,12 @@ RUN conda config --add channels defaults && \ conda config --add channels conda-forge && \ conda config --set channel_priority strict && \ conda install --quiet --yes --name base \ - mamba \ nextflow \ - nf-core \ nf-test \ prettier \ pre-commit \ ruff \ + mypy \ openjdk \ pytest-workflow && \ conda clean --all --force-pkgs-dirs --yes From 4fc2e1c2a7bb1e19de81ff76f1b29c495415b388 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 10:13:19 +0000 Subject: [PATCH 051/270] Update gitpod/workspace-base Docker digest to 1e133e5 --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index f04c7dcd04..b7aa72ee02 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:c15ee2f4de8902421ccbfbd77329a9e8e4abf18477e1b5ae8a1a572790254647 +FROM gitpod/workspace-base@sha256:1e133e5691add6c19443672594b9f3d7d9c3372ead4c86a4490c2701dbfa32e3 USER root From 085a12345c6ed330c907e25b20ebfd184f3e6f63 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 12 Mar 2024 12:54:49 +0000 Subject: [PATCH 052/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3239824803..6fb848daaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) - Switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.2 ([#2836](https://github.com/nf-core/tools/pull/2836)) +- Update gitpod/workspace-base Docker digest to 1e133e5 ([#2843](https://github.com/nf-core/tools/pull/2843)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 336f8b41b1cbe050125f033c9c1cd9aa55701fa0 Mon Sep 17 00:00:00 2001 From: asp8200 Date: Tue, 12 Mar 2024 14:57:41 +0000 Subject: [PATCH 053/270] Avoiding changing the template main.nf --- nf_core/lint/nextflow_config.py | 1 - nf_core/pipeline-template/main.nf | 2 -- tests/lint/nextflow_config.py | 15 +++++++-------- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nf_core/lint/nextflow_config.py b/nf_core/lint/nextflow_config.py index 2e142cde6e..47b7d78f5e 100644 --- a/nf_core/lint/nextflow_config.py +++ b/nf_core/lint/nextflow_config.py @@ -374,7 +374,6 @@ def nextflow_config(self): schema.load_schema() schema.get_schema_defaults() # Get default values from schema schema.get_schema_types() # Get types from schema - self.nf_config.keys() # Params in nextflow.config for param_name in schema.schema_defaults.keys(): param = "params." + param_name if param in ignore_defaults: diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf index d90f7616ce..2590f7467b 100644 --- a/nf_core/pipeline-template/main.nf +++ b/nf_core/pipeline-template/main.nf @@ -86,8 +86,6 @@ workflow { params.input ) - params.max_time == 42 - // // WORKFLOW: Run main workflow // diff --git a/tests/lint/nextflow_config.py b/tests/lint/nextflow_config.py index a8d343427f..06af8c4fb8 100644 --- a/tests/lint/nextflow_config.py +++ b/tests/lint/nextflow_config.py @@ -83,7 +83,6 @@ def test_default_values_fail(self): with open(nf_schema_file) as f: content = f.read() fail_content = re.sub(r'"default": "128.GB"', '"default": "18.GB"', content) - print(fail_content) with open(nf_schema_file, "w") as f: f.write(fail_content) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -103,14 +102,10 @@ def test_default_values_fail(self): def test_catch_params_assignment_in_main_nf(self): """Test linting fails if main.nf contains an assignment to a parameter from nextflow_schema.json.""" new_pipeline = self._make_pipeline_copy() - # Change reference of params.max_time to assignment + # Add parameter assignment in main.nf main_nf_file = Path(new_pipeline) / "main.nf" - with open(main_nf_file) as f: - content = f.read() - fail_content = re.sub(r"params.max_time ==", "params.max_time =", content) - print(fail_content) - with open(main_nf_file, "w") as f: - f.write(fail_content) + with open(main_nf_file, "a") as f: + f.write("params.max_time = 42") lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load_pipeline_config() result = lint_obj.nextflow_config() @@ -124,6 +119,10 @@ def test_catch_params_assignment_in_main_nf(self): def test_allow_params_reference_in_main_nf(self): """Test linting allows for references like `params.aligner == 'bwa'` in main.nf. The test will detect if the bug mentioned in GitHub-issue #2833 reemerges.""" new_pipeline = self._make_pipeline_copy() + # Add parameter reference in main.nf + main_nf_file = Path(new_pipeline) / "main.nf" + with open(main_nf_file, "a") as f: + f.write("params.max_time == 42") lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load_pipeline_config() result = lint_obj.nextflow_config() From 10e6579a78151f56af447ddb0d4a7b1f53b912fd Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Wed, 13 Mar 2024 11:00:23 +0100 Subject: [PATCH 054/270] Link to troubleshooting docs when pipeline fails --- .../subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 5bc51caab8..e02a113755 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -143,6 +143,10 @@ workflow PIPELINE_COMPLETION { imNotification(summary_params, hook_url) } } + + workflow.onError { + log.error "Pipeline failed. Please refer to troubleshooting docs: https://nf-co.re/docs/usage/troubleshooting" + } } /* From 2994b6d3f23ebd96c685830884a8ef8ca65d462f Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 13 Mar 2024 10:04:44 +0000 Subject: [PATCH 055/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71eb70b6b2..d01dd9190e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Add tests for assignment and referencing of params in main.nf ([#2841](https://github.com/nf-core/tools/pull/2841)) - Optimize layers in dockerfile ([#2842](https://github.com/nf-core/tools/pull/2842)) - Update gitpod/workspace-base Docker digest to 1e133e5 ([#2843](https://github.com/nf-core/tools/pull/2843)) +- Link to troubleshooting docs when pipeline fails ([#2845](https://github.com/nf-core/tools/pull/2845)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From eaec765633d08f47d699a66b7fcc649b08be96be Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 13 Mar 2024 10:10:37 +0000 Subject: [PATCH 056/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d01dd9190e..3d31e99700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Remove fasta default from nextflow.config ([#2828](https://github.com/nf-core/tools/pull/2828)) - Update templates to use nf-core/setup-nextflow v2 +- Link to troubleshooting docs when pipeline fails ([#2845](https://github.com/nf-core/tools/pull/2845)) ### Linting @@ -24,7 +25,6 @@ - Add tests for assignment and referencing of params in main.nf ([#2841](https://github.com/nf-core/tools/pull/2841)) - Optimize layers in dockerfile ([#2842](https://github.com/nf-core/tools/pull/2842)) - Update gitpod/workspace-base Docker digest to 1e133e5 ([#2843](https://github.com/nf-core/tools/pull/2843)) -- Link to troubleshooting docs when pipeline fails ([#2845](https://github.com/nf-core/tools/pull/2845)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From f066e6540301e4b636d4d125a5cbadbfe75ae0a6 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 13 Mar 2024 18:56:07 +0100 Subject: [PATCH 057/270] Add fallback to DownloadPipeline-Workflow in case stub runs are not supported by the pipeline. --- .../.github/workflows/download_pipeline.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 4fdec4e243..2cb0b16f9f 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -14,6 +14,8 @@ on: pull_request: types: - opened + - edited + - synchronize branches: - master pull_request_target: @@ -65,8 +67,16 @@ jobs: - name: Inspect download run: tree ./${{ env.REPOTITLE_LOWERCASE }} - - name: Run the downloaded pipeline + - name: Run the downloaded pipeline (stub) + id: stub_run_pipeline env: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true - run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results{% endraw %} + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results + - name: Run the downloaded pipeline (Stub run not supported) + id: run_pipeline + if: ${{ failure() && steps.stub_run_pipeline.conclusion == 'failure' }} + env: + NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_HOME_MOUNT: true + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results{% endraw %} From 393fd15db14ade4958c1a7586a92b75c069f550c Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 13 Mar 2024 19:17:15 +0100 Subject: [PATCH 058/270] Changelog. --- CHANGELOG.md | 1 + .../pipeline-template/.github/workflows/download_pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d31e99700..a18df7cffe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Remove fasta default from nextflow.config ([#2828](https://github.com/nf-core/tools/pull/2828)) - Update templates to use nf-core/setup-nextflow v2 - Link to troubleshooting docs when pipeline fails ([#2845](https://github.com/nf-core/tools/pull/2845)) +- Add fallback to `download_pipeline.yml` in case the pipeline does not support stub runs ([#2846](https://github.com/nf-core/tools/pull/2846)) ### Linting diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 2cb0b16f9f..f489cf3146 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -73,7 +73,7 @@ jobs: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results - - name: Run the downloaded pipeline (Stub run not supported) + - name: Run the downloaded pipeline (stub run not supported) id: run_pipeline if: ${{ failure() && steps.stub_run_pipeline.conclusion == 'failure' }} env: From 0a471a98e281a392829f91f91e987846f3c927b0 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 13 Mar 2024 20:39:06 +0100 Subject: [PATCH 059/270] Need continue-on-error: true so that the job does not fail if the step fails. Modify if-clause of fallback step accordingly. --- .../pipeline-template/.github/workflows/download_pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index f489cf3146..d4b920ef11 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -69,13 +69,14 @@ jobs: - name: Run the downloaded pipeline (stub) id: stub_run_pipeline + continue-on-error: true env: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results - name: Run the downloaded pipeline (stub run not supported) id: run_pipeline - if: ${{ failure() && steps.stub_run_pipeline.conclusion == 'failure' }} + if: ${{ job.steps.stub_run_pipeline.status == failure() }} env: NXF_SINGULARITY_CACHEDIR: ./ NXF_SINGULARITY_HOME_MOUNT: true From 784c0ea798a0e63fbf530dc7626205f0f917a754 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 17:05:34 +0000 Subject: [PATCH 060/270] Update python:3.11-slim Docker digest to a2eb07f --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 62431be140..60768499a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim@sha256:ce81dc539f0aedc9114cae640f8352fad83d37461c24a3615b01f081d0c0583a +FROM python:3.11-slim@sha256:a2eb07f336e4f194358382611b4fea136c632b40baa6314cb27a366deeaf0144 LABEL authors="phil.ewels@scilifelab.se,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for the nfcore tools" From 80cc517c279abac7fcfcaa0b446b62ff55201422 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 13 Mar 2024 20:30:38 +0000 Subject: [PATCH 061/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d31e99700..b767b63032 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Add tests for assignment and referencing of params in main.nf ([#2841](https://github.com/nf-core/tools/pull/2841)) - Optimize layers in dockerfile ([#2842](https://github.com/nf-core/tools/pull/2842)) - Update gitpod/workspace-base Docker digest to 1e133e5 ([#2843](https://github.com/nf-core/tools/pull/2843)) +- Update python:3.11-slim Docker digest to a2eb07f ([#2847](https://github.com/nf-core/tools/pull/2847)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 852e93c2341f4896398e0d1a644aa94c70bc7b06 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 15 Mar 2024 10:31:48 +0100 Subject: [PATCH 062/270] set topic variable correclty --- .../.github/workflows/release-announcements.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index 58ef1731f7..2c57060257 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -12,7 +12,7 @@ jobs: - name: get topics and convert to hashtags id: get_topics run: | - curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' >> $GITHUB_OUTPUT + echo "topics=$(curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ')" >> $GITHUB_OUTPUT - uses: rzr/fediverse-action@master with: @@ -25,7 +25,7 @@ jobs: Please see the changelog: ${{ github.event.release.html_url }} - ${{ steps.get_topics.outputs.GITHUB_OUTPUT }} #nfcore #openscience #nextflow #bioinformatics + ${{ steps.get_topics.outputs.topics }} #nfcore #openscience #nextflow #bioinformatics send-tweet: runs-on: ubuntu-latest From 292a8f1b7742814561bf49dc3bf6e37ce0bca9a3 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 15 Mar 2024 10:55:37 +0100 Subject: [PATCH 063/270] update changelog manually --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bb9843197..fddde7f010 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Update templates to use nf-core/setup-nextflow v2 - Link to troubleshooting docs when pipeline fails ([#2845](https://github.com/nf-core/tools/pull/2845)) - Add fallback to `download_pipeline.yml` in case the pipeline does not support stub runs ([#2846](https://github.com/nf-core/tools/pull/2846)) +- Set topic variable correctly in the mastodon announcement ([#2848](https://github.com/nf-core/tools/pull/2848)) ### Linting From 8f3ddb6488ab0254dcc712b1b085351627c86e58 Mon Sep 17 00:00:00 2001 From: Ramprasad Neethiraj <20065894+ramprasadn@users.noreply.github.com> Date: Fri, 15 Mar 2024 11:06:50 +0100 Subject: [PATCH 064/270] add cleanup step to download pipeline workflow --- .../pipeline-template/.github/workflows/download_pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index dcd7caabfc..b53a427e06 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -30,6 +30,9 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@v1 + - name: Disk space cleanup + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" From 514b65d646248dcf50ca5c70898c9c499b7cec29 Mon Sep 17 00:00:00 2001 From: Ramprasad Neethiraj <20065894+ramprasadn@users.noreply.github.com> Date: Fri, 15 Mar 2024 11:22:40 +0100 Subject: [PATCH 065/270] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bb9843197..813c2c3076 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Update templates to use nf-core/setup-nextflow v2 - Link to troubleshooting docs when pipeline fails ([#2845](https://github.com/nf-core/tools/pull/2845)) - Add fallback to `download_pipeline.yml` in case the pipeline does not support stub runs ([#2846](https://github.com/nf-core/tools/pull/2846)) +- Add a cleanup action to `download_pipeline.yml` to fix failures caused by inadequate storage space on the runner ([#2849](https://github.com/nf-core/tools/pull/2849)) ### Linting From 7aae5d1c27a81bc01144f06925ec4991d6915dce Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 19:16:03 +0000 Subject: [PATCH 066/270] Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d0d5381b9f..8a68848237 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.2 + rev: v0.3.3 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From f8d135ef43fcba2b8cf065b637861310340ef1f5 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 15 Mar 2024 19:18:54 +0000 Subject: [PATCH 067/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 813c2c3076..05f88b2c80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - Optimize layers in dockerfile ([#2842](https://github.com/nf-core/tools/pull/2842)) - Update gitpod/workspace-base Docker digest to 1e133e5 ([#2843](https://github.com/nf-core/tools/pull/2843)) - Update python:3.11-slim Docker digest to a2eb07f ([#2847](https://github.com/nf-core/tools/pull/2847)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 ([#2850](https://github.com/nf-core/tools/pull/2850)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 03646ebcf065747b2d354295653190e1ca7af691 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 17 Mar 2024 02:10:33 +0000 Subject: [PATCH 068/270] Update dawidd6/action-download-artifact digest to a430ac5 --- nf_core/pipeline-template/.github/workflows/linting_comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index bb3eafcc2c..67ef7b534d 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@f6b0bace624032e30a85a8fd9c1a7f8f611f5737 # v3 + uses: dawidd6/action-download-artifact@a430ac5786b39ad5869da25a98130624d2ce340c # v3 with: workflow: linting.yml workflow_conclusion: completed From 3cbcca7b6443ecadd8697c4a109bb3d627d8158c Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 18 Mar 2024 09:32:16 +0100 Subject: [PATCH 069/270] Remove 'Tower' from _almost_ everywhere.. --- CHANGELOG.md | 5 ++ README.md | 10 ++-- nf_core/__main__.py | 14 ++++-- nf_core/download.py | 46 +++++++++---------- nf_core/lint/actions_awsfulltest.py | 4 +- .../.github/workflows/awsfulltest.yml | 10 ++-- .../.github/workflows/awstest.yml | 12 ++--- nf_core/pipeline-template/README.md | 2 +- nf_core/utils.py | 2 +- requirements.txt | 2 +- tests/test_cli.py | 4 +- tests/test_download.py | 12 ++--- 12 files changed, 68 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 813c2c3076..c1147ff8b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ - Only match assignments of params in `main.nf` and not references like `params.aligner == ` ([#2833](https://github.com/nf-core/tools/pull/2833)) +### Download + +- Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. + ### Components ### General @@ -29,6 +33,7 @@ - Optimize layers in dockerfile ([#2842](https://github.com/nf-core/tools/pull/2842)) - Update gitpod/workspace-base Docker digest to 1e133e5 ([#2843](https://github.com/nf-core/tools/pull/2843)) - Update python:3.11-slim Docker digest to a2eb07f ([#2847](https://github.com/nf-core/tools/pull/2847)) +- Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] diff --git a/README.md b/README.md index cf0b01d210..5869ad9ac1 100644 --- a/README.md +++ b/README.md @@ -376,7 +376,7 @@ The pipeline files are automatically updated (`params.custom_config_base` is set So using `-profile ` should work if available within [nf-core/configs](https://github.com/nf-core/configs). > [!WARNING] -> This option is not available when downloading a pipeline for use with [Nextflow Tower](#adapting-downloads-to-nextflow-tower) because the application manages all configurations separately. +> This option is not available when downloading a pipeline for use with [Sqerea Platform](#adapting-downloads-to-seqera-platform) because the application manages all configurations separately. ### Downloading Apptainer containers @@ -428,14 +428,14 @@ Note that compressing many GBs of binary files can be slow, so specifying `--com If the download speeds are much slower than your internet connection is capable of, you can set `--parallel-downloads` to a large number to download loads of images at once. -### Adapting downloads to Nextflow Tower +### Adapting downloads to Seqera Platform -[seqeralabs® Nextflow Tower](https://cloud.tower.nf/) provides a graphical user interface to oversee pipeline runs, gather statistics and configure compute resources. While pipelines added to _Tower_ are preferably hosted at a Git service, providing them as disconnected, self-reliant repositories is also possible for premises with restricted network access. Choosing the `--tower` flag will download the pipeline in an appropriate form. +[Seqera Platform](https://seqera.io/platform/) (formerly _"Nextflow Tower"_) provides a graphical user interface to oversee pipeline runs, gather statistics and configure compute resources. While pipelines added to _Seqerea Platform_ are preferably hosted at a Git service, providing them as disconnected, self-reliant repositories is also possible for premises with restricted network access. Choosing the `--platform` flag will download the pipeline in an appropriate form. -Subsequently, the `*.git` folder can be moved to it's final destination and linked with a pipeline in _Tower_ using the `file:/` prefix. +Subsequently, the `*.git` folder can be moved to it's final destination and linked with a pipeline in _Seqerea Platform_ using the `file:/` prefix. > [!TIP] -> Also without access to Tower, pipelines downloaded with the `--tower` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. +> Also without access to Seqerea Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. ## Pipeline software licences diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d6f6077be9..082e5b8d71 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -359,10 +359,17 @@ def create_params_file(pipeline, revision, output, force, show_hidden): ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") @click.option( - "-t", "--tower", is_flag=True, default=False, + hidden=True, + help="Download for Seqera Platform. DEPRECATED: Please use --platform instead.", +) +@click.option( + "-t", + "--platform", + is_flag=True, + default=False, help="Download for Seqera Platform (formerly Nextflow Tower)", ) @click.option( @@ -370,7 +377,7 @@ def create_params_file(pipeline, revision, output, force, show_hidden): "--download-configuration", is_flag=True, default=False, - help="Include configuration profiles in download. Not available with `--tower`", + help="Include configuration profiles in download. Not available with `--platform`", ) # -c changed to -s for consistency with other --container arguments, where it is always the first letter of the last word. # Also -c might be used instead of -d for config in a later release, but reusing params for different options in two subsequent releases might be too error-prone. @@ -412,6 +419,7 @@ def download( compress, force, tower, + platform, download_configuration, container_system, container_library, @@ -433,7 +441,7 @@ def download( outdir, compress, force, - tower, + tower or platform, # True if either specified download_configuration, container_system, container_library, diff --git a/nf_core/download.py b/nf_core/download.py index d08e0ba40e..f6c12fa49f 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -90,7 +90,7 @@ class DownloadWorkflow: pipeline (str): A nf-core pipeline name. revision (List[str]): The workflow revision to download, like `1.0`. Defaults to None. container (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False. - tower (bool): Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. + platform (bool): Flag, to customize the download for Seqera Platform (convert to git bare repo). Defaults to False. outdir (str): Path to the local download directory. Defaults to None. """ @@ -101,7 +101,7 @@ def __init__( outdir=None, compress_type=None, force=False, - tower=False, + platform=False, download_configuration=None, container_system=None, container_library=None, @@ -120,11 +120,11 @@ def __init__( self.output_filename = None self.compress_type = compress_type self.force = force - self.tower = tower + self.platform = platform # if flag is not specified, do not assume deliberate choice and prompt config inclusion interactively. - # this implies that non-interactive "no" choice is only possible implicitly (e.g. with --tower or if prompt is suppressed by !stderr.is_interactive). + # this implies that non-interactive "no" choice is only possible implicitly (e.g. with --platform or if prompt is suppressed by !stderr.is_interactive). # only alternative would have been to make it a parameter with argument, e.g. -d="yes" or -d="no". - self.include_configs = True if download_configuration else False if bool(tower) else None + self.include_configs = True if download_configuration else False if bool(platform) else None # Specifying a cache index or container library implies that containers should be downloaded. self.container_system = "singularity" if container_cache_index or bool(container_library) else container_system # Manually specified container library (registry) @@ -165,8 +165,8 @@ def download_workflow(self): ) self.prompt_revision() self.get_revision_hash() - # Inclusion of configs is unnecessary for Tower. - if not self.tower and self.include_configs is None: + # Inclusion of configs is unnecessary for Seqera Platform. + if not self.platform and self.include_configs is None: self.prompt_config_inclusion() # If a remote cache is specified, it is safe to assume images should be downloaded. if not self.container_cache_utilisation == "remote": @@ -177,7 +177,7 @@ def download_workflow(self): self.prompt_singularity_cachedir_utilization() self.prompt_singularity_cachedir_remote() # Nothing meaningful to compress here. - if not self.tower: + if not self.platform: self.prompt_compression_type() except AssertionError as e: raise DownloadError(e) from e @@ -196,7 +196,7 @@ def download_workflow(self): ) # Set an output filename now that we have the outdir - if self.tower: + if self.platform: self.output_filename = f"{self.outdir}.git" summary_log.append(f"Output file: '{self.output_filename}'") elif self.compress_type is not None: @@ -205,11 +205,11 @@ def download_workflow(self): else: summary_log.append(f"Output directory: '{self.outdir}'") - if not self.tower: + if not self.platform: # Only show entry, if option was prompted. summary_log.append(f"Include default institutional configuration: '{self.include_configs}'") else: - summary_log.append(f"Enabled for seqeralabs® Nextflow Tower: '{self.tower}'") + summary_log.append(f"Enabled for Seqera Platform: '{self.platform}'") # Check that the outdir doesn't already exist if os.path.exists(self.outdir): @@ -233,8 +233,8 @@ def download_workflow(self): log.info("Saving '{}'\n {}".format(self.pipeline, "\n ".join(summary_log))) # Perform the actual download - if self.tower: - self.download_workflow_tower() + if self.platform: + self.download_workflow_platform() else: self.download_workflow_static() @@ -273,7 +273,7 @@ def download_workflow_static(self): log.info("Compressing output into archive") self.compress_download() - def download_workflow_tower(self, location=None): + def download_workflow_platform(self, location=None): """Create a bare-cloned git repository of the workflow, so it can be launched with `tw launch` as file:/ pipeline""" log.info("Collecting workflow from GitHub") @@ -289,7 +289,7 @@ def download_workflow_tower(self, location=None): # Remove tags for those revisions that had not been selected self.workflow_repo.tidy_tags_and_branches() - # create a bare clone of the modified repository needed for Tower + # create a bare clone of the modified repository needed for Seqera Platform self.workflow_repo.bare_clone(os.path.join(self.outdir, self.output_filename)) # extract the required containers @@ -306,9 +306,9 @@ def download_workflow_tower(self, location=None): except OSError as e: raise DownloadError(f"[red]{e}[/]") from e - # Justify why compression is skipped for Tower downloads (Prompt is not shown, but CLI argument could have been set) + # Justify why compression is skipped for Seqera Platform downloads (Prompt is not shown, but CLI argument could have been set) if self.compress_type is not None: - log.info("Compression choice is ignored for Tower downloads since nothing can be reasonably compressed.") + log.info("Compression choice is ignored for Seqera Platform downloads since nothing can be reasonably compressed.") def prompt_pipeline_name(self): """Prompt for the pipeline name if not set with a flag""" @@ -321,13 +321,13 @@ def prompt_revision(self): """ Prompt for pipeline revision / branch Prompt user for revision tag if '--revision' was not set - If --tower is specified, allow to select multiple revisions + If --platform is specified, allow to select multiple revisions Also the static download allows for multiple revisions, but we do not prompt this option interactively. """ if not bool(self.revision): (choice, tag_set) = nf_core.utils.prompt_pipeline_release_branch( - self.wf_revisions, self.wf_branches, multiple=self.tower + self.wf_revisions, self.wf_branches, multiple=self.platform ) """ The checkbox() prompt unfortunately does not support passing a Validator, @@ -384,7 +384,7 @@ def get_revision_hash(self): else: self.outdir = f"{self.pipeline.replace('/', '-').lower()}_{self.revision[0]}" - if not self.tower: + if not self.platform: for revision, wf_sha in self.wf_sha.items(): # Set the download URL and return - only applicable for classic downloads self.wf_download_url = { @@ -406,7 +406,7 @@ def prompt_config_inclusion(self): def prompt_container_download(self): """Prompt whether to download container images or not""" - if self.container_system is None and stderr.is_interactive and not self.tower: + if self.container_system is None and stderr.is_interactive and not self.platform: stderr.print("\nIn addition to the pipeline code, this tool can download software containers.") self.container_system = questionary.select( "Download software container images:", @@ -1621,7 +1621,7 @@ def setup_local_repo(self, remote, location=None, in_cache=True): def tidy_tags_and_branches(self): """ Function to delete all tags and branches that are not of interest to the downloader. - This allows a clutter-free experience in Tower. The untagged commits are evidently still available. + This allows a clutter-free experience in Seqera Platform. The untagged commits are evidently still available. However, due to local caching, the downloader might also want access to revisions that had been deleted before. In that case, don't bother with re-adding the tags and rather download anew from Github. @@ -1655,7 +1655,7 @@ def tidy_tags_and_branches(self): if self.repo.head.is_detached: self.repo.head.reset(index=True, working_tree=True) - # no branch exists, but one is required for Tower's UI to display revisions correctly). Thus, "latest" will be created. + # no branch exists, but one is required for Seqera Platform's UI to display revisions correctly). Thus, "latest" will be created. if not bool(self.repo.heads): if self.repo.is_valid_object("latest"): # "latest" exists as tag but not as branch diff --git a/nf_core/lint/actions_awsfulltest.py b/nf_core/lint/actions_awsfulltest.py index 66aa3f99bf..d5a061c935 100644 --- a/nf_core/lint/actions_awsfulltest.py +++ b/nf_core/lint/actions_awsfulltest.py @@ -17,7 +17,7 @@ def actions_awsfulltest(self): .. note:: You can manually trigger the AWS tests by going to the `Actions` tab on the pipeline GitHub repository and selecting the `nf-core AWS full size tests` workflow on the left. - .. tip:: For tests on full data prior to release, `Nextflow Tower `_ launch feature can be employed. + .. tip:: For tests on full data prior to release, `Seqera Platform `_ launch feature can be employed. The ``.github/workflows/awsfulltest.yml`` file is tested for the following: @@ -52,7 +52,7 @@ def actions_awsfulltest(self): # Warn if `-profile test` is still unchanged try: - steps = wf["jobs"]["run-tower"]["steps"] + steps = wf["jobs"]["run-platform"]["steps"] if not any(aws_profile in step["run"] for step in steps if "run" in step.keys()): raise AssertionError() except (AssertionError, KeyError, TypeError): diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 4c9fd69fcc..56ecb60302 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -8,12 +8,12 @@ on: types: [published] workflow_dispatch: jobs: - run-tower: + run-platform: name: Run AWS full tests if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - - name: Launch workflow via tower + - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) @@ -33,7 +33,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: Tower debug log file + name: Seqera Platform debug log file path: | - tower_action_*.log - tower_action_*.json + seqera_platform_action_*.log + seqera_platform_action_*.json diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 25726aa1c9..e1c26a71c7 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -5,13 +5,13 @@ name: nf-core AWS test on: workflow_dispatch: jobs: - run-tower: + run-platform: name: Run AWS tests if: github.repository == '{{ name }}' runs-on: ubuntu-latest steps: - # Launch workflow using Tower CLI tool action {%- raw %} - - name: Launch workflow via tower + # Launch workflow using Seqera Platform CLI tool action {%- raw %} + - name: Launch workflow via Seqera Platform uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} @@ -27,7 +27,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: Tower debug log file + name: Seqera Platform debug log file path: | - tower_action_*.log - tower_action_*.json + seqera_platform_action_*.log + seqera_platform_action_*.json diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index a3b141cf38..88e0f1719f 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -20,7 +20,7 @@ [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) -[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/{{ name }}) +[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://cloud.seqera.io/launch?pipeline=https://github.com/{{ name }}) {% endif -%} {%- if branded -%}[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23{{ short_name }}-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/{{ short_name }}){% endif -%} diff --git a/nf_core/utils.py b/nf_core/utils.py index 9261388a39..84b653ffe2 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -877,7 +877,7 @@ def prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False): Args: wf_releases (array): Array of repo releases as returned by the GitHub API wf_branches (array): Array of repo branches, as returned by the GitHub API - multiple (bool): Allow selection of multiple releases & branches (for Tower) + multiple (bool): Allow selection of multiple releases & branches (for Seqera Platform) Returns: choice (str): Selected release / branch name diff --git a/requirements.txt b/requirements.txt index 6b5b3ab57d..c10fe1f706 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -click +click>=7.0 filetype GitPython jinja2 diff --git a/tests/test_cli.py b/tests/test_cli.py index d1dff0fe7b..54e420f5e4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -166,7 +166,7 @@ def test_cli_download(self, mock_dl): "outdir": "/path/outdir", "compress": "tar.gz", "force": None, - "tower": None, + "platform": None, "download-configuration": None, "container-system": "singularity", "container-library": "quay.io", @@ -186,7 +186,7 @@ def test_cli_download(self, mock_dl): params["outdir"], params["compress"], "force" in params, - "tower" in params, + "platform" in params, "download-configuration" in params, params["container-system"], (params["container-library"],), diff --git a/tests/test_download.py b/tests/test_download.py index 621379842b..14a96be26f 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -563,16 +563,16 @@ def test_download_workflow_with_success(self, tmp_dir, mock_download_image, mock download_obj.download_workflow() # - # Test Download for Tower + # Test Download for Seqera Platform # @with_temporary_folder @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") - def test_download_workflow_for_tower(self, tmp_dir, _): + def test_download_workflow_for_platform(self, tmp_dir, _): download_obj = DownloadWorkflow( pipeline="nf-core/rnaseq", revision=("3.7", "3.9"), compress_type="none", - tower=True, + platform=True, container_system="singularity", ) @@ -592,7 +592,7 @@ def test_download_workflow_for_tower(self, tmp_dir, _): download_obj.get_revision_hash() - # download_obj.wf_download_url is not set for tower downloads, but the sha values are + # download_obj.wf_download_url is not set for Seqera Platform downloads, but the sha values are assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 2 assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 @@ -600,7 +600,7 @@ def test_download_workflow_for_tower(self, tmp_dir, _): assert bool(re.search(r"nf-core-rnaseq_\d{4}-\d{2}-\d{1,2}_\d{1,2}-\d{1,2}", download_obj.outdir, re.S)) download_obj.output_filename = f"{download_obj.outdir}.git" - download_obj.download_workflow_tower(location=tmp_dir) + download_obj.download_workflow_platform(location=tmp_dir) assert download_obj.workflow_repo assert isinstance(download_obj.workflow_repo, WorkflowRepo) @@ -614,7 +614,7 @@ def test_download_workflow_for_tower(self, tmp_dir, _): # assert that the download has a "latest" branch. assert "latest" in all_heads - # download_obj.download_workflow_tower(location=tmp_dir) will run container image detection for all requested revisions + # download_obj.download_workflow_platform(location=tmp_dir) will run container image detection for all requested revisions assert isinstance(download_obj.containers, list) and len(download_obj.containers) == 33 assert ( "https://depot.galaxyproject.org/singularity/bbmap:38.93--he522d1c_0" in download_obj.containers From 0680713b167df4a73ad5ad32059694af402da5cf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 18 Mar 2024 11:24:43 +0000 Subject: [PATCH 070/270] [automated] Fix code linting --- nf_core/download.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index f6c12fa49f..9d4decc424 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -308,7 +308,9 @@ def download_workflow_platform(self, location=None): # Justify why compression is skipped for Seqera Platform downloads (Prompt is not shown, but CLI argument could have been set) if self.compress_type is not None: - log.info("Compression choice is ignored for Seqera Platform downloads since nothing can be reasonably compressed.") + log.info( + "Compression choice is ignored for Seqera Platform downloads since nothing can be reasonably compressed." + ) def prompt_pipeline_name(self): """Prompt for the pipeline name if not set with a flag""" From 820510f9abd03c1d958903a0a2088fc4d38a9b51 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 18 Mar 2024 13:00:24 +0000 Subject: [PATCH 071/270] Fix the check for base dir to ensure we check if the supplied dir has a config before we start iterating up the tree --- nf_core/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 9261388a39..8319c8d31b 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1047,12 +1047,13 @@ def load_tools_config(directory: Union[str, Path] = "."): def determine_base_dir(directory="."): base_dir = start_dir = Path(directory).absolute() - while base_dir != base_dir.parent: + # Only iterate up the tree if the start dir doesn't have a config + while not get_first_available_path(base_dir, CONFIG_PATHS) and base_dir != base_dir.parent: base_dir = base_dir.parent config_fn = get_first_available_path(base_dir, CONFIG_PATHS) if config_fn: - return directory if base_dir == start_dir else base_dir - return directory + break + return directory if base_dir == start_dir else base_dir def get_first_available_path(directory, paths): From 4ea8b892392867977a1e0275d3a5fee263201866 Mon Sep 17 00:00:00 2001 From: vickylaram Date: Mon, 18 Mar 2024 15:43:15 +0100 Subject: [PATCH 072/270] Moved CONTRIBUTING.md to root dir --- .github/CONTRIBUTING.md => CONTRIBUTING.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/CONTRIBUTING.md => CONTRIBUTING.md (100%) diff --git a/.github/CONTRIBUTING.md b/CONTRIBUTING.md similarity index 100% rename from .github/CONTRIBUTING.md rename to CONTRIBUTING.md From 1dc076d5ea4373326288126c19fbd46a6e10ab69 Mon Sep 17 00:00:00 2001 From: vickylaram Date: Mon, 18 Mar 2024 15:57:08 +0100 Subject: [PATCH 073/270] Fixed indentation --- nf_core/module-template/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/tests/main.nf.test b/nf_core/module-template/tests/main.nf.test index e1b1dadf12..f31e92d659 100644 --- a/nf_core/module-template/tests/main.nf.test +++ b/nf_core/module-template/tests/main.nf.test @@ -29,7 +29,7 @@ nextflow_process { input[0] = [ [ id:'test', single_end:false ], // meta map file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) - ] + ] {%- else %} input[0] = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) {%- endif %} From 42c467d33cdf2adecf652dcaa359e3a72e947018 Mon Sep 17 00:00:00 2001 From: Matteo Fiandesio Date: Mon, 18 Mar 2024 16:43:41 +0100 Subject: [PATCH 074/270] fix: adds a short sha representation to modules list command pointing to the repo specific commit --- nf_core/components/list.py | 7 ++++--- nf_core/modules/modules_repo.py | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index b0c5af219f..f4407285cd 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -125,10 +125,11 @@ def pattern_msg(keywords: List[str]) -> str: version_sha = component_entry["git_sha"] try: # pass repo_name to get info on modules even outside nf-core/modules - message, date = ModulesRepo( + module = ModulesRepo( remote_url=repo_url, branch=component_entry["branch"], - ).get_commit_info(version_sha) + ) + message, date = module.get_commit_info(version_sha) except LookupError as e: log.warning(e) date = "[red]Not Available" @@ -140,7 +141,7 @@ def pattern_msg(keywords: List[str]) -> str: version_sha = "[red]Not Available" date = "[red]Not Available" message = "[red]Not Available" - table.add_row(component, repo_url, version_sha, message, date) + table.add_row(component, f'[link={repo_url}]{repo_url}[/link]', f'[link={module.gitless_repo()}/commit/{version_sha}]{version_sha[:7]}[/link]', message, date) components.append(component) if print_json: diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 204c20fd71..7f58641aa6 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -69,6 +69,14 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.avail_module_names = None + + def gitless_repo(self): + gitless_repo_url = self.remote_url + if(self.remote_url and '.git' in self.remote_url): + gitless_repo_url = gitless_repo_url[:-3] + return gitless_repo_url + + def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): """ Sets up the local git repository. If the repository has been cloned previously, it From ca5b78b675b46fc474eb1f494b25fd0c288958ed Mon Sep 17 00:00:00 2001 From: Matteo Fiandesio Date: Mon, 18 Mar 2024 17:32:58 +0100 Subject: [PATCH 075/270] adds changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3189edaa..3fb06f4509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ - Patch: handle file not found when it is an added file to a module ([#2771](https://github.com/nf-core/tools/pull/2771)) - Handle symlinks when migrating pytest ([#2770](https://github.com/nf-core/tools/pull/2770)) - Add `--profile` parameter to nf-test command ([#2767](https://github.com/nf-core/tools/pull/2767)) +- Reduce the size of the sha representation in the `list` command and links to the specific commit ([#2870](https://github.com/nf-core/tools/pull/2870)) ### General From 079974bff6d2ea597942e2f24e1198057f9676bb Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 18 Mar 2024 17:57:54 +0100 Subject: [PATCH 076/270] revert click pinning --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c10fe1f706..6b5b3ab57d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -click>=7.0 +click filetype GitPython jinja2 From 171914b538c1b746454bf2bac6e1e7651f8d3a37 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 18 Mar 2024 21:04:57 +0000 Subject: [PATCH 077/270] [automated] Fix code linting --- nf_core/components/list.py | 8 +++++++- nf_core/modules/modules_repo.py | 4 +--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index f4407285cd..1606cfe74a 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -141,7 +141,13 @@ def pattern_msg(keywords: List[str]) -> str: version_sha = "[red]Not Available" date = "[red]Not Available" message = "[red]Not Available" - table.add_row(component, f'[link={repo_url}]{repo_url}[/link]', f'[link={module.gitless_repo()}/commit/{version_sha}]{version_sha[:7]}[/link]', message, date) + table.add_row( + component, + f"[link={repo_url}]{repo_url}[/link]", + f"[link={module.gitless_repo()}/commit/{version_sha}]{version_sha[:7]}[/link]", + message, + date, + ) components.append(component) if print_json: diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 7f58641aa6..839f2a75ee 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -69,14 +69,12 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.avail_module_names = None - def gitless_repo(self): gitless_repo_url = self.remote_url - if(self.remote_url and '.git' in self.remote_url): + if self.remote_url and ".git" in self.remote_url: gitless_repo_url = gitless_repo_url[:-3] return gitless_repo_url - def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): """ Sets up the local git repository. If the repository has been cloned previously, it From 3b78154e8ac7e28efaeca05ca785bb9e3b5aa4d9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 18 Mar 2024 22:10:52 +0100 Subject: [PATCH 078/270] Fix typos --- README.md | 8 ++++---- nf_core/__main__.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5869ad9ac1..53e9f77c97 100644 --- a/README.md +++ b/README.md @@ -376,7 +376,7 @@ The pipeline files are automatically updated (`params.custom_config_base` is set So using `-profile ` should work if available within [nf-core/configs](https://github.com/nf-core/configs). > [!WARNING] -> This option is not available when downloading a pipeline for use with [Sqerea Platform](#adapting-downloads-to-seqera-platform) because the application manages all configurations separately. +> This option is not available when downloading a pipeline for use with [Seqera Platform](#adapting-downloads-to-seqera-platform) because the application manages all configurations separately. ### Downloading Apptainer containers @@ -430,12 +430,12 @@ If the download speeds are much slower than your internet connection is capable ### Adapting downloads to Seqera Platform -[Seqera Platform](https://seqera.io/platform/) (formerly _"Nextflow Tower"_) provides a graphical user interface to oversee pipeline runs, gather statistics and configure compute resources. While pipelines added to _Seqerea Platform_ are preferably hosted at a Git service, providing them as disconnected, self-reliant repositories is also possible for premises with restricted network access. Choosing the `--platform` flag will download the pipeline in an appropriate form. +[Seqera Platform](https://seqera.io/platform/) (formerly _"Nextflow Tower"_) provides a graphical user interface to oversee pipeline runs, gather statistics and configure compute resources. While pipelines added to _Seqera Platform_ are preferably hosted at a Git service, providing them as disconnected, self-reliant repositories is also possible for premises with restricted network access. Choosing the `--platform` flag will download the pipeline in an appropriate form. -Subsequently, the `*.git` folder can be moved to it's final destination and linked with a pipeline in _Seqerea Platform_ using the `file:/` prefix. +Subsequently, the `*.git` folder can be moved to it's final destination and linked with a pipeline in _Seqera Platform_ using the `file:/` prefix. > [!TIP] -> Also without access to Seqerea Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. +> Also without access to Seqera Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. ## Pipeline software licences diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 082e5b8d71..20cba43e4e 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -358,6 +358,7 @@ def create_params_file(pipeline, revision, output, force, show_hidden): help="Archive compression type", ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") +# TODO: Remove this in a future release. Deprecated in March 2024. @click.option( "--tower", is_flag=True, From 705816cf91ee12fe66d2d752e0a12ec07d9231ab Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 18 Mar 2024 22:15:25 +0100 Subject: [PATCH 079/270] Log debug instead of info for module input/outputs Prevents breaking the progress bar when running linting --- nf_core/components/nfcore_component.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index d9731ba7c9..bcba068afe 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -164,7 +164,7 @@ def get_inputs_from_main_nf(self): # path foo # don't match anything inside comments or after "output:" if "input:" not in data: - log.info(f"Could not find any inputs in {self.main_nf}") + log.debug(f"Could not find any inputs in {self.main_nf}") return inputs input_data = data.split("input:")[1].split("output:")[0] regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" @@ -176,7 +176,7 @@ def get_inputs_from_main_nf(self): elif match.group(4): input_val = match.group(4).split(",")[0] # handle `files, stageAs: "inputs/*"` cases inputs.append(input_val) - log.info(f"Found {len(inputs)} inputs in {self.main_nf}") + log.debug(f"Found {len(inputs)} inputs in {self.main_nf}") self.inputs = inputs def get_outputs_from_main_nf(self): @@ -185,12 +185,12 @@ def get_outputs_from_main_nf(self): data = f.read() # get output values from main.nf after "output:". the names are always after "emit:" if "output:" not in data: - log.info(f"Could not find any outputs in {self.main_nf}") + log.debug(f"Could not find any outputs in {self.main_nf}") return outputs output_data = data.split("output:")[1].split("when:")[0] regex = r"emit:\s*([^)\s,]+)" matches = re.finditer(regex, output_data, re.MULTILINE) for _, match in enumerate(matches, start=1): outputs.append(match.group(1)) - log.info(f"Found {len(outputs)} outputs in {self.main_nf}") + log.debug(f"Found {len(outputs)} outputs in {self.main_nf}") self.outputs = outputs From afaf2ff826eb039a7751d7e29b90a52b6c07f4b9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 18 Mar 2024 22:43:32 +0100 Subject: [PATCH 080/270] Hyperlink module name and file path in lint results --- nf_core/components/lint/__init__.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index c99934bca3..ee1bd58b15 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -11,6 +11,7 @@ import rich.box import rich.console import rich.panel +import rich.repr from rich.markdown import Markdown from rich.table import Table @@ -31,6 +32,7 @@ class LintExceptionError(Exception): pass +@rich.repr.auto class LintResult: """An object to hold the results of a lint test""" @@ -42,6 +44,7 @@ def __init__(self, component, lint_test, message, file_path): self.component_name = component.component_name +@rich.repr.auto class ComponentLint(ComponentCommand): """ An object for linting modules and subworkflows either in a clone of the 'nf-core/modules' @@ -231,9 +234,20 @@ def format_result(test_results, table): if last_modname and lint_result.component_name != last_modname: even_row = not even_row last_modname = lint_result.component_name + + # If this is an nf-core module, link to the nf-core webpage + if lint_result.component.repo_url == "https://github.com/nf-core/modules.git": + module_name = f"[link=https://nf-co.re/modules/{lint_result.component_name.replace("/","_")}]{lint_result.component_name}[/link]" + else: + module_name = lint_result.component_name + + # Make the filename clickable to open in VSCode + file_path = os.path.relpath(lint_result.file_path, self.dir) + file_path_link = f"[link=vscode://file/{os.path.abspath(file_path)}]{file_path}[/link]" + table.add_row( - Markdown(f"{lint_result.component_name}"), - os.path.relpath(lint_result.file_path, self.dir), + module_name, + file_path_link, Markdown(f"{lint_result.message}"), style="dim" if even_row else None, ) From fba35785979eb713480f81446fff231ab142d104 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 18 Mar 2024 22:51:06 +0100 Subject: [PATCH 081/270] Fix link and shorten GitHub repo name in nf-core modules list local --- nf_core/components/list.py | 5 +++-- nf_core/modules/modules_repo.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index 1606cfe74a..f5f2744e17 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -70,7 +70,7 @@ def pattern_msg(keywords: List[str]) -> str: # We have a pipeline - list what's installed else: # Check that we are in a pipeline directory - print(f"{self.repo_type=}") + log.info(f"Repository type: [blue]{self.repo_type}") try: if self.repo_type != "pipeline": raise UserWarning( @@ -141,9 +141,10 @@ def pattern_msg(keywords: List[str]) -> str: version_sha = "[red]Not Available" date = "[red]Not Available" message = "[red]Not Available" + nice_repo_name = repo_url.replace("https://github.com/", "").replace(".git", "") table.add_row( component, - f"[link={repo_url}]{repo_url}[/link]", + f"[link={repo_url}]{nice_repo_name}[/link]", f"[link={module.gitless_repo()}/commit/{version_sha}]{version_sha[:7]}[/link]", message, date, diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 839f2a75ee..9694920274 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -72,7 +72,7 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa def gitless_repo(self): gitless_repo_url = self.remote_url if self.remote_url and ".git" in self.remote_url: - gitless_repo_url = gitless_repo_url[:-3] + gitless_repo_url = gitless_repo_url[:-4] return gitless_repo_url def setup_local_repo(self, remote, branch, hide_progress=True, in_cache=False): From 554d5d719b5851a017b38e601ab7039625d04fce Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 18 Mar 2024 22:52:50 +0100 Subject: [PATCH 082/270] extend changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e51597e89c..1e84fc8043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,7 +79,8 @@ - Patch: handle file not found when it is an added file to a module ([#2771](https://github.com/nf-core/tools/pull/2771)) - Handle symlinks when migrating pytest ([#2770](https://github.com/nf-core/tools/pull/2770)) - Add `--profile` parameter to nf-test command ([#2767](https://github.com/nf-core/tools/pull/2767)) -- Reduce the size of the sha representation in the `list` command and links to the specific commit ([#2870](https://github.com/nf-core/tools/pull/2870)) +- Reduce the sha length in the `nf-core modules list local` and add links to the specific commit ([#2870](https://github.com/nf-core/tools/pull/2870)) +- Add links the nf-core module page and to open the local file in VSCode for module lint results ([#2870](https://github.com/nf-core/tools/pull/2870)) ### General From 54afd8d54c8523715dd3e9475574407a903f46d1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 18 Mar 2024 23:00:50 +0100 Subject: [PATCH 083/270] Try to fix linting --- nf_core/components/lint/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index ee1bd58b15..564dcfaf6b 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -237,7 +237,8 @@ def format_result(test_results, table): # If this is an nf-core module, link to the nf-core webpage if lint_result.component.repo_url == "https://github.com/nf-core/modules.git": - module_name = f"[link=https://nf-co.re/modules/{lint_result.component_name.replace("/","_")}]{lint_result.component_name}[/link]" + module_url = "https://nf-co.re/modules/" + lint_result.component_name.replace("/", "_") + module_name = f"[link={module_url}]{lint_result.component_name}[/link]" else: module_name = lint_result.component_name From 68ab4feb415843f18209f7d02acdad8165f59492 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 19 Mar 2024 10:01:35 +0000 Subject: [PATCH 084/270] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0d37627a..a47fda7270 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ - Update python:3.11-slim Docker digest to a2eb07f ([#2847](https://github.com/nf-core/tools/pull/2847)) - Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 ([#2850](https://github.com/nf-core/tools/pull/2850)) +- Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From c8f3dc0b02d0b33a5c1c8fb8eed9e9034054b710 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Tue, 19 Mar 2024 10:04:10 +0000 Subject: [PATCH 085/270] Add nf_core_version to .nf-core.yml --- nf_core/pipeline-template/.nf-core.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/.nf-core.yml b/nf_core/pipeline-template/.nf-core.yml index 3805dc81c1..e8140bfb12 100644 --- a/nf_core/pipeline-template/.nf-core.yml +++ b/nf_core/pipeline-template/.nf-core.yml @@ -1 +1,2 @@ repository_type: pipeline +nf_core_version: "{{ nf_core_version }}" From c1d9dbd09f918050e14951ade2d8a7354f476512 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Tue, 19 Mar 2024 10:10:49 +0000 Subject: [PATCH 086/270] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e3189edaa..d7b58a0319 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Fix topic extraction step for hashtags in toots ([#2810](https://github.com/nf-core/tools/pull/2810)) - Update modules and subworkflows in the template ([#2811](https://github.com/nf-core/tools/pull/2811)) - Unpin setup-nextflow and action-tower-launch ([#2806](https://github.com/nf-core/tools/pull/2806)) +- Add nf-core-version to `.nf-core.yml` ([#2874](https://github.com/nf-core/tools/pull/2874)) ### Download From 05083f662f7abf9c7a25e82d35a059edcbb2ec67 Mon Sep 17 00:00:00 2001 From: Weronika Sosnowska Date: Tue, 19 Mar 2024 11:41:52 +0100 Subject: [PATCH 087/270] Add type of values to the error description for schema errors --- nf_core/schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index df04dc5a1e..ceb17a2e29 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -807,8 +807,11 @@ def add_schema_found_configs(self): elif p_key in self.schema_defaults and (s_def := self.schema_defaults[p_key]) != ( p_def := self.build_schema_param(p_val).get("default") ): + p_type = self.build_schema_param(p_val).get("type") + s_type = self.schema_types[p_key] if self.no_prompts or Confirm.ask( - f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{s_def}' | config: '{p_def}'). " + f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. " + f"(schema: '{s_def}' {s_type} | config: '{p_def}' {p_type}). " "[blue]Update pipeline schema?" ): s_key_def = s_key + ("default",) From d1a384be88393c74ce6fb00d7314005745987144 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 19 Mar 2024 11:47:37 +0000 Subject: [PATCH 088/270] fix: ensure path object converted to string before stripping quotes --- nf_core/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 373f8bbaa1..ef45b201fc 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -582,7 +582,7 @@ def make_skeleton_schema(self): ) schema_template = env.get_template("nextflow_schema.json") template_vars = { - "name": self.pipeline_manifest.get("name", Path(self.schema_filename).parent).strip("'"), + "name": self.pipeline_manifest.get("name", str(Path(self.schema_filename).parent)).strip("'"), "description": self.pipeline_manifest.get("description", "").strip("'"), } self.schema = json.loads(schema_template.render(template_vars)) From e7f5a48a9204c590159a30ea428d5c8294a9b4f4 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 19 Mar 2024 11:49:27 +0000 Subject: [PATCH 089/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a47fda7270..449edff6ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ - Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 ([#2850](https://github.com/nf-core/tools/pull/2850)) - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) +- fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 7dc7bd7abc2b246f37f5e051d2ced2763d2ac312 Mon Sep 17 00:00:00 2001 From: Weronika Sosnowska Date: Tue, 19 Mar 2024 12:53:14 +0100 Subject: [PATCH 090/270] Detect type from actual value --- nf_core/schema.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 5131650b0e..99a72d7bb5 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -799,11 +799,8 @@ def add_schema_found_configs(self): elif p_key in self.schema_defaults and (s_def := self.schema_defaults[p_key]) != ( p_def := self.build_schema_param(p_val).get("default") ): - p_type = self.build_schema_param(p_val).get("type") - s_type = self.schema_types[p_key] if self.no_prompts or Confirm.ask( - f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. " - f"(schema: '{s_def}' {s_type} | config: '{p_def}' {p_type}). " + f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{s_def}' {type(s_def)} | config: '{p_def}' {type(p_def)}). " "[blue]Update pipeline schema?" ): s_key_def = s_key + ("default",) From 64a164a979898bf98691bab99539bf9fdf03b07c Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:01:46 +0000 Subject: [PATCH 091/270] fix: add word_wrap to Syntax object for console printing to ensure schema docs output is not truncated --- nf_core/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 373f8bbaa1..a1b595d5b0 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -494,7 +494,7 @@ def print_documentation( if not output_fn: console = rich.console.Console() - console.print("\n", Syntax(prettified_docs, format), "\n") + console.print("\n", Syntax(prettified_docs, format, word_wrap=True), "\n") else: if Path(output_fn).exists() and not force: log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") From fa6504b5532320b4acf2006f9aff4c5ade09b8ba Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 19 Mar 2024 12:03:42 +0000 Subject: [PATCH 092/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a47fda7270..a1e72ababd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ - Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 ([#2850](https://github.com/nf-core/tools/pull/2850)) - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) +- Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From b425c96f32de017a65cfda61b8a9e8ceb9706d6e Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Mar 2024 14:27:53 +0100 Subject: [PATCH 093/270] Remove pyproject.toml --- nf_core/pipeline-template/pyproject.toml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 nf_core/pipeline-template/pyproject.toml diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml deleted file mode 100644 index 56110621e7..0000000000 --- a/nf_core/pipeline-template/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff. -# Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. -[tool.ruff] -line-length = 120 -target-version = "py38" -cache-dir = "~/.cache/ruff" - -[tool.ruff.lint] -select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] - -[tool.ruff.lint.isort] -known-first-party = ["nf_core"] - -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["E402", "F401"] From c083f641e3c6fc594afd63ed6ba676ca57f0729b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Mar 2024 15:51:35 +0100 Subject: [PATCH 094/270] Remove pyproject.toml from linting --- nf_core/lint/files_exist.py | 44 ++++++++++++++++----------------- nf_core/lint/files_unchanged.py | 3 +-- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 5d62a23bf8..ae105d8d2c 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -66,31 +66,31 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: conf/igenomes.config .github/workflows/awstest.yml .github/workflows/awsfulltest.yml - pyproject.toml Files that *must not* be present, due to being renamed or removed in the template: .. code-block:: bash - Singularity - parameters.settings.json - pipeline_template.yml # saving information in .nf-core.yml - .nf-core.yaml # NB: Should be yml, not yaml - bin/markdown_to_html.r - conf/aws.config - .github/workflows/push_dockerhub.yml .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/feature_request.md - docs/images/nf-core-PIPELINE_logo.png + .github/workflows/push_dockerhub.yml .markdownlint.yml + .nf-core.yaml # NB: Should be yml, not yaml .yamllint.yml + bin/markdown_to_html.r + conf/aws.config + docs/images/nf-core-PIPELINE_logo.png lib/Checks.groovy lib/Completion.groovy - lib/Workflow.groovy - lib/WorkflowPIPELINE.groovy lib/NfcoreTemplate.groovy lib/Utils.groovy + lib/Workflow.groovy lib/WorkflowMain.groovy + lib/WorkflowPIPELINE.groovy + parameters.settings.json + pipeline_template.yml # saving information in .nf-core.yml + pyproject.toml + Singularity Files that *should not* be present: @@ -176,30 +176,30 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: [Path(".github", "workflows", "awstest.yml")], [Path(".github", "workflows", "awsfulltest.yml")], [Path("modules.json")], - [Path("pyproject.toml")], ] # List of strings. Fails / warns if any of the strings exist. files_fail_ifexists = [ - Path("Singularity"), - Path("parameters.settings.json"), - Path("pipeline_template.yml"), # saving information in .nf-core.yml - Path(".nf-core.yaml"), # yml not yaml - Path("bin", "markdown_to_html.r"), - Path("conf", "aws.config"), - Path(".github", "workflows", "push_dockerhub.yml"), Path(".github", "ISSUE_TEMPLATE", "bug_report.md"), Path(".github", "ISSUE_TEMPLATE", "feature_request.md"), - Path("docs", "images", f"nf-core-{short_name}_logo.png"), + Path(".github", "workflows", "push_dockerhub.yml"), Path(".markdownlint.yml"), + Path(".nf-core.yaml"), # yml not yaml Path(".yamllint.yml"), + Path("bin", "markdown_to_html.r"), + Path("conf", "aws.config"), + Path("docs", "images", f"nf-core-{short_name}_logo.png"), Path("lib", "Checks.groovy"), Path("lib", "Completion.groovy"), - Path("lib", "Workflow.groovy"), + Path("lib", "NfcoreTemplate.groovy"), Path("lib", "Utils.groovy"), + Path("lib", "Workflow.groovy"), Path("lib", "WorkflowMain.groovy"), - Path("lib", "NfcoreTemplate.groovy"), Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy"), + Path("parameters.settings.json"), + Path("pipeline_template.yml"), # saving information in .nf-core.yml + Path("pyproject.toml"), + Path("Singularity"), ] files_warn_ifexists = [Path(".travis.yml")] files_fail_ifinconfig: List[Tuple[Path, Dict[str, str]]] = [ diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 3a3a0cb74a..1cd1f7fdb1 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -47,7 +47,6 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: .gitignore .prettierignore - pyproject.toml .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting @@ -106,7 +105,7 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: [Path("docs", "README.md")], ] files_partial = [ - [Path(".gitignore"), Path(".prettierignore"), Path("pyproject.toml")], + [Path(".gitignore"), Path(".prettierignore")], ] # Only show error messages from pipeline creation From 1dbc1302916cd2b23b7bd79e3601a21bf7f6feb2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 19 Mar 2024 15:52:17 +0100 Subject: [PATCH 095/270] Add to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b31cc30ec..e6ab87636d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Set topic variable correctly in the mastodon announcement ([#2848](https://github.com/nf-core/tools/pull/2848)) - Add a cleanup action to `download_pipeline.yml` to fix failures caused by inadequate storage space on the runner ([#2849](https://github.com/nf-core/tools/pull/2849)) - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) +- Remove `pyproject.toml` from template root ### Linting From d82d28ec457aa21edaf9abe6b1293cf6e56eb8bd Mon Sep 17 00:00:00 2001 From: leomrtns Date: Tue, 19 Mar 2024 15:30:47 +0000 Subject: [PATCH 096/270] error message if wrong GITHUB_TOKEN leads to 401 --- nf_core/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/utils.py b/nf_core/utils.py index e1778b55b3..3feeae7824 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -453,6 +453,7 @@ def __init__(self): # pylint: disable=super-init-not-called self.auth_mode = None self.return_ok = [200, 201] self.return_retry = [403] + self.return_unauthorised = [401] self.has_init = False def lazy_init(self): @@ -545,6 +546,8 @@ def safe_get(self, url): raise e else: return r + elif request.status_code in self.return_unauthorised: + raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN") return request From 738323b86969e24602538c894fa02fc3aeb0adae Mon Sep 17 00:00:00 2001 From: leomrtns Date: Tue, 19 Mar 2024 16:05:07 +0000 Subject: [PATCH 097/270] merged with dev instead of master; ruff formatted; changelog updated --- CHANGELOG.md | 1 + nf_core/utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b31cc30ec..94cc5cb407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ ### Download - Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. +- Better error message when GITHUB_TOKEN exists but is wrong/outdated ### Components diff --git a/nf_core/utils.py b/nf_core/utils.py index 5307363ce3..2c3f954f99 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -548,7 +548,7 @@ def safe_get(self, url): else: return r elif request.status_code in self.return_unauthorised: - raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN") + raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN") return request From 81d55097df2a38eb19abd83de33ab378f398f5d9 Mon Sep 17 00:00:00 2001 From: leomrtns Date: Tue, 19 Mar 2024 16:21:23 +0000 Subject: [PATCH 098/270] suggested change by Matthias --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 2c3f954f99..461f5a9bad 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -548,7 +548,7 @@ def safe_get(self, url): else: return r elif request.status_code in self.return_unauthorised: - raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN") + raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") return request From 63d5de8fe69b2ede6eb63a5693d65747c47a37a4 Mon Sep 17 00:00:00 2001 From: Emilio Palumbo Date: Tue, 19 Mar 2024 18:07:30 +0100 Subject: [PATCH 099/270] Rename main.nf.test template for modules and subworkflows --- nf_core/components/create.py | 2 +- nf_core/module-template/tests/{main.nf.test => main.nf.test.j2} | 0 .../tests/{main.nf.test => main.nf.test.j2} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename nf_core/module-template/tests/{main.nf.test => main.nf.test.j2} (100%) rename nf_core/subworkflow-template/tests/{main.nf.test => main.nf.test.j2} (100%) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index d2169e3a72..6c9c01b496 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -394,7 +394,7 @@ def _get_component_dirs(self): if self.component_type == "modules": file_paths["environment.yml"] = component_dir / "environment.yml" file_paths["tests/tags.yml"] = component_dir / "tests" / "tags.yml" - file_paths["tests/main.nf.test"] = component_dir / "tests" / "main.nf.test" + file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" return file_paths diff --git a/nf_core/module-template/tests/main.nf.test b/nf_core/module-template/tests/main.nf.test.j2 similarity index 100% rename from nf_core/module-template/tests/main.nf.test rename to nf_core/module-template/tests/main.nf.test.j2 diff --git a/nf_core/subworkflow-template/tests/main.nf.test b/nf_core/subworkflow-template/tests/main.nf.test.j2 similarity index 100% rename from nf_core/subworkflow-template/tests/main.nf.test rename to nf_core/subworkflow-template/tests/main.nf.test.j2 From 7364a536661b1f3d2053fb42c6df0755e2124fb0 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 08:48:57 +0000 Subject: [PATCH 100/270] Add version check in snapshot file --- nf_core/modules/lint/module_tests.py | 16 ++++++++++++++++ nf_core/subworkflows/lint/subworkflow_tests.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index b1a611d70a..b227f684bd 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -107,6 +107,22 @@ def module_tests(_, module: NFCoreComponent): snap_file, ) ) + if "versions" in str(snap_content[test_name]): + module.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, + ) + ) + else: + module.failed.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, + ) + ) except json.decoder.JSONDecodeError as e: module.failed.append( ( diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 796a56d018..0c0589a1e2 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -114,6 +114,22 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): snap_file, ) ) + if "versions" in str(snap_content[test_name]): + subworkflow.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, + ) + ) + else: + subworkflow.warned.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, + ) + ) except json.decoder.JSONDecodeError as e: subworkflow.failed.append( ( From 8a5c964f2b9e9b407799bbe7981e7771b5d25e5f Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 20 Mar 2024 09:00:15 +0000 Subject: [PATCH 101/270] [automated] Fix code linting --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 461f5a9bad..4271e971a1 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -548,7 +548,7 @@ def safe_get(self, url): else: return r elif request.status_code in self.return_unauthorised: - raise RuntimeError(f"GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") + raise RuntimeError("GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") return request From eccf0db977aba848d4b4e77d8075fe0cee2287cd Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 09:29:51 +0000 Subject: [PATCH 102/270] update changelog.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94cc5cb407..9177060b09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### Linting - Only match assignments of params in `main.nf` and not references like `params.aligner == ` ([#2833](https://github.com/nf-core/tools/pull/2833)) +- Include test for presence of versions in snapshot ([#2888](https://github.com/nf-core/tools/pull/2888)) ### Download From 44bf8005cba0c76ded34062a698ae7a55207aee7 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 10:27:32 +0000 Subject: [PATCH 103/270] adding pytests --- tests/modules/lint.py | 21 +++++++++++++++++++++ tests/subworkflows/lint.py | 19 +++++++++++++++++++ tests/test_modules.py | 1 + tests/test_subworkflows.py | 1 + 4 files changed, 42 insertions(+) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 9bd280ddd8..5e3bcfbd30 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -642,3 +642,24 @@ def test_nftest_failing_linting(self): assert module_lint.failed[2].lint_test == "test_main_tags" assert "kallisto/index" in module_lint.failed[2].message assert module_lint.failed[3].lint_test == "test_tags_yml" + + +def test_modules_absent_version(self): + """Test linting a nf-test module if the versions is absent in the snapshot file `""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap")) as fh: + content = fh.read() + new_content = content.replace("versions", "foo") + with open( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap"), "w" + ) as fh: + fh.write(new_content) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + with open( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap"), "w" + ) as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_snap_versions" diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index b89b7b78ce..7da5f1640f 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -181,3 +181,22 @@ def test_subworkflows_lint_capitalization_fail(self): # cleanup self.subworkflow_remove.remove("bam_stats_samtools", force=True) + + +def test_subworkflows_absent_version(self): + """Test linting a nf-test module if the versions is absent in the snapshot file `""" + self.subworkflow_install.install("fastq_align_dna") + with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_dna", "test", "main.nf.test.snap")) as fh: + content = fh.read() + new_content = content.replace("versions", "foo") + with open( + Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_dna", "test", "main.nf.test.snap"), "w" + ) as fh: + fh.write(new_content) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, subworkflow="fastq_align_dna") + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert any([x.lint_test == "test_snap_versions" for x in subworkflow_lint.warned]) diff --git a/tests/test_modules.py b/tests/test_modules.py index 944a09f670..da336aa67a 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -179,6 +179,7 @@ def test_modulesrepo_class(self): test_modules_install_trimgalore_twice, ) from .modules.lint import ( # type: ignore[misc] + test_modules_absent_version, test_modules_environment_yml_file_doesnt_exists, test_modules_environment_yml_file_name_mismatch, test_modules_environment_yml_file_not_array, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 6163faa7a9..775a08547f 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -119,6 +119,7 @@ def tearDown(self): test_subworkflows_install_tracking_added_super_subworkflow, ) from .subworkflows.lint import ( # type: ignore[misc] + test_subworkflows_absent_version, test_subworkflows_lint, test_subworkflows_lint_capitalization_fail, test_subworkflows_lint_empty, From c77cfc87613f197d14048ee964b0273401f47bbc Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 11:04:09 +0000 Subject: [PATCH 104/270] adding additional logic for split up snapshots --- nf_core/modules/lint/module_tests.py | 2 +- nf_core/subworkflows/lint/subworkflow_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index b227f684bd..52cbaf04e5 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -107,7 +107,7 @@ def module_tests(_, module: NFCoreComponent): snap_file, ) ) - if "versions" in str(snap_content[test_name]): + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): module.passed.append( ( "test_snap_versions", diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 0c0589a1e2..30d4296aac 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -114,7 +114,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): snap_file, ) ) - if "versions" in str(snap_content[test_name]): + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): subworkflow.passed.append( ( "test_snap_versions", From 099c01dbfd4ff86c1a1c8235067e07d5ca0292d9 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 11:41:57 +0000 Subject: [PATCH 105/270] setup .nf-core.yml lint --- nf_core/lint/nfcore_yml.py | 61 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 nf_core/lint/nfcore_yml.py diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py new file mode 100644 index 0000000000..4b165c78e4 --- /dev/null +++ b/nf_core/lint/nfcore_yml.py @@ -0,0 +1,61 @@ +def nfcore_yml(self): + """Repository ``.nf-core.yml`` tests + + The ``.nf-core.yml`` contains metadata for nf-core tools to correctly apply its features. + + * repository type: + + * Check that the repository type is set. + + * nf core version: + + * Check if the nf-core version is set to the latest version. + + """ + passed = [] + warned = [] + failed = [] + + # Remove field that should be ignored according to the linting config + # ignore_configs = self.lint_config.get(".nf-core", []) + + # with open(os.path.join(self.wf_path, ".nf-core.yml")) as fh: + # content = fh.read() + + # if "nextflow_badge" not in ignore_configs: + # # Check that there is a readme badge showing the minimum required version of Nextflow + # # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) + # # and that it has the correct version + # nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" + # match = re.search(nf_badge_re, content) + # if match: + # nf_badge_version = match.group(1).strip("'\"") + # try: + # if nf_badge_version != self.minNextflowVersion: + # raise AssertionError() + # except (AssertionError, KeyError): + # failed.append( + # f"README Nextflow minimum version badge does not match config. Badge: `{nf_badge_version}`, " + # f"Config: `{self.minNextflowVersion}`" + # ) + # else: + # passed.append( + # f"README Nextflow minimum version badge matched config. Badge: `{nf_badge_version}`, " + # f"Config: `{self.minNextflowVersion}`" + # ) + # else: + # warned.append("README did not have a Nextflow minimum version badge.") + + # if "zenodo_doi" not in ignore_configs: + # # Check that zenodo.XXXXXXX has been replaced with the zendo.DOI + # zenodo_re = r"/zenodo\.X+" + # match = re.search(zenodo_re, content) + # if match: + # warned.append( + # "README contains the placeholder `zenodo.XXXXXXX`. " + # "This should be replaced with the zenodo doi (after the first release)." + # ) + # else: + # passed.append("README Zenodo placeholder was replaced with DOI.") + + return {"passed": passed, "warned": warned, "failed": failed} From 2ee4a204f0bcb4ee12aaef73385c7a075edf7410 Mon Sep 17 00:00:00 2001 From: Weronika Sosnowska Date: Wed, 20 Mar 2024 12:49:44 +0100 Subject: [PATCH 106/270] Apply suggestion from review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 8aa863a4ee..4096c80f93 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -800,7 +800,7 @@ def add_schema_found_configs(self): p_def := self.build_schema_param(p_val).get("default") ): if self.no_prompts or Confirm.ask( - f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{s_def}' {type(s_def)} | config: '{p_def}' {type(p_def)}). " + f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{type(s_def)}: {s_def}' | config: '{type(p_def)}: {p_def}'). " "[blue]Update pipeline schema?" ): s_key_def = s_key + ("default",) From 07bb48295e9967da5ef59ac7f81906e1bde16bbb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 03:07:27 +0000 Subject: [PATCH 107/270] Update dawidd6/action-download-artifact digest to 09f2f74 --- nf_core/pipeline-template/.github/workflows/linting_comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 67ef7b534d..ea408fd6f8 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@a430ac5786b39ad5869da25a98130624d2ce340c # v3 + uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3 with: workflow: linting.yml workflow_conclusion: completed From 353fc04e90591bf79a22b07f12716c4c974e8443 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 13:26:43 +0000 Subject: [PATCH 108/270] Add nfcore_yml linting module*** --- nf_core/lint/__init__.py | 2 + nf_core/lint/nfcore_yml.py | 93 ++++++++++++++++++++++---------------- tests/lint/nfcore_yml.py | 53 ++++++++++++++++++++++ tests/test_lint.py | 5 ++ 4 files changed, 113 insertions(+), 40 deletions(-) create mode 100644 tests/lint/nfcore_yml.py diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index be9ac183a6..a14ac15691 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -206,6 +206,7 @@ class PipelineLint(nf_core.utils.Pipeline): from .modules_structure import modules_structure # type: ignore[misc] from .multiqc_config import multiqc_config # type: ignore[misc] from .nextflow_config import nextflow_config # type: ignore[misc] + from .nfcore_yml import nfcore_yml # type: ignore[misc] from .pipeline_name_conventions import ( # type: ignore[misc] pipeline_name_conventions, ) @@ -264,6 +265,7 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "nfcore_yml", ] + (["version_consistency"] if release_mode else []) def _load(self): diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index 4b165c78e4..6d9f664871 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -1,3 +1,11 @@ +import os +import re + +from nf_core import __version__ + +REPOSITORY_TYPES = ["pipeline", "modules"] + + def nfcore_yml(self): """Repository ``.nf-core.yml`` tests @@ -15,47 +23,52 @@ def nfcore_yml(self): passed = [] warned = [] failed = [] + ignored = [] # Remove field that should be ignored according to the linting config - # ignore_configs = self.lint_config.get(".nf-core", []) - - # with open(os.path.join(self.wf_path, ".nf-core.yml")) as fh: - # content = fh.read() - - # if "nextflow_badge" not in ignore_configs: - # # Check that there is a readme badge showing the minimum required version of Nextflow - # # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) - # # and that it has the correct version - # nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" - # match = re.search(nf_badge_re, content) - # if match: - # nf_badge_version = match.group(1).strip("'\"") - # try: - # if nf_badge_version != self.minNextflowVersion: - # raise AssertionError() - # except (AssertionError, KeyError): - # failed.append( - # f"README Nextflow minimum version badge does not match config. Badge: `{nf_badge_version}`, " - # f"Config: `{self.minNextflowVersion}`" - # ) - # else: - # passed.append( - # f"README Nextflow minimum version badge matched config. Badge: `{nf_badge_version}`, " - # f"Config: `{self.minNextflowVersion}`" - # ) - # else: - # warned.append("README did not have a Nextflow minimum version badge.") - - # if "zenodo_doi" not in ignore_configs: - # # Check that zenodo.XXXXXXX has been replaced with the zendo.DOI - # zenodo_re = r"/zenodo\.X+" - # match = re.search(zenodo_re, content) - # if match: - # warned.append( - # "README contains the placeholder `zenodo.XXXXXXX`. " - # "This should be replaced with the zenodo doi (after the first release)." - # ) - # else: - # passed.append("README Zenodo placeholder was replaced with DOI.") + ignore_configs = self.lint_config.get(".nf-core", []) + + try: + with open(os.path.join(self.wf_path, ".nf-core.yml")) as fh: + content = fh.read() + except FileNotFoundError: + with open(os.path.join(self.wf_path, ".nf-core.yaml")) as fh: + content = fh.read() + + if "repository_type" not in ignore_configs: + # Check that the repository type is set in the .nf-core.yml + repo_type_re = r"repository_type: (.+)" + match = re.search(repo_type_re, content) + if match: + repo_type = match.group(1) + if repo_type not in REPOSITORY_TYPES: + failed.append( + f"Repository type in .nf-core.yml is not valid. " + f"Should be one of {', '.join(REPOSITORY_TYPES)} but was {repo_type}" + ) + else: + passed.append(f"Repository type in .nf-core.yml is valid: {repo_type}") + else: + warned.append("Repository type not set in .nf-core.yml") + else: + ignored.append(".nf-core.yml variable ignored 'repository_type'") + + if "nf_core_version" not in ignore_configs: + # Check that the nf-core version is set in the .nf-core.yml + nf_core_version_re = r"nf_core_version: (.+)" + match = re.search(nf_core_version_re, content) + if match: + nf_core_version = match.group(1) + if nf_core_version != __version__ and "dev" not in __version__: + warned.append( + f"nf-core version in .nf-core.yml is not set to the latest version. " + f"Should be {__version__} but was {nf_core_version}" + ) + else: + passed.append(f"nf-core version in .nf-core.yml is set to the latest version: {nf_core_version}") + else: + warned.append("nf-core version not set in .nf-core.yml") + else: + ignored.append(".nf-core.yml variable ignored 'nf_core_version'") return {"passed": passed, "warned": warned, "failed": failed} diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py new file mode 100644 index 0000000000..ea5fa8dee5 --- /dev/null +++ b/tests/lint/nfcore_yml.py @@ -0,0 +1,53 @@ +import re +from pathlib import Path + +import nf_core.create +import nf_core.lint + + +def test_nfcore_yml_pass(self): + """Lint test: nfcore_yml - PASS""" + self.lint_obj._load() + results = self.lint_obj.nfcore_yml() + + assert "Repository type in .nf-core.yml is valid" in str(results["passed"]) + assert "nf-core version in .nf-core.yml is set to the latest version" in str(results["passed"]) + assert len(results.get("warned", [])) == 0 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 + + +def test_nfcore_yml_fail_repo_type(self): + """Lint test: nfcore_yml - FAIL - repository type not set""" + new_pipeline = self._make_pipeline_copy() + nf_core_yml = Path(new_pipeline) / ".nf-core.yml" + with open(nf_core_yml) as fh: + content = fh.read() + new_content = content.replace("repository_type: pipeline", "repository_type: foo") + with open(nf_core_yml, "w") as fh: + fh.write(new_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + results = lint_obj.nfcore_yml() + assert "Repository type in .nf-core.yml is not valid." in str(results["failed"]) + assert len(results.get("warned", [])) == 0 + assert len(results.get("passed", [])) >= 0 + assert len(results.get("ignored", [])) == 0 + + +def test_nfcore_yml_fail_nfcore_version(self): + """Lint test: nfcore_yml - FAIL - nf-core version not set""" + new_pipeline = self._make_pipeline_copy() + nf_core_yml = Path(new_pipeline) / ".nf-core.yml" + with open(nf_core_yml) as fh: + content = fh.read() + new_content = (re.sub(r"nf-core_version: [\s]*", "nf-core_version: foo", content),) + with open(nf_core_yml, "w") as fh: + fh.write(new_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + results = lint_obj.nfcore_yml() + assert "nf-core version in .nf-core.yml is not set to the latest version." in str(results["warned"]) + assert len(results.get("failed", [])) == 0 + assert len(results.get("passed", [])) >= 0 + assert len(results.get("ignored", [])) == 0 diff --git a/tests/test_lint.py b/tests/test_lint.py index d10cef37e4..0d767dc1db 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -233,6 +233,11 @@ def test_sphinx_md_files(self): test_nextflow_config_example_pass, test_nextflow_config_missing_test_profile_failed, ) + from .lint.nfcore_yml import ( # type: ignore[misc] + test_nfcore_yml_fail_nfcore_version, + test_nfcore_yml_fail_repo_type, + test_nfcore_yml_pass, + ) from .lint.template_strings import ( # type: ignore[misc] test_template_strings, test_template_strings_ignore_file, From 907df3bd8d95affb5fc01d90b31c7f45dd1b8582 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 13:47:36 +0000 Subject: [PATCH 109/270] Update nf-core_version in .nf-core.yml file --- tests/lint/nfcore_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py index ea5fa8dee5..4c36d27c25 100644 --- a/tests/lint/nfcore_yml.py +++ b/tests/lint/nfcore_yml.py @@ -41,7 +41,7 @@ def test_nfcore_yml_fail_nfcore_version(self): nf_core_yml = Path(new_pipeline) / ".nf-core.yml" with open(nf_core_yml) as fh: content = fh.read() - new_content = (re.sub(r"nf-core_version: [\s]*", "nf-core_version: foo", content),) + new_content = re.sub(r"nf-core_version: [\s]*", "nf-core_version: foo", content) with open(nf_core_yml, "w") as fh: fh.write(new_content) lint_obj = nf_core.lint.PipelineLint(new_pipeline) From f6561ee78086de631f80fe9a240169fa8cee53eb Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 14:00:16 +0000 Subject: [PATCH 110/270] Fix linting test for subworkflows --- tests/subworkflows/lint.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 7da5f1640f..84ed6d4234 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -185,18 +185,20 @@ def test_subworkflows_lint_capitalization_fail(self): def test_subworkflows_absent_version(self): """Test linting a nf-test module if the versions is absent in the snapshot file `""" - self.subworkflow_install.install("fastq_align_dna") - with open(Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_dna", "test", "main.nf.test.snap")) as fh: + self.subworkflow_install.install("bam_sort_stats_samtools") + with open( + Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_sort_stats_samtools", "test", "main.nf.test.snap") + ) as fh: content = fh.read() new_content = content.replace("versions", "foo") with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_dna", "test", "main.nf.test.snap"), "w" + Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_sort_stats_samtools", "test", "main.nf.test.snap"), "w" ) as fh: fh.write(new_content) subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, subworkflow="fastq_align_dna") + subworkflow_lint.lint(print_results=False, subworkflow="bam_sort_stats_samtools") assert len(subworkflow_lint.failed) == 0 assert len(subworkflow_lint.passed) > 0 - assert len(subworkflow_lint.warned) >= 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" assert any([x.lint_test == "test_snap_versions" for x in subworkflow_lint.warned]) From 3b2aaa9d6c6fe0464175cc52b0df3eb2094cd8ae Mon Sep 17 00:00:00 2001 From: Pieter Moris <13552343+pmoris@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:06:16 +0100 Subject: [PATCH 111/270] Make cli-provided module/subworkflow names case insensitive Uses a click callback function to normalize the name of a module/subworkflow that is provided by the user on the CLI, so that names written in uppercase still resolve to the expected lowercase name. In case no argument is provided, the callback function simply returns None, and thus the current behaviour of a user being asked to select a component via an interactive prompt is maintained. The .casefold() method is used in favour of .lower() for slightly more robust case normalization. See: https://stackoverflow.com/questions/45745661/lower-vs-casefold-in-string-matching-and-converting-to-lowercase The callback is applied to all occurrences of "tool/module/subworkflow" arguments, except those in the the "create" functions, because these already contain an internal check that prompts to user to adhere to the lowercase naming convention. Updates to CHANGELOG.md. Co-authored-by: Adam Talbot Co-authored-by: Phil Ewels Co-authored-by: Arthur Gymer <24782660+awgymer@users.noreply.github.com> --- CHANGELOG.md | 1 + nf_core/__main__.py | 36 ++++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94cc5cb407..d8e740d2af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) +- Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 20cba43e4e..dc17c6e6c1 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -103,6 +103,14 @@ def selective_traceback_hook(exctype, value, traceback): sys.excepthook = selective_traceback_hook +# Define callback function to normalize the case of click arguments, +# which is used to make the module/subworkflow names, provided by the +# user on the cli, case insensitive. +def normalize_case(ctx, param, component_name): + if component_name is not None: + return component_name.casefold() + + def run_nf_core(): # print nf-core header if environment variable is not set if os.environ.get("_NF_CORE_COMPLETE") is None: @@ -786,7 +794,7 @@ def modules_list_local(ctx, keywords, json, dir): # pylint: disable=redefined-b # nf-core modules install @modules.command("install") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -838,7 +846,7 @@ def modules_install(ctx, tool, dir, prompt, force, sha): # nf-core modules update @modules.command("update") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -930,7 +938,7 @@ def modules_update( # nf-core modules patch @modules.command() @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -967,7 +975,7 @@ def patch(ctx, tool, dir, remove): # nf-core modules remove @modules.command("remove") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1121,7 +1129,7 @@ def create_module( # nf-core modules test @modules.command("test") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1180,7 +1188,7 @@ def test_module(ctx, tool, dir, no_prompts, update, once, profile): # nf-core modules lint @modules.command("lint") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1267,7 +1275,7 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, # nf-core modules info @modules.command("info") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1306,7 +1314,7 @@ def modules_info(ctx, tool, dir): # nf-core modules bump-versions @modules.command() @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1392,7 +1400,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): # nf-core subworkflows test @subworkflows.command("test") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1519,7 +1527,7 @@ def subworkflows_list_local(ctx, keywords, json, dir): # pylint: disable=redefi # nf-core subworkflows lint @subworkflows.command("lint") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1600,7 +1608,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo # nf-core subworkflows info @subworkflows.command("info") @click.pass_context -@click.argument("tool", type=str, required=False, metavar="subworkflow name") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1639,7 +1647,7 @@ def subworkflows_info(ctx, tool, dir): # nf-core subworkflows install @subworkflows.command("install") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1697,7 +1705,7 @@ def subworkflows_install(ctx, subworkflow, dir, prompt, force, sha): # nf-core subworkflows remove @subworkflows.command("remove") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1727,7 +1735,7 @@ def subworkflows_remove(ctx, dir, subworkflow): # nf-core subworkflows update @subworkflows.command("update") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", From 2cdb907cc1e89909b2f7b8d0a74998b0bac735a6 Mon Sep 17 00:00:00 2001 From: Pieter Moris <13552343+pmoris@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:21:58 +0100 Subject: [PATCH 112/270] Rename subworkflow argument in subworkflows info command The subworkflow info command referred to the subworkflow name as "tool" in its list of arguments. It has been renamed to "subworkflow" to align with the other subworkflow commands. --- nf_core/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index dc17c6e6c1..807bc776bb 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1608,7 +1608,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo # nf-core subworkflows info @subworkflows.command("info") @click.pass_context -@click.argument("tool", type=str, callback=normalize_case, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1616,7 +1616,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def subworkflows_info(ctx, tool, dir): +def subworkflows_info(ctx, subworkflow, dir): """ Show developer usage information about a given subworkflow. @@ -1633,7 +1633,7 @@ def subworkflows_info(ctx, tool, dir): try: subworkflow_info = SubworkflowInfo( dir, - tool, + subworkflow, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], From 73f507049cd26aa7ec3ed768abc3eddc26a41f48 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 14:30:06 +0000 Subject: [PATCH 113/270] this feels like cheating --- tests/test_modules.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_modules.py b/tests/test_modules.py index da336aa67a..5f64a233f2 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -56,6 +56,7 @@ def create_modules_repo_dummy(tmp_dir): test_snap_path.touch() with open(test_snap_path, "w") as fh: fh.write('{\n "my test": {}\n}') + fh.write('{\n "versions": {}\n}') # remove "TODO" statements from main.nf main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") From 0caee7148734c4aa64b361a4ea3d6bfa9b324355 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 14:41:00 +0000 Subject: [PATCH 114/270] Add nfcore_yml.md for pipeline lint tests --- docs/api/_src/pipeline_lint_tests/nfcore_yml.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/api/_src/pipeline_lint_tests/nfcore_yml.md diff --git a/docs/api/_src/pipeline_lint_tests/nfcore_yml.md b/docs/api/_src/pipeline_lint_tests/nfcore_yml.md new file mode 100644 index 0000000000..f7e797a29c --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/nfcore_yml.md @@ -0,0 +1,5 @@ +# nfcore_yml + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.nfcore_yml +``` From ebabece0487aea0d38f9f7ac26d7e404161db943 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 15:38:37 +0000 Subject: [PATCH 115/270] Add logging module and fix nf_core_version regex --- nf_core/lint/nfcore_yml.py | 8 ++++---- tests/lint/nfcore_yml.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index 6d9f664871..64fefa50b8 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -44,7 +44,7 @@ def nfcore_yml(self): if repo_type not in REPOSITORY_TYPES: failed.append( f"Repository type in .nf-core.yml is not valid. " - f"Should be one of {', '.join(REPOSITORY_TYPES)} but was {repo_type}" + f"Should be one of [{', '.join(REPOSITORY_TYPES)}] but was {repo_type}" ) else: passed.append(f"Repository type in .nf-core.yml is valid: {repo_type}") @@ -58,8 +58,8 @@ def nfcore_yml(self): nf_core_version_re = r"nf_core_version: (.+)" match = re.search(nf_core_version_re, content) if match: - nf_core_version = match.group(1) - if nf_core_version != __version__ and "dev" not in __version__: + nf_core_version = match.group(1).strip('"') + if nf_core_version != __version__ and "dev" not in nf_core_version: warned.append( f"nf-core version in .nf-core.yml is not set to the latest version. " f"Should be {__version__} but was {nf_core_version}" @@ -71,4 +71,4 @@ def nfcore_yml(self): else: ignored.append(".nf-core.yml variable ignored 'nf_core_version'") - return {"passed": passed, "warned": warned, "failed": failed} + return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py index 4c36d27c25..74d42fe229 100644 --- a/tests/lint/nfcore_yml.py +++ b/tests/lint/nfcore_yml.py @@ -41,7 +41,7 @@ def test_nfcore_yml_fail_nfcore_version(self): nf_core_yml = Path(new_pipeline) / ".nf-core.yml" with open(nf_core_yml) as fh: content = fh.read() - new_content = re.sub(r"nf-core_version: [\s]*", "nf-core_version: foo", content) + new_content = re.sub(r"nf_core_version:.+", "nf_core_version: foo", content) with open(nf_core_yml, "w") as fh: fh.write(new_content) lint_obj = nf_core.lint.PipelineLint(new_pipeline) From faf3a8750277c7382e52e31dcdfdc8f16bf5f698 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:48:51 +0000 Subject: [PATCH 116/270] fix: assertions for called_with/called_once_with were silently returning mocks before 3.12. Broken in 3.12. --- tests/test_cli.py | 4 ++-- tests/test_sync.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 54e420f5e4..618ef0d290 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -364,7 +364,7 @@ def test_schema_lint(self, mock_get_schema_path): """Test nf-core schema lint defaults to nextflow_schema.json""" cmd = ["schema", "lint"] result = self.invoke_cli(cmd) - assert mock_get_schema_path.called_with("nextflow_schema.json") + mock_get_schema_path.assert_called_with("nextflow_schema.json") assert "nextflow_schema.json" in result.output @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") @@ -372,7 +372,7 @@ def test_schema_lint_filename(self, mock_get_schema_path): """Test nf-core schema lint accepts a filename""" cmd = ["schema", "lint", "some_other_filename"] result = self.invoke_cli(cmd) - assert mock_get_schema_path.called_with("some_other_filename") + mock_get_schema_path.assert_called_with("some_other_filename") assert "some_other_filename" in result.output assert "nextflow_schema.json" not in result.output diff --git a/tests/test_sync.py b/tests/test_sync.py index 6f0e502e8b..13f8559045 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -374,7 +374,7 @@ def test_close_open_pr(self, mock_patch, mock_post): } assert psync.close_open_pr(pr) - assert mock_patch.called_once_with("url_to_update_pr") + mock_patch.assert_called_once_with("url_to_update_pr") @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) @@ -397,7 +397,7 @@ def test_close_open_pr_fail(self, mock_patch, mock_post): } assert not psync.close_open_pr(pr) - assert mock_patch.called_once_with("bad_url_to_update_pr") + mock_patch.assert_called_once_with("bad_url_to_update_pr") def test_reset_target_dir(self): """Try resetting target pipeline directory""" From 7cde82f0d97c4161aa5ea144ba83a893709b2453 Mon Sep 17 00:00:00 2001 From: Joon-Klaps Date: Wed, 20 Mar 2024 15:52:35 +0000 Subject: [PATCH 117/270] fix test-datasets --- tests/subworkflows/lint.py | 4 ++-- tests/test_modules.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 84ed6d4234..a315ea3912 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -187,12 +187,12 @@ def test_subworkflows_absent_version(self): """Test linting a nf-test module if the versions is absent in the snapshot file `""" self.subworkflow_install.install("bam_sort_stats_samtools") with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_sort_stats_samtools", "test", "main.nf.test.snap") + Path(self.pipeline_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") ) as fh: content = fh.read() new_content = content.replace("versions", "foo") with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "bam_sort_stats_samtools", "test", "main.nf.test.snap"), "w" + Path(self.pipeline_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap"), "w" ) as fh: fh.write(new_content) diff --git a/tests/test_modules.py b/tests/test_modules.py index 5f64a233f2..26fdea41ac 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -55,8 +55,8 @@ def create_modules_repo_dummy(tmp_dir): test_snap_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") test_snap_path.touch() with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n}') - fh.write('{\n "versions": {}\n}') + fh.write('{\n "my test": {}\n') + fh.write(' "versions": {}\n}') # remove "TODO" statements from main.nf main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") From a124c1479c020ad950b50e79706313b0de549ae6 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 20 Mar 2024 16:00:39 +0000 Subject: [PATCH 118/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94cc5cb407..ee67744a69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) +- Fix incorrect assertions for called_with on mocks ([#2891](https://github.com/nf-core/tools/pull/2891)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From d29f2ba52d505a89abc2e8f93ba4496d649ce9ae Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 20 Mar 2024 16:29:41 +0000 Subject: [PATCH 119/270] Update expected assertion for mock patch calls --- tests/test_sync.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 13f8559045..b94968cd4c 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -26,7 +26,12 @@ def setUp(self): self.pipeline_dir = os.path.join(self.tmp_dir, "testpipeline") default_branch = "master" self.create_obj = nf_core.create.PipelineCreate( - "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True, default_branch=default_branch + "testing", + "test pipeline", + "tester", + outdir=self.pipeline_dir, + plain=True, + default_branch=default_branch, ) self.create_obj.init_pipeline() self.remote_path = os.path.join(self.tmp_dir, "remote_repo") @@ -374,7 +379,7 @@ def test_close_open_pr(self, mock_patch, mock_post): } assert psync.close_open_pr(pr) - mock_patch.assert_called_once_with("url_to_update_pr") + mock_patch.assert_called_once_with(url="url_to_update_pr", data='{"state": "closed"}') @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) @@ -397,7 +402,7 @@ def test_close_open_pr_fail(self, mock_patch, mock_post): } assert not psync.close_open_pr(pr) - mock_patch.assert_called_once_with("bad_url_to_update_pr") + mock_patch.assert_called_once_with(url="bad_url_to_update_pr", data='{"state": "closed"}') def test_reset_target_dir(self): """Try resetting target pipeline directory""" From 304e41fee50cb817e29568bbf4d442c8f2d6b430 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 21 Mar 2024 19:25:19 +0100 Subject: [PATCH 120/270] use pathlib --- nf_core/lint/nfcore_yml.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index 64fefa50b8..aeae000e05 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -1,5 +1,5 @@ -import os import re +from pathlib import Path from nf_core import __version__ @@ -29,10 +29,10 @@ def nfcore_yml(self): ignore_configs = self.lint_config.get(".nf-core", []) try: - with open(os.path.join(self.wf_path, ".nf-core.yml")) as fh: + with open(Path(self.wf_path, ".nf-core.yml")) as fh: content = fh.read() except FileNotFoundError: - with open(os.path.join(self.wf_path, ".nf-core.yaml")) as fh: + with open(Path(self.wf_path, ".nf-core.yaml")) as fh: content = fh.read() if "repository_type" not in ignore_configs: From 585d591dea93259b9393d6f0c7a39f8a8a3b0780 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 21 Mar 2024 19:28:43 +0100 Subject: [PATCH 121/270] add type hints --- nf_core/lint/nfcore_yml.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index aeae000e05..9f45355e76 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -1,12 +1,13 @@ import re from pathlib import Path +from typing import Dict, List from nf_core import __version__ REPOSITORY_TYPES = ["pipeline", "modules"] -def nfcore_yml(self): +def nfcore_yml(self) -> Dict[str, List[str]]: """Repository ``.nf-core.yml`` tests The ``.nf-core.yml`` contains metadata for nf-core tools to correctly apply its features. @@ -20,10 +21,10 @@ def nfcore_yml(self): * Check if the nf-core version is set to the latest version. """ - passed = [] - warned = [] - failed = [] - ignored = [] + passed: List[str] = [] + warned: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] # Remove field that should be ignored according to the linting config ignore_configs = self.lint_config.get(".nf-core", []) From e3b0949d7d3bba550513fa634b5760a2a0c34556 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 21 Mar 2024 19:31:51 +0100 Subject: [PATCH 122/270] fix formatting --- nf_core/lint/nfcore_yml.py | 20 ++++++++++---------- tests/lint/nfcore_yml.py | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py index 9f45355e76..f23b2f1a84 100644 --- a/nf_core/lint/nfcore_yml.py +++ b/nf_core/lint/nfcore_yml.py @@ -44,15 +44,15 @@ def nfcore_yml(self) -> Dict[str, List[str]]: repo_type = match.group(1) if repo_type not in REPOSITORY_TYPES: failed.append( - f"Repository type in .nf-core.yml is not valid. " - f"Should be one of [{', '.join(REPOSITORY_TYPES)}] but was {repo_type}" + f"Repository type in `.nf-core.yml` is not valid. " + f"Should be one of `[{', '.join(REPOSITORY_TYPES)}]` but was `{repo_type}`" ) else: - passed.append(f"Repository type in .nf-core.yml is valid: {repo_type}") + passed.append(f"Repository type in `.nf-core.yml` is valid: `{repo_type}`") else: - warned.append("Repository type not set in .nf-core.yml") + warned.append("Repository type not set in `.nf-core.yml`") else: - ignored.append(".nf-core.yml variable ignored 'repository_type'") + ignored.append("`.nf-core.yml` variable ignored 'repository_type'") if "nf_core_version" not in ignore_configs: # Check that the nf-core version is set in the .nf-core.yml @@ -62,14 +62,14 @@ def nfcore_yml(self) -> Dict[str, List[str]]: nf_core_version = match.group(1).strip('"') if nf_core_version != __version__ and "dev" not in nf_core_version: warned.append( - f"nf-core version in .nf-core.yml is not set to the latest version. " - f"Should be {__version__} but was {nf_core_version}" + f"nf-core version in `.nf-core.yml` is not set to the latest version. " + f"Should be `{__version__}` but was `{nf_core_version}`" ) else: - passed.append(f"nf-core version in .nf-core.yml is set to the latest version: {nf_core_version}") + passed.append(f"nf-core version in `.nf-core.yml` is set to the latest version: `{nf_core_version}`") else: - warned.append("nf-core version not set in .nf-core.yml") + warned.append("nf-core version not set in `.nf-core.yml`") else: - ignored.append(".nf-core.yml variable ignored 'nf_core_version'") + ignored.append("`.nf-core.yml` variable ignored 'nf_core_version'") return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py index 74d42fe229..b75115fea4 100644 --- a/tests/lint/nfcore_yml.py +++ b/tests/lint/nfcore_yml.py @@ -10,8 +10,8 @@ def test_nfcore_yml_pass(self): self.lint_obj._load() results = self.lint_obj.nfcore_yml() - assert "Repository type in .nf-core.yml is valid" in str(results["passed"]) - assert "nf-core version in .nf-core.yml is set to the latest version" in str(results["passed"]) + assert "Repository type in `.nf-core.yml` is valid" in str(results["passed"]) + assert "nf-core version in `.nf-core.yml` is set to the latest version" in str(results["passed"]) assert len(results.get("warned", [])) == 0 assert len(results.get("failed", [])) == 0 assert len(results.get("ignored", [])) == 0 @@ -29,7 +29,7 @@ def test_nfcore_yml_fail_repo_type(self): lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() results = lint_obj.nfcore_yml() - assert "Repository type in .nf-core.yml is not valid." in str(results["failed"]) + assert "Repository type in `.nf-core.yml` is not valid." in str(results["failed"]) assert len(results.get("warned", [])) == 0 assert len(results.get("passed", [])) >= 0 assert len(results.get("ignored", [])) == 0 From 6ec89d842079151f994ec42170454978fe85657d Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 21 Mar 2024 21:13:27 +0100 Subject: [PATCH 123/270] fix test --- tests/lint/nfcore_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py index b75115fea4..474ccd48fc 100644 --- a/tests/lint/nfcore_yml.py +++ b/tests/lint/nfcore_yml.py @@ -47,7 +47,7 @@ def test_nfcore_yml_fail_nfcore_version(self): lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() results = lint_obj.nfcore_yml() - assert "nf-core version in .nf-core.yml is not set to the latest version." in str(results["warned"]) + assert "nf-core version in `.nf-core.yml` is not set to the latest version." in str(results["warned"]) assert len(results.get("failed", [])) == 0 assert len(results.get("passed", [])) >= 0 assert len(results.get("ignored", [])) == 0 From 958dfa64a92e6fd69793ea0a56e28dc70b605bfa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 20:31:31 +0000 Subject: [PATCH 124/270] Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a68848237..3ab0dceba2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.3 + rev: v0.3.4 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From e9667f449d35835c0bc96e25e446d3cb29806628 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 21 Mar 2024 20:32:16 +0000 Subject: [PATCH 125/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e740d2af..cd76d3f723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 2cb50ca50c76e724fd71b69598b60ce0cbed0d3d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:13:48 +0000 Subject: [PATCH 126/270] chore(deps): update gitpod/workspace-base docker digest to fedb722 --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 3136d02e97..32eb2f1931 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:1e133e5691add6c19443672594b9f3d7d9c3372ead4c86a4490c2701dbfa32e3 +FROM gitpod/workspace-base@sha256:fedb7224944615d57661e4fd39f4873f0cb92234f55adf83a67a0fc8b4a817dd USER root From bf0aa71fd03e85cf53d4626db7790ad906233471 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:37:28 +0000 Subject: [PATCH 127/270] chore(deps): update gitpod/workspace-base docker digest to 168d78b --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 32eb2f1931..40aa998a50 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:fedb7224944615d57661e4fd39f4873f0cb92234f55adf83a67a0fc8b4a817dd +FROM gitpod/workspace-base@sha256:168d78bc249332f9437b11a1844819308ac7ec1bc6bab61bf5adf8cbe53bf8e6 USER root From f0b9a1be7f2a43895304bfe6edebad280108fbaf Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 02:24:28 +0000 Subject: [PATCH 128/270] Pin python Docker tag to 5dc6f84 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6e785f7a60..9f49001089 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim +FROM python:3.12-slim@sha256:5dc6f84b5e97bfb0c90abfb7c55f3cacc2cb6687c8f920b64a833a2219875997 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From e28d3e2280c705ce96d099c5123a75456a03be23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 26 Mar 2024 10:36:10 +0100 Subject: [PATCH 129/270] Remove pyproject.toml from files that must not exist --- nf_core/lint/files_exist.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index ae105d8d2c..d801caf704 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -89,7 +89,6 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: lib/WorkflowPIPELINE.groovy parameters.settings.json pipeline_template.yml # saving information in .nf-core.yml - pyproject.toml Singularity @@ -198,7 +197,6 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy"), Path("parameters.settings.json"), Path("pipeline_template.yml"), # saving information in .nf-core.yml - Path("pyproject.toml"), Path("Singularity"), ] files_warn_ifexists = [Path(".travis.yml")] From 0fd48bb296382a06c62bf17ec7f2d108172af001 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Mar 2024 11:30:26 +0100 Subject: [PATCH 130/270] Remove old references to CUSTOMDUMPSOFTWAREVERSIONS --- nf_core/pipeline-template/conf/base.config | 3 --- nf_core/pipeline-template/conf/modules.config | 8 -------- 2 files changed, 11 deletions(-) diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index f73c5afaa4..9c62bf0634 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -59,7 +59,4 @@ process { errorStrategy = 'retry' maxRetries = 2 } - withName:CUSTOM_DUMPSOFTWAREVERSIONS { - cache = false - } } diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index e3ea8fa6c4..d203d2b6e6 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -22,14 +22,6 @@ process { ext.args = '--quiet' } - withName: CUSTOM_DUMPSOFTWAREVERSIONS { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - pattern: '*_versions.yml' - ] - } - withName: 'MULTIQC' { ext.args = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' } publishDir = [ From 48a23bf6a006fb3807aa1d90998bb81b679c6c2a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 26 Mar 2024 10:32:08 +0000 Subject: [PATCH 131/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e740d2af..3f589f1104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) +- Remove old references to CUSTOMDUMPSOFTWAREVERSIONS ([#2897](https://github.com/nf-core/tools/pull/2897)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 2e863d34952f9948564bd8977f5c06eba08bd270 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Mar 2024 13:12:08 +0100 Subject: [PATCH 132/270] Apply suggestions from code review --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71dfb8eff8..cc77a01831 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,6 @@ - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) - Remove old references to CUSTOMDUMPSOFTWAREVERSIONS ([#2897](https://github.com/nf-core/tools/pull/2897)) - ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] ### Template From fb2ded4bbf57de8c5c3e3f9e97c34fc7e639ed6b Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 13:06:27 +0100 Subject: [PATCH 133/270] fix type hints --- nf_core/lint/__init__.py | 296 +++++++++++++++++++-------------------- nf_core/schema.py | 6 +- 2 files changed, 152 insertions(+), 150 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index be9ac183a6..c19d107610 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -9,7 +9,7 @@ import logging import os from pathlib import Path -from typing import List, Union +from typing import List, Tuple, Union import git import rich @@ -25,6 +25,7 @@ import nf_core.subworkflows.lint import nf_core.utils from nf_core import __version__ +from nf_core.components.lint import ComponentLint from nf_core.lint_utils import console from nf_core.utils import plural_s as _s from nf_core.utils import strip_ansi_codes @@ -32,148 +33,6 @@ log = logging.getLogger(__name__) -def run_linting( - pipeline_dir, - release_mode=False, - fix=(), - key=(), - show_passed=False, - fail_ignored=False, - fail_warned=False, - sort_by="test", - md_fn=None, - json_fn=None, - hide_progress=False, -): - """Runs all nf-core linting checks on a given Nextflow pipeline project - in either `release` mode or `normal` mode (default). Returns an object - of type :class:`PipelineLint` after finished. - - Args: - pipeline_dir (str): The path to the Nextflow pipeline root directory - release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. - See :class:`PipelineLint` for more information. - - Returns: - An object of type :class:`PipelineLint` that contains all the linting results. - An object of type :class:`ComponentLint` that contains all the linting results for the modules. - An object of type :class:`ComponentLint` that contains all the linting results for the subworkflows. - """ - - # Verify that the requested tests exist - if key: - all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union( - set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)) - ) - bad_keys = [k for k in key if k not in all_tests] - if len(bad_keys) > 0: - raise AssertionError( - "Test name{} not recognised: '{}'".format( - _s(bad_keys), - "', '".join(bad_keys), - ) - ) - log.info("Only running tests: '{}'".format("', '".join(key))) - - # Check if we were given any keys, and if they match any pipeline tests - if key: - pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) - else: - # If no key is supplied, run all tests - pipeline_keys = None - - # Create the lint object - lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored, fail_warned, hide_progress) - - # Load the various pipeline configs - lint_obj._load_lint_config() - lint_obj._load_pipeline_config() - lint_obj._list_files() - - # Create the modules lint object - module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) - # Create the subworkflows lint object - try: - subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) - except LookupError: - subworkflow_lint_obj = None - - # Verify that the pipeline is correctly configured and has a modules.json file - module_lint_obj.has_valid_directory() - module_lint_obj.has_modules_file() - - # Run only the tests we want - if key: - # Select only the module lint tests - module_lint_tests = list( - set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) - ) - # Select only the subworkflow lint tests - subworkflow_lint_tests = list( - set(key).intersection( - set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) - ) - ) - else: - # If no key is supplied, run the default modules tests - module_lint_tests = ("module_changes", "module_version") - subworkflow_lint_tests = ("subworkflow_changes", "subworkflow_version") - module_lint_obj.filter_tests_by_key(module_lint_tests) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) - - # Set up files for component linting test - module_lint_obj.set_up_pipeline_files() - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.set_up_pipeline_files() - - # Run the pipeline linting tests - try: - lint_obj._lint_pipeline() - except AssertionError as e: - log.critical(f"Critical error: {e}") - log.info("Stopping tests...") - return lint_obj, module_lint_obj - - # Run the module lint tests - if len(module_lint_obj.all_local_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) - if len(module_lint_obj.all_remote_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) - # Run the subworkflows lint tests - if subworkflow_lint_obj is not None: - if len(subworkflow_lint_obj.all_local_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) - if len(subworkflow_lint_obj.all_remote_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) - - # Print the results - lint_obj._print_results(show_passed) - module_lint_obj._print_results(show_passed, sort_by=sort_by) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) - nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) - nf_core.lint_utils.print_fixes(lint_obj) - - # Save results to Markdown file - if md_fn is not None: - log.info(f"Writing lint results to {md_fn}") - markdown = lint_obj._get_results_md() - with open(md_fn, "w") as fh: - fh.write(markdown) - - # Save results to JSON file - if json_fn is not None: - lint_obj._save_json_results(json_fn) - - # Reminder about --release mode flag if we had failures - if len(lint_obj.failed) > 0: - if release_mode: - log.info("Reminder: Lint tests were run in --release mode.") - - return lint_obj, module_lint_obj, subworkflow_lint_obj - - class PipelineLint(nf_core.utils.Pipeline): """Object to hold linting information and results. @@ -409,7 +268,7 @@ def format_result(test_results): # Table of passed tests if len(self.passed) > 0 and show_passed: console.print( - rich.panel.Panel( + Panel( format_result(self.passed), title=rf"[bold][✔] {len(self.passed)} Pipeline Test{_s(self.passed)} Passed", title_align="left", @@ -421,7 +280,7 @@ def format_result(test_results): # Table of fixed tests if len(self.fixed) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.fixed), title=rf"[bold][?] {len(self.fixed)} Pipeline Test{_s(self.fixed)} Fixed", title_align="left", @@ -433,7 +292,7 @@ def format_result(test_results): # Table of ignored tests if len(self.ignored) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.ignored), title=rf"[bold][?] {len(self.ignored)} Pipeline Test{_s(self.ignored)} Ignored", title_align="left", @@ -445,7 +304,7 @@ def format_result(test_results): # Table of warning tests if len(self.warned) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.warned), title=rf"[bold][!] {len(self.warned)} Pipeline Test Warning{_s(self.warned)}", title_align="left", @@ -457,7 +316,7 @@ def format_result(test_results): # Table of failing tests if len(self.failed) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.failed), title=rf"[bold][✗] {len(self.failed)} Pipeline Test{_s(self.failed)} Failed", title_align="left", @@ -638,3 +497,144 @@ def _wrap_quotes(self, files: Union[List[str], List[Path], Path]) -> str: files = [files] bfiles = [f"`{str(f)}`" for f in files] return " or ".join(bfiles) + + +def run_linting( + pipeline_dir, + release_mode: bool = False, + fix=(), + key=(), + show_passed: bool = False, + fail_ignored: bool = False, + fail_warned: bool = False, + sort_by: str = "test", + md_fn=None, + json_fn=None, + hide_progress: bool = False, +) -> Tuple[PipelineLint, ComponentLint, ComponentLint | None]: + """Runs all nf-core linting checks on a given Nextflow pipeline project + in either `release` mode or `normal` mode (default). Returns an object + of type :class:`PipelineLint` after finished. + + Args: + pipeline_dir (str): The path to the Nextflow pipeline root directory + release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. + See :class:`PipelineLint` for more information. + + Returns: + An object of type :class:`PipelineLint` that contains all the linting results. + An object of type :class:`ComponentLint` that contains all the linting results for the modules. + An object of type :class:`ComponentLint` that contains all the linting results for the subworkflows. + """ + + # Verify that the requested tests exist + if key: + all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union( + set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)) + ) + bad_keys = [k for k in key if k not in all_tests] + if len(bad_keys) > 0: + raise AssertionError( + "Test name{} not recognised: '{}'".format( + _s(bad_keys), + "', '".join(bad_keys), + ) + ) + log.info("Only running tests: '{}'".format("', '".join(key))) + + # Check if we were given any keys, and if they match any pipeline tests + if key: + pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) + else: + # If no key is supplied, run all tests + pipeline_keys = None + + # Create the lint object + lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored, fail_warned, hide_progress) + + # Load the various pipeline configs + lint_obj._load_lint_config() + lint_obj._load_pipeline_config() + lint_obj._list_files() + + # Create the modules lint object + module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) + # Create the subworkflows lint object + try: + subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) + except LookupError: + subworkflow_lint_obj = None + + # Verify that the pipeline is correctly configured and has a modules.json file + module_lint_obj.has_valid_directory() + module_lint_obj.has_modules_file() + # Run only the tests we want + if key: + # Select only the module lint tests + module_lint_tests = list( + set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) + ) + # Select only the subworkflow lint tests + subworkflow_lint_tests = list( + set(key).intersection( + set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) + ) + ) + else: + # If no key is supplied, run the default modules tests + module_lint_tests = list(("module_changes", "module_version")) + subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version")) + module_lint_obj.filter_tests_by_key(module_lint_tests) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) + + # Set up files for component linting test + module_lint_obj.set_up_pipeline_files() + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.set_up_pipeline_files() + + # Run the pipeline linting tests + try: + lint_obj._lint_pipeline() + except AssertionError as e: + log.critical(f"Critical error: {e}") + log.info("Stopping tests...") + return lint_obj, module_lint_obj, subworkflow_lint_obj + + # Run the module lint tests + if len(module_lint_obj.all_local_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) + if len(module_lint_obj.all_remote_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) + # Run the subworkflows lint tests + if subworkflow_lint_obj is not None: + if len(subworkflow_lint_obj.all_local_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) + if len(subworkflow_lint_obj.all_remote_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) + + # Print the results + lint_obj._print_results(show_passed) + module_lint_obj._print_results(show_passed, sort_by=sort_by) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) + nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) + nf_core.lint_utils.print_fixes(lint_obj) + + # Save results to Markdown file + if md_fn is not None: + log.info(f"Writing lint results to {md_fn}") + markdown = lint_obj._get_results_md() + with open(md_fn, "w") as fh: + fh.write(markdown) + + # Save results to JSON file + if json_fn is not None: + lint_obj._save_json_results(json_fn) + + # Reminder about --release mode flag if we had failures + if len(lint_obj.failed) > 0: + if release_mode: + log.info("Reminder: Lint tests were run in --release mode.") + + return lint_obj, module_lint_obj, subworkflow_lint_obj diff --git a/nf_core/schema.py b/nf_core/schema.py index 4096c80f93..eabef74833 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -62,8 +62,10 @@ def get_schema_path(self, path, local_only=False, revision=None): # Path does not exist - assume a name of a remote workflow elif not local_only: - self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision) - self.schema_filename = Path(self.pipeline_dir, "nextflow_schema.json") + if self.pipeline_dir is not None: + self.schema_filename = Path(self.pipeline_dir, "nextflow_schema.json") + else: + self.schema_filename = None # Only looking for local paths, overwrite with None to be safe else: From 4fa15142ac03aebab4f9cb0323a2fdc3411346de Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 13:59:08 +0100 Subject: [PATCH 134/270] fix tests --- tests/test_cli.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 618ef0d290..1f7368af12 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -31,13 +31,13 @@ def test_header_outdated(mock_check_outdated, mock_nf_core_cli, capsys): class TestCli(unittest.TestCase): - """Class for testing the commandline interface""" + """Class for testing the command line interface""" def setUp(self): self.runner = CliRunner() def assemble_params(self, params): - """Assemble a dictionnary of parameters into a list of arguments for the cli + """Assemble a dictionary of parameters into a list of arguments for the cli Note: if the value of a parameter is None, it will be considered a flag. @@ -363,18 +363,23 @@ def test_lint_log_user_warning(self, mock_lint, mock_is_pipeline): def test_schema_lint(self, mock_get_schema_path): """Test nf-core schema lint defaults to nextflow_schema.json""" cmd = ["schema", "lint"] - result = self.invoke_cli(cmd) + Path("nextflow_schema.json").touch() + self.invoke_cli(cmd) mock_get_schema_path.assert_called_with("nextflow_schema.json") - assert "nextflow_schema.json" in result.output + + # clean up + Path("nextflow_schema.json").unlink() @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") def test_schema_lint_filename(self, mock_get_schema_path): """Test nf-core schema lint accepts a filename""" + Path("some_other_filename").touch() cmd = ["schema", "lint", "some_other_filename"] - result = self.invoke_cli(cmd) + self.invoke_cli(cmd) mock_get_schema_path.assert_called_with("some_other_filename") - assert "some_other_filename" in result.output - assert "nextflow_schema.json" not in result.output + + # clean up + Path("some_other_filename").unlink() @mock.patch("nf_core.create_logo.create_logo") def test_create_logo(self, mock_create_logo): From 016db6d9f3e84fb079258d7f1600e020e07d592b Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 15:21:15 +0100 Subject: [PATCH 135/270] try to fix the remote schema functionality --- nf_core/components/components_command.py | 2 +- nf_core/modules/modules_json.py | 22 ++++++++-------- nf_core/schema.py | 32 +++++++++++++++++++----- tests/test_schema.py | 7 +++--- 4 files changed, 42 insertions(+), 21 deletions(-) diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 8332429835..4df67639e2 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -245,7 +245,7 @@ def check_patch_paths(self, patch_path: Path, module_name: str) -> None: # Update path in modules.json if the file is in the correct format modules_json = ModulesJson(self.dir) modules_json.load() - if modules_json.has_git_url_and_modules(): + if modules_json.has_git_url_and_modules() and modules_json.modules_json is not None: modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ self.modules_repo.repo_path ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index f68c27b2d8..7d78268e92 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -6,7 +6,6 @@ import shutil import tempfile from pathlib import Path -from typing import Union import git import questionary @@ -32,7 +31,7 @@ class ModulesJson: An object for handling a 'modules.json' file in a pipeline """ - def __init__(self, pipeline_dir): + def __init__(self, pipeline_dir: str): """ Initialise the object. @@ -43,7 +42,7 @@ def __init__(self, pipeline_dir): self.modules_dir = Path(self.dir, "modules") self.subworkflows_dir = Path(self.dir, "subworkflows") self.modules_json_path = Path(self.dir, "modules.json") - self.modules_json: Union(dict, None) = None + self.modules_json = None self.pipeline_modules = None self.pipeline_subworkflows = None self.pipeline_components = None @@ -1051,17 +1050,18 @@ def get_component_branch(self, component_type, component, repo_url, install_dir) ) return branch - def dump(self, run_prettier: bool = False): + def dump(self, run_prettier: bool = False) -> None: """ Sort the modules.json, and write it to file """ - # Sort the modules.json - self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) - if run_prettier: - dump_json_with_prettier(self.modules_json_path, self.modules_json) - else: - with open(self.modules_json_path, "w") as fh: - json.dump(self.modules_json, fh, indent=4) + if self.modules_json is not None: + # Sort the modules.json + self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) + if run_prettier: + dump_json_with_prettier(self.modules_json_path, self.modules_json) + else: + with open(self.modules_json_path, "w") as fh: + json.dump(self.modules_json, fh, indent=4) def resolve_missing_installation(self, missing_installation, component_type): missing_but_in_mod_json = [ diff --git a/nf_core/schema.py b/nf_core/schema.py index eabef74833..a4b635c819 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -6,6 +6,7 @@ import tempfile import webbrowser from pathlib import Path +from typing import Union import jinja2 import jsonschema @@ -46,11 +47,14 @@ def __init__(self): self.web_schema_build_web_url = None self.web_schema_build_api_url = None - def get_schema_path(self, path, local_only=False, revision=None): + def get_schema_path( + self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None + ) -> None: """Given a pipeline name, directory, or path, set self.schema_filename""" path = Path(path) # Supplied path exists - assume a local pipeline directory or schema if path.exists(): + log.debug(f"Path exists: {path}. Assuming local pipeline directory or schema") if revision is not None: log.warning(f"Local workflow supplied, ignoring revision '{revision}'") if path.is_dir(): @@ -62,17 +66,20 @@ def get_schema_path(self, path, local_only=False, revision=None): # Path does not exist - assume a name of a remote workflow elif not local_only: - if self.pipeline_dir is not None: - self.schema_filename = Path(self.pipeline_dir, "nextflow_schema.json") + self.schema_filename = path / "nextflow_schema.json" + if revision is not None: + self.load_remote_schema( + f"https://raw.githubusercontent.com/nf-core/{path}/{revision}/nextflow_schema.json" + ) else: - self.schema_filename = None + self.load_remote_schema(f"https://raw.githubusercontent.com/nf-core/{path}/master/nextflow_schema.json") # Only looking for local paths, overwrite with None to be safe else: self.schema_filename = None # Check that the schema file exists - if self.schema_filename is None or not Path(self.schema_filename).exists(): + if self.schema_filename is None or not Path(self.schema_filename).exists() and local_only: error = f"Could not find pipeline schema for '{path}': {self.schema_filename}" log.error(error) raise AssertionError(error) @@ -105,7 +112,7 @@ def load_lint_schema(self): def load_schema(self): """Load a pipeline schema from a file""" - if self.schema_filename is None: + if self.schema_filename is None or not Path(self.schema_filename).exists(): raise AssertionError("Pipeline schema filename could not be found.") with open(self.schema_filename) as fh: @@ -114,6 +121,19 @@ def load_schema(self): self.schema_params = {} log.debug(f"JSON file loaded: {self.schema_filename}") + def load_remote_schema(self, url): + """Load a pipeline schema from a remote URL""" + import requests + + response = requests.get(url) + if response.status_code != 200: + raise AssertionError(f"Could not load schema from {url}") + self.schema = response.json() + self.schema_filename = url + self.schema_defaults = {} + self.schema_params = {} + log.debug(f"JSON file loaded: {self.schema_filename}") + def sanitise_param_default(self, param): """ Given a param, ensure that the default value is the correct variable type diff --git a/tests/test_schema.py b/tests/test_schema.py index 29f4921985..5c28ee92bf 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -5,6 +5,7 @@ import shutil import tempfile import unittest +from pathlib import Path from unittest import mock import pytest @@ -46,7 +47,7 @@ def test_load_lint_schema(self): def test_load_lint_schema_nofile(self): """Check that linting raises properly if a non-existant file is given""" - with pytest.raises(RuntimeError): + with pytest.raises(AssertionError): self.schema_obj.get_schema_path("fake_file") def test_load_lint_schema_notjson(self): @@ -314,9 +315,9 @@ def test_build_schema_from_scratch(self, tmp_dir): Pretty much a copy of test_launch.py test_make_pipeline_schema """ - test_pipeline_dir = os.path.join(tmp_dir, "wf") + test_pipeline_dir = Path(tmp_dir, "wf") shutil.copytree(self.template_dir, test_pipeline_dir) - os.remove(os.path.join(test_pipeline_dir, "nextflow_schema.json")) + Path(test_pipeline_dir, "nextflow_schema.json").unlink() self.schema_obj.build_schema(test_pipeline_dir, True, False, None) From b69f328c2d441e64aa5ded97a8bd850741e1720d Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 15:24:43 +0100 Subject: [PATCH 136/270] add isolated filesystem to cli tests --- tests/test_cli.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 1f7368af12..913a4aac1d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -363,23 +363,21 @@ def test_lint_log_user_warning(self, mock_lint, mock_is_pipeline): def test_schema_lint(self, mock_get_schema_path): """Test nf-core schema lint defaults to nextflow_schema.json""" cmd = ["schema", "lint"] - Path("nextflow_schema.json").touch() - self.invoke_cli(cmd) - mock_get_schema_path.assert_called_with("nextflow_schema.json") - - # clean up - Path("nextflow_schema.json").unlink() + with self.runner.isolated_filesystem(): + with open("nextflow_schema.json", "w") as f: + f.write("{}") + self.invoke_cli(cmd) + mock_get_schema_path.assert_called_with("nextflow_schema.json") @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") def test_schema_lint_filename(self, mock_get_schema_path): """Test nf-core schema lint accepts a filename""" - Path("some_other_filename").touch() cmd = ["schema", "lint", "some_other_filename"] - self.invoke_cli(cmd) - mock_get_schema_path.assert_called_with("some_other_filename") - - # clean up - Path("some_other_filename").unlink() + with self.runner.isolated_filesystem(): + with open("some_other_filename", "w") as f: + f.write("{}") + self.invoke_cli(cmd) + mock_get_schema_path.assert_called_with("some_other_filename") @mock.patch("nf_core.create_logo.create_logo") def test_create_logo(self, mock_create_logo): From 738e4e0db6a07b9c59ed6a1f0853af6e4f326eb7 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 15:31:40 +0100 Subject: [PATCH 137/270] use correct types --- nf_core/lint/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 1dda6a2078..2a8576d5fb 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -513,7 +513,7 @@ def run_linting( md_fn=None, json_fn=None, hide_progress: bool = False, -) -> Tuple[PipelineLint, ComponentLint, ComponentLint | None]: +) -> Tuple[PipelineLint, ComponentLint, Union[ComponentLint, None]]: """Runs all nf-core linting checks on a given Nextflow pipeline project in either `release` mode or `normal` mode (default). Returns an object of type :class:`PipelineLint` after finished. From 029617703a05429f61411b2e92cc9acdebc1e58c Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 16:58:29 +0100 Subject: [PATCH 138/270] use base class for moduels and base config --- nf_core/lint/__init__.py | 4 ++ nf_core/lint/base_config.py | 14 +++++++ nf_core/lint/modules_config.py | 14 +++++++ nf_core/lint_utils.py | 68 ++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 nf_core/lint/base_config.py create mode 100644 nf_core/lint/modules_config.py diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index be9ac183a6..92a699ac1c 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -199,9 +199,11 @@ class PipelineLint(nf_core.utils.Pipeline): from .actions_schema_validation import ( # type: ignore[misc] actions_schema_validation, ) + from .base_config import base_config # type: ignore[misc] from .files_exist import files_exist # type: ignore[misc] from .files_unchanged import files_unchanged # type: ignore[misc] from .merge_markers import merge_markers # type: ignore[misc] + from .modules_config import modules_config # type: ignore[misc] from .modules_json import modules_json # type: ignore[misc] from .modules_structure import modules_structure # type: ignore[misc] from .multiqc_config import multiqc_config # type: ignore[misc] @@ -264,6 +266,8 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "base_config", + "modules_config", ] + (["version_consistency"] if release_mode else []) def _load(self): diff --git a/nf_core/lint/base_config.py b/nf_core/lint/base_config.py new file mode 100644 index 0000000000..cd1385b1ca --- /dev/null +++ b/nf_core/lint/base_config.py @@ -0,0 +1,14 @@ +from pathlib import Path +from typing import Dict, List + +from nf_core.lint_utils import LintFile + + +def base_config(self) -> Dict[str, List[str]]: + """Make sure the conf/base.config file follows the nf-core template, especially removed sections.""" + + result = LintFile(self.wf_path, self.lint_config).lint_file( + "base_config", Path("conf", "base.config"), ["withName:CUSTOM_DUMPSOFTWAREVERSIONS"] + ) + + return result diff --git a/nf_core/lint/modules_config.py b/nf_core/lint/modules_config.py new file mode 100644 index 0000000000..53c58766ea --- /dev/null +++ b/nf_core/lint/modules_config.py @@ -0,0 +1,14 @@ +from pathlib import Path +from typing import Dict, List + +from nf_core.lint_utils import LintFile + + +def modules_config(self) -> Dict[str, List[str]]: + """Make sure the conf/modules.config file follows the nf-core template, especially removed sections.""" + + result = LintFile(self.wf_path, self.lint_config).lint_file( + "modules_config", Path("conf", "modules.config"), ["withName:CUSTOM_DUMPSOFTWAREVERSIONS"] + ) + + return result diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 6eca6522d4..e099fcabf3 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -2,6 +2,7 @@ import logging import subprocess from pathlib import Path +from typing import Dict, List, Tuple import rich from rich.console import Console @@ -16,6 +17,42 @@ console = Console(force_terminal=nf_core.utils.rich_force_colors()) +class LintFile: + def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): + self.wf_path = wf_path + self.lint_config = lint_config + + def lint_file(self, lint_name: str, file_path: Path, removed_sections: List[str]) -> Dict[str, List[str]]: + """Lint a file and add the result to the passed or failed list.""" + + fn = Path(self.wf_path, file_path) + passed: List[str] = [] + failed: List[str] = [] + + ignore_configs = self.lint_config.get(lint_name, []) + + # Return a failed status if we can't find the file + if not fn.is_file(): + return {"ignored": [f"`${file_path}` not found"]} + + try: + with open(fn) as fh: + modules_config = fh.read() + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]} + + # check if removed sections are absent + + for section in removed_sections: + if section in modules_config and section not in ignore_configs: + failed.append(f"`${file_path}` contains `{section}`") + return {"passed": passed, "failed": failed} + else: + passed.append(f"`${file_path}` does not contain `{section}`") + + return {"passed": passed, "failed": failed} + + def print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj): """Print a joint summary of the general pipe lint tests and the module and subworkflow lint tests""" swf_passed = 0 @@ -101,3 +138,34 @@ def dump_json_with_prettier(file_name, file_content): with open(file_name, "w") as fh: json.dump(file_content, fh, indent=4) run_prettier_on_file(file_name) + + +def parse_config_file(self, lint_name: str, file_path: Path) -> Tuple[dict, dict]: + """Parse different kind of config files and return a dict.""" + + # Remove field that should be ignored according to the linting config + ignore_configs = self.lint_config.get(lint_name, []) + + fn = Path(self.wf_path, file_path) + + # Return a failed status if we can't find the file + if not fn.is_file(): + return {"ignored": [f"`${file_path}` not found"]}, ignore_configs + + try: + if fn.suffix == ".json": + import json + + with open(fn) as fh: + config = json.load(fh) + return config, ignore_configs + elif fn.suffix == ".yml" or fn.suffix == ".yaml": + import yaml + + with open(fn) as fh: + config = yaml.safe_load(fh) + return config, ignore_configs + else: + return {"failed": [f"Could not parse file: {fn}, unknown file type"]}, ignore_configs + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]}, ignore_configs From e9de4d9fd9cbc0268e9a48bd058fc09e89cccbba Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 26 Mar 2024 16:25:36 +0000 Subject: [PATCH 139/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e740d2af..adceeca6e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) +- Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From e9d0895144d685e296eb5f5b56bedc6274fb876f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 18:22:46 +0100 Subject: [PATCH 140/270] rewrite everything in a different way: Part III --- .../_src/pipeline_lint_tests/base_config.md | 5 ++ .../pipeline_lint_tests/modules_config.md | 5 ++ nf_core/lint/__init__.py | 3 +- nf_core/lint/base_config.py | 14 ---- nf_core/lint/configs.py | 64 +++++++++++++++++++ nf_core/lint/modules_config.py | 14 ---- nf_core/lint_utils.py | 38 +---------- nf_core/utils.py | 25 ++++---- 8 files changed, 89 insertions(+), 79 deletions(-) create mode 100644 docs/api/_src/pipeline_lint_tests/base_config.md create mode 100644 docs/api/_src/pipeline_lint_tests/modules_config.md delete mode 100644 nf_core/lint/base_config.py create mode 100644 nf_core/lint/configs.py delete mode 100644 nf_core/lint/modules_config.py diff --git a/docs/api/_src/pipeline_lint_tests/base_config.md b/docs/api/_src/pipeline_lint_tests/base_config.md new file mode 100644 index 0000000000..4a56ef9789 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/base_config.md @@ -0,0 +1,5 @@ +# base_config + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.base_config +``` diff --git a/docs/api/_src/pipeline_lint_tests/modules_config.md b/docs/api/_src/pipeline_lint_tests/modules_config.md new file mode 100644 index 0000000000..2a4f51c5a4 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/modules_config.md @@ -0,0 +1,5 @@ +# modules_config + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.modules_config +``` diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 92a699ac1c..aaa46d0424 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -199,11 +199,10 @@ class PipelineLint(nf_core.utils.Pipeline): from .actions_schema_validation import ( # type: ignore[misc] actions_schema_validation, ) - from .base_config import base_config # type: ignore[misc] + from .configs import base_config, modules_config # type: ignore[misc] from .files_exist import files_exist # type: ignore[misc] from .files_unchanged import files_unchanged # type: ignore[misc] from .merge_markers import merge_markers # type: ignore[misc] - from .modules_config import modules_config # type: ignore[misc] from .modules_json import modules_json # type: ignore[misc] from .modules_structure import modules_structure # type: ignore[misc] from .multiqc_config import multiqc_config # type: ignore[misc] diff --git a/nf_core/lint/base_config.py b/nf_core/lint/base_config.py deleted file mode 100644 index cd1385b1ca..0000000000 --- a/nf_core/lint/base_config.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path -from typing import Dict, List - -from nf_core.lint_utils import LintFile - - -def base_config(self) -> Dict[str, List[str]]: - """Make sure the conf/base.config file follows the nf-core template, especially removed sections.""" - - result = LintFile(self.wf_path, self.lint_config).lint_file( - "base_config", Path("conf", "base.config"), ["withName:CUSTOM_DUMPSOFTWAREVERSIONS"] - ) - - return result diff --git a/nf_core/lint/configs.py b/nf_core/lint/configs.py new file mode 100644 index 0000000000..5cae126bb9 --- /dev/null +++ b/nf_core/lint/configs.py @@ -0,0 +1,64 @@ +from pathlib import Path +from typing import Dict, List + + +class LintConfig: + def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): + self.wf_path = wf_path + self.lint_config = lint_config + + def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: + """Lint a file and add the result to the passed or failed list.""" + + fn = Path(self.wf_path, file_path) + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + + ignore_configs = self.lint_config.get(lint_name, []) + + # Return a failed status if we can't find the file + if not fn.is_file(): + return {"failed": [f"`${file_path}` not found"]} + + try: + with open(fn) as fh: + config = fh.read() + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]} + + # find sections with a withName: prefix + sections = [line.split(":")[1].strip().split(" ")[0] for line in config.split("\n") if "withName:" in line] + + # find all .nf files in the workflow directory + nf_files = list(Path(self.wf_path).rglob("*.nf")) + + # check if withName sections are present in config, but not in workflow files + for section in sections: + if section not in ignore_configs: + if not any(section in nf_file.read_text() for nf_file in nf_files): + failed.append( + f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the following workflow files: `{nf_files}`." + ) + else: + passed.append(f"both `{file_path}` and `{[str(f) for f in nf_files]} contain `{section}`.") + else: + ignored.append(f"``{section}` is ignored") + + return {"passed": passed, "failed": failed} + + +def modules_config(self) -> Dict[str, List[str]]: + """Make sure the conf/modules.config file follows the nf-core template, especially removed sections.""" + + result = LintConfig(self.wf_path, self.lint_config).lint_file("modules_config", Path("conf", "modules.config")) + + return result + + +def base_config(self) -> Dict[str, List[str]]: + """Make sure the conf/base.config file follows the nf-core template, especially removed sections.""" + + result = LintConfig(self.wf_path, self.lint_config).lint_file("base_config", Path("conf", "base.config")) + + return result diff --git a/nf_core/lint/modules_config.py b/nf_core/lint/modules_config.py deleted file mode 100644 index 53c58766ea..0000000000 --- a/nf_core/lint/modules_config.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path -from typing import Dict, List - -from nf_core.lint_utils import LintFile - - -def modules_config(self) -> Dict[str, List[str]]: - """Make sure the conf/modules.config file follows the nf-core template, especially removed sections.""" - - result = LintFile(self.wf_path, self.lint_config).lint_file( - "modules_config", Path("conf", "modules.config"), ["withName:CUSTOM_DUMPSOFTWAREVERSIONS"] - ) - - return result diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index e099fcabf3..27fb36568f 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -2,7 +2,7 @@ import logging import subprocess from pathlib import Path -from typing import Dict, List, Tuple +from typing import Tuple import rich from rich.console import Console @@ -17,42 +17,6 @@ console = Console(force_terminal=nf_core.utils.rich_force_colors()) -class LintFile: - def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): - self.wf_path = wf_path - self.lint_config = lint_config - - def lint_file(self, lint_name: str, file_path: Path, removed_sections: List[str]) -> Dict[str, List[str]]: - """Lint a file and add the result to the passed or failed list.""" - - fn = Path(self.wf_path, file_path) - passed: List[str] = [] - failed: List[str] = [] - - ignore_configs = self.lint_config.get(lint_name, []) - - # Return a failed status if we can't find the file - if not fn.is_file(): - return {"ignored": [f"`${file_path}` not found"]} - - try: - with open(fn) as fh: - modules_config = fh.read() - except Exception as e: - return {"failed": [f"Could not parse file: {fn}, {e}"]} - - # check if removed sections are absent - - for section in removed_sections: - if section in modules_config and section not in ignore_configs: - failed.append(f"`${file_path}` contains `{section}`") - return {"passed": passed, "failed": failed} - else: - passed.append(f"`${file_path}` does not contain `{section}`") - - return {"passed": passed, "failed": failed} - - def print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj): """Print a joint summary of the general pipe lint tests and the module and subworkflow lint tests""" swf_passed = 0 diff --git a/nf_core/utils.py b/nf_core/utils.py index 4271e971a1..83136c114e 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -216,7 +216,7 @@ def is_pipeline_directory(wf_path): raise UserWarning(f"'{wf_path}' is not a pipeline - '{fn}' is missing") -def fetch_wf_config(wf_path, cache_config=True): +def fetch_wf_config(wf_path: str, cache_config: bool = True) -> dict: """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. @@ -236,13 +236,13 @@ def fetch_wf_config(wf_path, cache_config=True): cache_path = None # Nextflow home directory - use env var if set, or default to ~/.nextflow - nxf_home = os.environ.get("NXF_HOME", os.path.join(os.getenv("HOME"), ".nextflow")) + nxf_home = Path(os.environ.get("NXF_HOME", Path(os.getenv("HOME") or "", ".nextflow"))) # Build a cache directory if we can - if os.path.isdir(nxf_home): - cache_basedir = os.path.join(nxf_home, "nf-core") - if not os.path.isdir(cache_basedir): - os.mkdir(cache_basedir) + if nxf_home.is_dir(): + cache_basedir = Path(nxf_home, "nf-core") + if not cache_basedir.is_dir(): + cache_basedir.mkdir(parents=True, exist_ok=True) # If we're given a workflow object with a commit, see if we have a cached copy cache_fn = None @@ -250,7 +250,7 @@ def fetch_wf_config(wf_path, cache_config=True): concat_hash = "" for fn in ["nextflow.config", "main.nf"]: try: - with open(os.path.join(wf_path, fn), "rb") as fh: + with open(Path(wf_path, fn), "rb") as fh: concat_hash += hashlib.sha256(fh.read()).hexdigest() except FileNotFoundError: pass @@ -260,8 +260,8 @@ def fetch_wf_config(wf_path, cache_config=True): cache_fn = f"wf-config-cache-{bighash[:25]}.json" if cache_basedir and cache_fn: - cache_path = os.path.join(cache_basedir, cache_fn) - if os.path.isfile(cache_path) and cache_config is True: + cache_path = Path(cache_basedir, cache_fn) + if cache_path.is_file() and cache_config is True: log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path) as fh: try: @@ -286,10 +286,11 @@ def fetch_wf_config(wf_path, cache_config=True): # Scrape main.nf for additional parameter declarations # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. try: - main_nf = os.path.join(wf_path, "main.nf") - with open(main_nf) as fh: + main_nf = Path(wf_path, "main.nf") + with open(main_nf, "rb") as fh: for line in fh: - match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line) + line_str = line.decode("utf-8") + match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line_str) if match: config[match.group(1)] = "null" except FileNotFoundError as e: From f7abb401beada50acb64f3749e1c47d1c51d04f6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 18:23:46 +0100 Subject: [PATCH 141/270] add tests for linting tests --- tests/lint/configs.py | 43 +++++++++++++++++++++++++++++++++++++++++++ tests/test_lint.py | 5 +++++ 2 files changed, 48 insertions(+) create mode 100644 tests/lint/configs.py diff --git a/tests/lint/configs.py b/tests/lint/configs.py new file mode 100644 index 0000000000..a4bbe0147c --- /dev/null +++ b/tests/lint/configs.py @@ -0,0 +1,43 @@ +from pathlib import Path + +import nf_core.create +import nf_core.lint + + +def test_withname_in_modules_config(self): + """Tests finding withName in modules.config passes linting.""" + + new_pipeline = self._make_pipeline_copy() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["failed"]) == 0 + assert any(["contain `FASTQC`" in passed for passed in result["passed"]]) + + +def test_superfluous_withname_in_modules_config_fails(self): + """Tests finding withName in modules.config fails linting.""" + new_pipeline = self._make_pipeline_copy() + # Add withName to modules.config + modules_config = Path(new_pipeline) / "conf" / "modules.config" + with open(modules_config, "a") as f: + f.write("withName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["failed"]) == 1 + assert result["failed"][0].startswith("`conf/modules.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") + + +def test_superfluous_withname_in_base_config_fails(self): + """Tests finding withName in base.config fails linting.""" + new_pipeline = self._make_pipeline_copy() + # Add withName to base.config + base_config = Path(new_pipeline) / "conf" / "base.config" + with open(base_config, "a") as f: + f.write("withName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.base_config() + assert len(result["failed"]) == 1 + assert result["failed"][0].startswith("`conf/base.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") diff --git a/tests/test_lint.py b/tests/test_lint.py index d10cef37e4..290b9b73c7 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -198,6 +198,11 @@ def test_sphinx_md_files(self): test_actions_schema_validation_missing_jobs, test_actions_schema_validation_missing_on, ) + from .lint.configs import ( # type: ignore[misc] + test_superfluous_withname_in_base_config_fails, + test_superfluous_withname_in_modules_config_fails, + test_withname_in_modules_config, + ) from .lint.files_exist import ( # type: ignore[misc] test_files_exist_depreciated_file, test_files_exist_fail_conditional, From a7e5593a62ad3e93f28ddc9adfc0a84e65cebf51 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 18:50:34 +0100 Subject: [PATCH 142/270] revert remote changes --- nf_core/schema.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index a4b635c819..08c8403d85 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -66,13 +66,8 @@ def get_schema_path( # Path does not exist - assume a name of a remote workflow elif not local_only: - self.schema_filename = path / "nextflow_schema.json" - if revision is not None: - self.load_remote_schema( - f"https://raw.githubusercontent.com/nf-core/{path}/{revision}/nextflow_schema.json" - ) - else: - self.load_remote_schema(f"https://raw.githubusercontent.com/nf-core/{path}/master/nextflow_schema.json") + self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision) + self.schema_filename = Path(self.pipeline_dir or "", "nextflow_schema.json") # Only looking for local paths, overwrite with None to be safe else: @@ -121,19 +116,6 @@ def load_schema(self): self.schema_params = {} log.debug(f"JSON file loaded: {self.schema_filename}") - def load_remote_schema(self, url): - """Load a pipeline schema from a remote URL""" - import requests - - response = requests.get(url) - if response.status_code != 200: - raise AssertionError(f"Could not load schema from {url}") - self.schema = response.json() - self.schema_filename = url - self.schema_defaults = {} - self.schema_params = {} - log.debug(f"JSON file loaded: {self.schema_filename}") - def sanitise_param_default(self, param): """ Given a param, ensure that the default value is the correct variable type From 3fa8131a30cb60d727d4dd6a9b56454fe10d1e13 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 26 Mar 2024 19:02:08 +0100 Subject: [PATCH 143/270] fix tests --- nf_core/schema.py | 4 +++- tests/test_schema.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 08c8403d85..a4e8ad7cdc 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -68,7 +68,9 @@ def get_schema_path( elif not local_only: self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision) self.schema_filename = Path(self.pipeline_dir or "", "nextflow_schema.json") - + # check if the schema file exists + if not self.schema_filename.exists(): + self.schema_filename = None # Only looking for local paths, overwrite with None to be safe else: self.schema_filename = None diff --git a/tests/test_schema.py b/tests/test_schema.py index 5c28ee92bf..b9cb108fae 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -47,7 +47,7 @@ def test_load_lint_schema(self): def test_load_lint_schema_nofile(self): """Check that linting raises properly if a non-existant file is given""" - with pytest.raises(AssertionError): + with pytest.raises(RuntimeError): self.schema_obj.get_schema_path("fake_file") def test_load_lint_schema_notjson(self): From 0248740c255afbf75b0b7e6009d1afb0c83e0b7f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 00:56:45 +0000 Subject: [PATCH 144/270] chore(deps): update github actions --- .github/workflows/changelog.yml | 4 ++-- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/deploy-pypi.yml | 2 +- .github/workflows/fix-linting.yml | 2 +- .github/workflows/lint-code.yml | 2 +- .github/workflows/pytest.yml | 6 +++--- .github/workflows/rich-codex.yml | 2 +- .github/workflows/sync.yml | 2 +- .github/workflows/update_components_template.yml | 2 +- .../.github/workflows/download_pipeline.yml | 2 +- nf_core/pipeline-template/.github/workflows/fix-linting.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 4 ++-- .../.github/workflows/release-announcements.yml | 2 +- 15 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 50d835fd5c..bed104e2d9 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -36,7 +36,7 @@ jobs: fi gh pr checkout $PR_NUMBER - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" @@ -65,7 +65,7 @@ jobs: echo "File changed: ${{ env.changed }}" - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" cache: "pip" diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 3e93918ed1..996bdf2799 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -53,7 +53,7 @@ jobs: # Set up nf-core/tools - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" cache: pip diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 988e5bb5b0..4ef448e5ec 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -65,7 +65,7 @@ jobs: name: Check out source-code repository - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 3b7c02ca86..952181c044 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -52,7 +52,7 @@ jobs: name: Check out source-code repository - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index ca698dad1f..ec94ddb88f 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -17,7 +17,7 @@ jobs: name: Check out source-code repository - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 3e6c4d4ef6..b3f73c5a8c 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -32,7 +32,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} # Install and run pre-commit - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 53151a8989..51a543c143 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" cache: "pip" diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b03aab7cdf..93a44aea5e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -91,7 +91,7 @@ jobs: name: Check out source-code repository - name: Set up Python ${{ needs.setup.outputs.python-version }} - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: ${{ needs.setup.outputs.python-version }} cache: "pip" @@ -160,7 +160,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 env: AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/ with: @@ -184,7 +184,7 @@ jobs: coverage report coverage xml - - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4 + - uses: codecov/codecov-action@c16abc29c95fcf9174b58eb7e1abf4c866893bc8 # v4 with: files: coverage.xml env: diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 13fd71a883..fc8af8a7bc 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -8,7 +8,7 @@ jobs: - name: Check out the repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: 3.x cache: pip diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 9eeb32ccf3..2adb970f64 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -57,7 +57,7 @@ jobs: fetch-depth: "0" - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index 0fa3adaf20..c5413cd557 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.x" diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 3b791980f2..ebea16c5cb 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -35,7 +35,7 @@ jobs: - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" architecture: "x64" diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index e976b9f09c..bc15ce42e8 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -32,7 +32,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} # Install and run pre-commit - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 90cca8c2ce..a8208b31ac 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.12 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" cache: "pip" @@ -37,7 +37,7 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@v2 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.12" architecture: "x64" diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index 2c57060257..8fee061fdd 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.10" - name: Install dependencies From fe4383a61786756c6ac4d92f58322961206a7331 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 28 Mar 2024 02:23:07 +0000 Subject: [PATCH 145/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0a91902d3..1a3eb103bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) - Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) +- Update GitHub Actions ([#2902](https://github.com/nf-core/tools/pull/2902)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 97da643b4d968809a8853b079f44d39e91d52f10 Mon Sep 17 00:00:00 2001 From: Alex Hermida Date: Tue, 19 Mar 2024 22:18:52 +0100 Subject: [PATCH 146/270] Get immediate path parent name --- nf_core/schema.py | 4 +++- tests/test_schema.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index a4e8ad7cdc..4f5acfa0af 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -587,10 +587,12 @@ def make_skeleton_schema(self): loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) schema_template = env.get_template("nextflow_schema.json") + template_vars = { - "name": self.pipeline_manifest.get("name", str(Path(self.schema_filename).parent)).strip("'"), + "name": self.pipeline_manifest.get("name", Path(self.schema_filename).parent.name).strip("'"), "description": self.pipeline_manifest.get("description", "").strip("'"), } + self.schema = json.loads(schema_template.render(template_vars)) self.get_schema_defaults() diff --git a/tests/test_schema.py b/tests/test_schema.py index b9cb108fae..a05fdf6b92 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -218,6 +218,13 @@ def test_make_skeleton_schema(self): self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) + def test_make_skeleton_schema_absent_name(self): + """Test making a new schema skeleton""" + self.schema_obj.schema_filename = self.template_schema + self.schema_obj.pipeline_manifest["description"] = "Test pipeline" + self.schema_obj.make_skeleton_schema() + self.schema_obj.validate_schema(self.schema_obj.schema) + def test_get_wf_params(self): """Test getting the workflow parameters from a pipeline""" self.schema_obj.schema_filename = self.template_schema From b9c4746ff8bb648c4e84f6f38bd868678df7ade0 Mon Sep 17 00:00:00 2001 From: Alex Hermida Date: Tue, 19 Mar 2024 22:34:08 +0100 Subject: [PATCH 147/270] Update Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac60a404e4..79d29ca32d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ - Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) - Update GitHub Actions ([#2902](https://github.com/nf-core/tools/pull/2902)) +- Get immediate parent path name for schema creation ([#2886](https://github.com/nf-core/tools/pull/2886)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 31240de0d2c826c120813a9e84237ba7a77a4d83 Mon Sep 17 00:00:00 2001 From: Alex Hermida Date: Wed, 20 Mar 2024 14:56:19 +0100 Subject: [PATCH 148/270] Update tests --- tests/test_schema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_schema.py b/tests/test_schema.py index a05fdf6b92..b69b61762f 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -217,6 +217,7 @@ def test_make_skeleton_schema(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) + assert self.schema_obj.schema['title'] == "nf-core/test pipeline parameters" def test_make_skeleton_schema_absent_name(self): """Test making a new schema skeleton""" @@ -224,6 +225,7 @@ def test_make_skeleton_schema_absent_name(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) + assert self.schema_obj.schema['title'] == "wf pipeline parameters" def test_get_wf_params(self): """Test getting the workflow parameters from a pipeline""" From 2975f6af13bd4476739aadb499d6c707ddefa02a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 20 Mar 2024 14:23:08 +0000 Subject: [PATCH 149/270] [automated] Fix code linting --- tests/test_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index b69b61762f..e0921908d4 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -217,7 +217,7 @@ def test_make_skeleton_schema(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) - assert self.schema_obj.schema['title'] == "nf-core/test pipeline parameters" + assert self.schema_obj.schema["title"] == "nf-core/test pipeline parameters" def test_make_skeleton_schema_absent_name(self): """Test making a new schema skeleton""" @@ -225,7 +225,7 @@ def test_make_skeleton_schema_absent_name(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) - assert self.schema_obj.schema['title'] == "wf pipeline parameters" + assert self.schema_obj.schema["title"] == "wf pipeline parameters" def test_get_wf_params(self): """Test getting the workflow parameters from a pipeline""" From 654a205590d8660af66e1390c494359e1b8aeec7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 18:49:57 +0000 Subject: [PATCH 150/270] Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ab0dceba2..915bbc345d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.4 + rev: v0.3.5 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 5d9cb5af53983baa89f00eacd38d3999216980a5 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 1 Apr 2024 18:50:51 +0000 Subject: [PATCH 151/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac60a404e4..cbfef3bcc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ - Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) - Update GitHub Actions ([#2902](https://github.com/nf-core/tools/pull/2902)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 ([#2903](https://github.com/nf-core/tools/pull/2903)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 9af060d6b37499be9777941990f9b7b19f82d9a6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 2 Apr 2024 10:23:23 +0200 Subject: [PATCH 152/270] update according to review comments --- nf_core/lint/configs.py | 48 +++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/nf_core/lint/configs.py b/nf_core/lint/configs.py index 5cae126bb9..ef8be27e87 100644 --- a/nf_core/lint/configs.py +++ b/nf_core/lint/configs.py @@ -1,6 +1,10 @@ +import logging +import re from pathlib import Path from typing import Dict, List +log = logging.getLogger(__name__) + class LintConfig: def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): @@ -19,7 +23,10 @@ def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: # Return a failed status if we can't find the file if not fn.is_file(): - return {"failed": [f"`${file_path}` not found"]} + if ignore_configs: + return {"ignored": [f"`{file_path}` not found, but it is ignored."]} + else: + return {"failed": [f"`${file_path}` not found"]} try: with open(fn) as fh: @@ -28,14 +35,15 @@ def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: return {"failed": [f"Could not parse file: {fn}, {e}"]} # find sections with a withName: prefix - sections = [line.split(":")[1].strip().split(" ")[0] for line in config.split("\n") if "withName:" in line] + sections = re.findall(r"['\"](.*)['\"]", config) # find all .nf files in the workflow directory nf_files = list(Path(self.wf_path).rglob("*.nf")) + log.debug(f"found nf_files: {nf_files}") # check if withName sections are present in config, but not in workflow files for section in sections: - if section not in ignore_configs: + if section not in ignore_configs or section.lower() not in ignore_configs: if not any(section in nf_file.read_text() for nf_file in nf_files): failed.append( f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the following workflow files: `{nf_files}`." @@ -45,11 +53,29 @@ def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: else: ignored.append(f"``{section}` is ignored") - return {"passed": passed, "failed": failed} + return {"passed": passed, "failed": failed, "ignored": ignored} def modules_config(self) -> Dict[str, List[str]]: - """Make sure the conf/modules.config file follows the nf-core template, especially removed sections.""" + """Make sure the conf/modules.config file follows the nf-core template, especially removed sections. + + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: + + .. code-block:: yaml + + lint: + modules_config: False + + To disable this test only for specific modules, you can specify a list of module names. + + .. code-block:: yaml + + lint: + modules_config: + - fastqc + + """ result = LintConfig(self.wf_path, self.lint_config).lint_file("modules_config", Path("conf", "modules.config")) @@ -57,7 +83,17 @@ def modules_config(self) -> Dict[str, List[str]]: def base_config(self) -> Dict[str, List[str]]: - """Make sure the conf/base.config file follows the nf-core template, especially removed sections.""" + """Make sure the conf/base.config file follows the nf-core template, especially removed sections. + + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: + + .. code-block:: yaml + + lint: + base_config: False + + """ result = LintConfig(self.wf_path, self.lint_config).lint_file("base_config", Path("conf", "base.config")) From 6c717dcb2189353a7e25e63cbfd9740c2274407e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 2 Apr 2024 17:33:15 +0200 Subject: [PATCH 153/270] combine file check and add tests --- nf_core/lint/configs.py | 67 +++++------ nf_core/lint/multiqc_config.py | 196 +++++++++++++++++---------------- nf_core/lint_utils.py | 52 +++++---- nf_core/utils.py | 9 +- tests/lint/configs.py | 54 ++++++++- tests/lint/multiqc_config.py | 45 ++++++-- tests/test_lint.py | 4 +- 7 files changed, 255 insertions(+), 172 deletions(-) diff --git a/nf_core/lint/configs.py b/nf_core/lint/configs.py index ef8be27e87..2741529198 100644 --- a/nf_core/lint/configs.py +++ b/nf_core/lint/configs.py @@ -3,6 +3,8 @@ from pathlib import Path from typing import Dict, List +from nf_core.lint_utils import ignore_file + log = logging.getLogger(__name__) @@ -14,44 +16,43 @@ def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: """Lint a file and add the result to the passed or failed list.""" - fn = Path(self.wf_path, file_path) passed: List[str] = [] failed: List[str] = [] ignored: List[str] = [] + ignore_configs: List[str] = [] + + fn = Path(self.wf_path, file_path) - ignore_configs = self.lint_config.get(lint_name, []) - - # Return a failed status if we can't find the file - if not fn.is_file(): - if ignore_configs: - return {"ignored": [f"`{file_path}` not found, but it is ignored."]} - else: - return {"failed": [f"`${file_path}` not found"]} - - try: - with open(fn) as fh: - config = fh.read() - except Exception as e: - return {"failed": [f"Could not parse file: {fn}, {e}"]} - - # find sections with a withName: prefix - sections = re.findall(r"['\"](.*)['\"]", config) - - # find all .nf files in the workflow directory - nf_files = list(Path(self.wf_path).rglob("*.nf")) - log.debug(f"found nf_files: {nf_files}") - - # check if withName sections are present in config, but not in workflow files - for section in sections: - if section not in ignore_configs or section.lower() not in ignore_configs: - if not any(section in nf_file.read_text() for nf_file in nf_files): - failed.append( - f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the following workflow files: `{nf_files}`." - ) + passed, failed, ignored, ignore_configs = ignore_file(lint_name, file_path, Path(self.wf_path)) + + error_message = f"`{file_path}` not found" + # check for partial match in failed or ignored + if not any(f.startswith(error_message) for f in (failed + ignored)): + try: + with open(fn) as fh: + config = fh.read() + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]} + + # find sections with a withName: prefix + sections = re.findall(r"withName:\s*['\"]?(\w+)['\"]?", config) + log.debug(f"found sections: {sections}") + + # find all .nf files in the workflow directory + nf_files = list(Path(self.wf_path).rglob("*.nf")) + log.debug(f"found nf_files: {[str(f) for f in nf_files]}") + + # check if withName sections are present in config, but not in workflow files + for section in sections: + if section not in ignore_configs or section.lower() not in ignore_configs: + if not any(section in nf_file.read_text() for nf_file in nf_files): + failed.append( + f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the Nextflow scripts." + ) + else: + passed.append(f"`{section}` found in `{file_path}` and Nextflow scripts.") else: - passed.append(f"both `{file_path}` and `{[str(f) for f in nf_files]} contain `{section}`.") - else: - ignored.append(f"``{section}` is ignored") + ignored.append(f"``{section}` is ignored.") return {"passed": passed, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index 9b9c80c44e..cd281dd5c9 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -1,8 +1,10 @@ -import os +from pathlib import Path from typing import Dict, List import yaml +from nf_core.lint_utils import ignore_file + def multiqc_config(self) -> Dict[str, List[str]]: """Make sure basic multiQC plugins are installed and plots are exported @@ -21,100 +23,110 @@ def multiqc_config(self) -> Dict[str, List[str]]: order: -1001 export_plots: true - """ - - passed: List[str] = [] - failed: List[str] = [] - - # Remove field that should be ignored according to the linting config - ignore_configs = self.lint_config.get("multiqc_config", []) + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: - fn = os.path.join(self.wf_path, "assets", "multiqc_config.yml") + .. code-block:: yaml - # Return a failed status if we can't find the file - if not os.path.isfile(fn): - return {"ignored": ["'assets/multiqc_config.yml' not found"]} + lint: + multiqc_config: False - try: - with open(fn) as fh: - mqc_yml = yaml.safe_load(fh) - except Exception as e: - return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} - - # check if requried sections are present - required_sections = ["report_section_order", "export_plots", "report_comment"] - for section in required_sections: - if section not in mqc_yml and section not in ignore_configs: - failed.append(f"'assets/multiqc_config.yml' does not contain `{section}`") - return {"passed": passed, "failed": failed} - else: - passed.append(f"'assets/multiqc_config.yml' contains `{section}`") - - try: - orders = {} - summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" - min_plugins = ["software_versions", summary_plugin_name] - for plugin in min_plugins: - if plugin not in mqc_yml["report_section_order"]: - raise AssertionError(f"Section {plugin} missing in report_section_order") - if "order" not in mqc_yml["report_section_order"][plugin]: - raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0") - plugin_order = mqc_yml["report_section_order"][plugin]["order"] - if plugin_order >= 0: - raise AssertionError(f"Section {plugin} 'order' must be < 0") - - for plugin in mqc_yml["report_section_order"]: - if "order" in mqc_yml["report_section_order"][plugin]: - orders[plugin] = mqc_yml["report_section_order"][plugin]["order"] - - if orders[summary_plugin_name] != min(orders.values()): - raise AssertionError(f"Section {summary_plugin_name} should have the lowest order") - orders.pop(summary_plugin_name) - if orders["software_versions"] != min(orders.values()): - raise AssertionError("Section software_versions should have the second lowest order") - except (AssertionError, KeyError, TypeError) as e: - failed.append(f"'assets/multiqc_config.yml' does not meet requirements: {e}") - else: - passed.append("'assets/multiqc_config.yml' follows the ordering scheme of the minimally required plugins.") - - if "report_comment" not in ignore_configs: - # Check that the minimum plugins exist and are coming first in the summary - version = self.nf_config.get("manifest.version", "").strip(" '\"") - if "dev" in version: - version = "dev" - report_comments = ( - f'This report has been generated by the nf-core/{self.pipeline_name}' - f" analysis pipeline. For information about how to interpret these results, please see the " - f'documentation.' - ) + """ + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + + fn = Path(self.wf_path, "assets", "multiqc_config.yml") + file_path = fn.relative_to(self.wf_path) + passed, failed, ignored, ignore_configs = ignore_file("multiqc_config", file_path, self.wf_path) + print(f"passed: {passed}") + print(f"failed: {failed}") + print(f"ignored: {ignored}") + # skip other tests if the file is not found + error_message = f"`{file_path}` not found" + # check for partial match in failed or ignored + if not any(f.startswith(error_message) for f in (failed + ignored)): + try: + with open(fn) as fh: + mqc_yml = yaml.safe_load(fh) + except Exception as e: + return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} + + # check if required sections are present + required_sections = ["report_section_order", "export_plots", "report_comment"] + for section in required_sections: + if section not in mqc_yml and section not in ignore_configs: + failed.append(f"`assets/multiqc_config.yml` does not contain `{section}`") + return {"passed": passed, "failed": failed} + else: + passed.append(f"`assets/multiqc_config.yml` contains `{section}`") + + try: + orders = {} + summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" + min_plugins = ["software_versions", summary_plugin_name] + for plugin in min_plugins: + if plugin not in mqc_yml["report_section_order"]: + raise AssertionError(f"Section {plugin} missing in report_section_order") + if "order" not in mqc_yml["report_section_order"][plugin]: + raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0") + plugin_order = mqc_yml["report_section_order"][plugin]["order"] + if plugin_order >= 0: + raise AssertionError(f"Section {plugin} 'order' must be < 0") + + for plugin in mqc_yml["report_section_order"]: + if "order" in mqc_yml["report_section_order"][plugin]: + orders[plugin] = mqc_yml["report_section_order"][plugin]["order"] + + if orders[summary_plugin_name] != min(orders.values()): + raise AssertionError(f"Section {summary_plugin_name} should have the lowest order") + orders.pop(summary_plugin_name) + if orders["software_versions"] != min(orders.values()): + raise AssertionError("Section software_versions should have the second lowest order") + except (AssertionError, KeyError, TypeError) as e: + failed.append(f"`assets/multiqc_config.yml` does not meet requirements: {e}") else: - report_comments = ( - f'This report has been generated by the nf-core/{self.pipeline_name}' - f" analysis pipeline. For information about how to interpret these results, please see the " - f'documentation.' - ) - - if mqc_yml["report_comment"].strip() != report_comments: - # find where the report_comment is wrong and give it as a hint - hint = report_comments - failed.append( - f"'assets/multiqc_config.yml' does not contain a matching 'report_comment'. \n" - f"The expected comment is: \n" - f"```{hint}``` \n" - f"The current comment is: \n" - f"```{ mqc_yml['report_comment'].strip()}```" - ) + passed.append("`assets/multiqc_config.yml` follows the ordering scheme of the minimally required plugins.") + + if "report_comment" not in ignore_configs: + # Check that the minimum plugins exist and are coming first in the summary + version = self.nf_config.get("manifest.version", "").strip(" '\"") + if "dev" in version: + version = "dev" + report_comments = ( + f'This report has been generated by the nf-core/{self.pipeline_name}' + f" analysis pipeline. For information about how to interpret these results, please see the " + f'documentation.' + ) + + else: + report_comments = ( + f'This report has been generated by the nf-core/{self.pipeline_name}' + f" analysis pipeline. For information about how to interpret these results, please see the " + f'documentation.' + ) + + if mqc_yml["report_comment"].strip() != report_comments: + # find where the report_comment is wrong and give it as a hint + hint = report_comments + failed.append( + f"`assets/multiqc_config.yml` does not contain a matching 'report_comment'. \n" + f"The expected comment is: \n" + f"```{hint}``` \n" + f"The current comment is: \n" + f"```{ mqc_yml['report_comment'].strip()}```" + ) + else: + passed.append("`assets/multiqc_config.yml` contains a matching 'report_comment'.") + + # Check that export_plots is activated + try: + if not mqc_yml["export_plots"]: + raise AssertionError() + except (AssertionError, KeyError, TypeError): + failed.append("`assets/multiqc_config.yml` does not contain 'export_plots: true'.") else: - passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.") - - # Check that export_plots is activated - try: - if not mqc_yml["export_plots"]: - raise AssertionError() - except (AssertionError, KeyError, TypeError): - failed.append("'assets/multiqc_config.yml' does not contain 'export_plots: true'.") - else: - passed.append("'assets/multiqc_config.yml' contains 'export_plots: true'.") - - return {"passed": passed, "failed": failed} + passed.append("`assets/multiqc_config.yml` contains 'export_plots: true'.") + + return {"passed": passed, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 27fb36568f..0ca84c46aa 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -2,7 +2,7 @@ import logging import subprocess from pathlib import Path -from typing import Tuple +from typing import List import rich from rich.console import Console @@ -104,32 +104,30 @@ def dump_json_with_prettier(file_name, file_content): run_prettier_on_file(file_name) -def parse_config_file(self, lint_name: str, file_path: Path) -> Tuple[dict, dict]: - """Parse different kind of config files and return a dict.""" - - # Remove field that should be ignored according to the linting config - ignore_configs = self.lint_config.get(lint_name, []) - - fn = Path(self.wf_path, file_path) +def ignore_file(lint_name: str, file_path: Path, dir_path: Path) -> List[List[str]]: + """Ignore a file and add the result to the ignored list. Return the passed, failed, ignored and ignore_configs lists.""" + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + _, lint_conf = nf_core.utils.load_tools_config(dir_path) + lint_conf = lint_conf.get("lint", {}) + print(f"lint_conf: {lint_conf}") + ignore_entry: List[str] | bool = lint_conf.get(lint_name, []) + print(f"ignore_entry: {ignore_entry}") + full_path = dir_path / file_path # Return a failed status if we can't find the file - if not fn.is_file(): - return {"ignored": [f"`${file_path}` not found"]}, ignore_configs - - try: - if fn.suffix == ".json": - import json - - with open(fn) as fh: - config = json.load(fh) - return config, ignore_configs - elif fn.suffix == ".yml" or fn.suffix == ".yaml": - import yaml - - with open(fn) as fh: - config = yaml.safe_load(fh) - return config, ignore_configs + if not full_path.is_file(): + if isinstance(ignore_entry, bool) and not ignore_entry: + ignored.append(f"`{file_path}` not found, but it is ignored.") + ignore_entry = [] else: - return {"failed": [f"Could not parse file: {fn}, unknown file type"]}, ignore_configs - except Exception as e: - return {"failed": [f"Could not parse file: {fn}, {e}"]}, ignore_configs + failed.append(f"`{file_path}` not found.") + else: + passed.append(f"`{file_path}` found and not ignored.") + + # we handled the only case where ignore_entry should be a bool, convert it to a list, to make downstream code easier + if isinstance(ignore_entry, bool): + ignore_entry = [] + + return [passed, failed, ignored, ignore_entry] diff --git a/nf_core/utils.py b/nf_core/utils.py index 83136c114e..6af2f25165 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -213,7 +213,11 @@ def is_pipeline_directory(wf_path): for fn in ["main.nf", "nextflow.config"]: path = os.path.join(wf_path, fn) if not os.path.isfile(path): - raise UserWarning(f"'{wf_path}' is not a pipeline - '{fn}' is missing") + if wf_path == ".": + warning = f"Current directory is not a pipeline - '{fn}' is missing." + else: + warning = f"'{wf_path}' is not a pipeline - '{fn}' is missing." + raise UserWarning(warning) def fetch_wf_config(wf_path: str, cache_config: bool = True) -> dict: @@ -1013,7 +1017,7 @@ def get_repo_releases_branches(pipeline, wfs): DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"] -def load_tools_config(directory: Union[str, Path] = "."): +def load_tools_config(directory: Union[str, Path] = ".") -> Tuple[Path, dict]: """ Parse the nf-core.yml configuration file @@ -1041,7 +1045,6 @@ def load_tools_config(directory: Union[str, Path] = "."): with open(config_fn) as fh: tools_config = yaml.safe_load(fh) - # If the file is empty tools_config = tools_config or {} diff --git a/tests/lint/configs.py b/tests/lint/configs.py index a4bbe0147c..d5dcd27fad 100644 --- a/tests/lint/configs.py +++ b/tests/lint/configs.py @@ -1,5 +1,7 @@ from pathlib import Path +import yaml + import nf_core.create import nf_core.lint @@ -12,7 +14,9 @@ def test_withname_in_modules_config(self): lint_obj._load() result = lint_obj.modules_config() assert len(result["failed"]) == 0 - assert any(["contain `FASTQC`" in passed for passed in result["passed"]]) + assert any( + ["`FASTQC` found in `conf/modules.config` and Nextflow scripts." in passed for passed in result["passed"]] + ) def test_superfluous_withname_in_modules_config_fails(self): @@ -21,12 +25,32 @@ def test_superfluous_withname_in_modules_config_fails(self): # Add withName to modules.config modules_config = Path(new_pipeline) / "conf" / "modules.config" with open(modules_config, "a") as f: - f.write("withName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") - lint_obj = nf_core.lint.PipelineLint(new_pipeline) + f.write("\nwithName: 'BPIPE' {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline, hide_progress=False) lint_obj._load() result = lint_obj.modules_config() assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("`conf/modules.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") + assert result["failed"][0].startswith("`conf/modules.config` contains `withName:BPIPE`") + + +def test_ignore_modules_config(self): + """Tests ignoring the modules.config passes linting.""" + new_pipeline = self._make_pipeline_copy() + # ignore modules.config in linting + with open(Path(new_pipeline) / ".nf-core.yml") as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"modules_config": False} + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(content, f) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["ignored"]) == 1 + assert result["ignored"][0].startswith("`conf/modules.config` not found, but it is ignored.") + # cleanup + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(old_content, f) def test_superfluous_withname_in_base_config_fails(self): @@ -35,9 +59,29 @@ def test_superfluous_withname_in_base_config_fails(self): # Add withName to base.config base_config = Path(new_pipeline) / "conf" / "base.config" with open(base_config, "a") as f: - f.write("withName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") + f.write("\nwithName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() result = lint_obj.base_config() assert len(result["failed"]) == 1 assert result["failed"][0].startswith("`conf/base.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") + + +def test_ignore_base_config(self): + """Tests ignoring the base.config passes linting.""" + new_pipeline = self._make_pipeline_copy() + # ignore base.config in linting + with open(Path(new_pipeline) / ".nf-core.yml") as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"base_config": False} + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(content, f) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.base_config() + assert len(result["ignored"]) == 1 + assert result["ignored"][0].startswith("`conf/base.config` not found, but it is ignored.") + # cleanup + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(old_content, f) diff --git a/tests/lint/multiqc_config.py b/tests/lint/multiqc_config.py index 721560ce81..463d5e7654 100644 --- a/tests/lint/multiqc_config.py +++ b/tests/lint/multiqc_config.py @@ -5,14 +5,37 @@ import nf_core.lint -def test_multiqc_config_exists_ignore(self): +def test_multiqc_config_exists(self): """Test that linting fails if the multiqc_config.yml file is missing""" # Delete the file new_pipeline = self._make_pipeline_copy() Path(Path(new_pipeline, "assets", "multiqc_config.yml")).unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() result = lint_obj.multiqc_config() - assert result["ignored"] == ["'assets/multiqc_config.yml' not found"] + assert result["failed"] == ["`assets/multiqc_config.yml` not found."] + + +def test_multiqc_config_ignore(self): + """Test that linting succeeds if the multiqc_config.yml file is missing but ignored""" + # Delete the file + new_pipeline = self._make_pipeline_copy() + Path(Path(new_pipeline, "assets", "multiqc_config.yml")).unlink() + with open(Path(new_pipeline, ".nf-core.yml")) as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"multiqc_config": False} + with open(Path(new_pipeline, ".nf-core.yml"), "w") as f: + yaml.dump(content, f) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.multiqc_config() + assert result["ignored"] == ["`assets/multiqc_config.yml` not found, but it is ignored."] + + # cleanup + with open(Path(new_pipeline, ".nf-core.yml"), "w") as f: + yaml.dump(old_content, f) def test_multiqc_config_missing_report_section_order(self): @@ -20,7 +43,7 @@ def test_multiqc_config_missing_report_section_order(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml.pop("report_section_order") with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -30,7 +53,7 @@ def test_multiqc_config_missing_report_section_order(self): # Reset the file with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) - assert result["failed"] == ["'assets/multiqc_config.yml' does not contain `report_section_order`"] + assert result["failed"] == ["`assets/multiqc_config.yml` does not contain `report_section_order`"] def test_multiqc_incorrect_export_plots(self): @@ -38,7 +61,7 @@ def test_multiqc_incorrect_export_plots(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml["export_plots"] = False with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -48,7 +71,7 @@ def test_multiqc_incorrect_export_plots(self): # Reset the file with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) - assert result["failed"] == ["'assets/multiqc_config.yml' does not contain 'export_plots: true'."] + assert result["failed"] == ["`assets/multiqc_config.yml` does not contain 'export_plots: true'."] def test_multiqc_config_report_comment_fail(self): @@ -56,7 +79,7 @@ def test_multiqc_config_report_comment_fail(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml["report_comment"] = "This is a test" with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -67,7 +90,7 @@ def test_multiqc_config_report_comment_fail(self): with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + assert result["failed"][0].startswith("`assets/multiqc_config.yml` does not contain a matching 'report_comment'.") def test_multiqc_config_report_comment_release_fail(self): @@ -75,7 +98,7 @@ def test_multiqc_config_report_comment_release_fail(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -87,7 +110,7 @@ def test_multiqc_config_report_comment_release_fail(self): with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + assert result["failed"][0].startswith("`assets/multiqc_config.yml` does not contain a matching 'report_comment'.") def test_multiqc_config_report_comment_release_succeed(self): @@ -103,4 +126,4 @@ def test_multiqc_config_report_comment_release_succeed(self): # lint again lint_obj._load() result = lint_obj.multiqc_config() - assert "'assets/multiqc_config.yml' contains a matching 'report_comment'." in result["passed"] + assert "`assets/multiqc_config.yml` contains a matching 'report_comment'." in result["passed"] diff --git a/tests/test_lint.py b/tests/test_lint.py index 290b9b73c7..51c52fd54e 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -199,6 +199,7 @@ def test_sphinx_md_files(self): test_actions_schema_validation_missing_on, ) from .lint.configs import ( # type: ignore[misc] + test_ignore_modules_config, test_superfluous_withname_in_base_config_fails, test_superfluous_withname_in_modules_config_fails, test_withname_in_modules_config, @@ -218,7 +219,8 @@ def test_sphinx_md_files(self): from .lint.merge_markers import test_merge_markers_found # type: ignore[misc] from .lint.modules_json import test_modules_json_pass # type: ignore[misc] from .lint.multiqc_config import ( # type: ignore[misc] - test_multiqc_config_exists_ignore, + test_multiqc_config_exists, + test_multiqc_config_ignore, test_multiqc_config_missing_report_section_order, test_multiqc_config_report_comment_fail, test_multiqc_config_report_comment_release_fail, From cac553c5f1d7d5be3a98244b1d6d6e1ccfb655d9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 18:16:45 +0000 Subject: [PATCH 154/270] Update gitpod/workspace-base Docker digest to 450d764 --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 40aa998a50..e476a7f140 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:168d78bc249332f9437b11a1844819308ac7ec1bc6bab61bf5adf8cbe53bf8e6 +FROM gitpod/workspace-base@sha256:450d764c8575345b32e3e22d197b5150152eff9f9a14482003f70f05d03f726a USER root From 0f9da5d421947f9a7fb72dccced41cd1257940a0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 3 Apr 2024 09:39:38 +0200 Subject: [PATCH 155/270] fix tests --- nf_core/lint/multiqc_config.py | 4 +--- nf_core/lint_utils.py | 2 -- tests/lint/configs.py | 2 ++ tests/test_lint.py | 1 + 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index cd281dd5c9..8b4fa2120f 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -40,9 +40,7 @@ def multiqc_config(self) -> Dict[str, List[str]]: fn = Path(self.wf_path, "assets", "multiqc_config.yml") file_path = fn.relative_to(self.wf_path) passed, failed, ignored, ignore_configs = ignore_file("multiqc_config", file_path, self.wf_path) - print(f"passed: {passed}") - print(f"failed: {failed}") - print(f"ignored: {ignored}") + # skip other tests if the file is not found error_message = f"`{file_path}` not found" # check for partial match in failed or ignored diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 0ca84c46aa..167600bfc0 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -112,9 +112,7 @@ def ignore_file(lint_name: str, file_path: Path, dir_path: Path) -> List[List[st ignored: List[str] = [] _, lint_conf = nf_core.utils.load_tools_config(dir_path) lint_conf = lint_conf.get("lint", {}) - print(f"lint_conf: {lint_conf}") ignore_entry: List[str] | bool = lint_conf.get(lint_name, []) - print(f"ignore_entry: {ignore_entry}") full_path = dir_path / file_path # Return a failed status if we can't find the file if not full_path.is_file(): diff --git a/tests/lint/configs.py b/tests/lint/configs.py index d5dcd27fad..b50a1393aa 100644 --- a/tests/lint/configs.py +++ b/tests/lint/configs.py @@ -43,6 +43,7 @@ def test_ignore_modules_config(self): content["lint"] = {"modules_config": False} with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: yaml.dump(content, f) + Path(new_pipeline, "conf", "modules.config").unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() result = lint_obj.modules_config() @@ -77,6 +78,7 @@ def test_ignore_base_config(self): content["lint"] = {"base_config": False} with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: yaml.dump(content, f) + Path(new_pipeline, "conf", "base.config").unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() result = lint_obj.base_config() diff --git a/tests/test_lint.py b/tests/test_lint.py index 1af5c40461..b72a6bfdfa 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -199,6 +199,7 @@ def test_sphinx_md_files(self): test_actions_schema_validation_missing_on, ) from .lint.configs import ( # type: ignore[misc] + test_ignore_base_config, test_ignore_modules_config, test_superfluous_withname_in_base_config_fails, test_superfluous_withname_in_modules_config_fails, From c29513766b21d4444b904c619f954867f8614fb6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 3 Apr 2024 18:56:52 +0200 Subject: [PATCH 156/270] fix tests --- nf_core/modules/lint/module_tests.py | 28 +++++++++---------- .../subworkflows/lint/subworkflow_tests.py | 28 +++++++++---------- tests/test_modules.py | 2 +- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 52cbaf04e5..b2b6c2221f 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -107,22 +107,22 @@ def module_tests(_, module: NFCoreComponent): snap_file, ) ) - if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): - module.passed.append( - ( - "test_snap_versions", - "versions found in snapshot file", - snap_file, - ) + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): + module.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, ) - else: - module.failed.append( - ( - "test_snap_versions", - "versions not found in snapshot file", - snap_file, - ) + ) + else: + module.failed.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, ) + ) except json.decoder.JSONDecodeError as e: module.failed.append( ( diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 30d4296aac..02a429b661 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -114,22 +114,22 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): snap_file, ) ) - if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): - subworkflow.passed.append( - ( - "test_snap_versions", - "versions found in snapshot file", - snap_file, - ) + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): + subworkflow.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, ) - else: - subworkflow.warned.append( - ( - "test_snap_versions", - "versions not found in snapshot file", - snap_file, - ) + ) + else: + subworkflow.warned.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, ) + ) except json.decoder.JSONDecodeError as e: subworkflow.failed.append( ( diff --git a/tests/test_modules.py b/tests/test_modules.py index 26fdea41ac..354bd2bb28 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -55,7 +55,7 @@ def create_modules_repo_dummy(tmp_dir): test_snap_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") test_snap_path.touch() with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n') + fh.write('{\n "my test": {},\n') fh.write(' "versions": {}\n}') # remove "TODO" statements from main.nf From c554caed7a396bf9deb61526e66591b69771d8e9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 07:56:01 +0200 Subject: [PATCH 157/270] fix subworkflow tests --- tests/subworkflows/lint.py | 18 +++++++++--------- tests/test_subworkflows.py | 5 +++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index a315ea3912..224278a5c8 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -185,20 +185,20 @@ def test_subworkflows_lint_capitalization_fail(self): def test_subworkflows_absent_version(self): """Test linting a nf-test module if the versions is absent in the snapshot file `""" - self.subworkflow_install.install("bam_sort_stats_samtools") - with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") - ) as fh: + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + with open(snap_file) as fh: content = fh.read() new_content = content.replace("versions", "foo") - with open( - Path(self.pipeline_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap"), "w" - ) as fh: + with open(snap_file, "w") as fh: fh.write(new_content) - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, subworkflow="bam_sort_stats_samtools") + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 0 assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" assert any([x.lint_test == "test_snap_versions" for x in subworkflow_lint.warned]) + + # cleanup + with open(snap_file, "w") as fh: + fh.write(content) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 775a08547f..329156227d 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -23,7 +23,6 @@ def create_modules_repo_dummy(tmp_dir): root_dir = Path(tmp_dir, "modules") Path(root_dir, "modules").mkdir(parents=True, exist_ok=True) - Path(root_dir, "subworkflows").mkdir(parents=True, exist_ok=True) Path(root_dir, "subworkflows", "nf-core").mkdir(parents=True, exist_ok=True) Path(root_dir, "tests", "config").mkdir(parents=True, exist_ok=True) with open(Path(root_dir, ".nf-core.yml"), "w") as fh: @@ -34,9 +33,11 @@ def create_modules_repo_dummy(tmp_dir): # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + test_snap_path.parent.mkdir(parents=True, exist_ok=True) test_snap_path.touch() with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n}') + fh.write('{\n "my test": {},\n') + fh.write(' "versions": {}\n}') return root_dir From 3ae9d0e32b78eb1805ef5fd925ef54814dbc2006 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Thu, 4 Apr 2024 13:11:41 +0200 Subject: [PATCH 158/270] Shorten lines in pipeline template --- .../pipeline-template/workflows/pipeline.nf | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 68adbaa328..8cdff1baa3 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -40,22 +40,44 @@ workflow {{ short_name|upper }} { // Collate and save software versions // softwareVersionsToYAML(ch_versions) - .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) - .set { ch_collated_versions } + .collectFile( + storeDir: "${params.outdir}/pipeline_info", + name: 'nf_core_pipeline_software_mqc_versions.yml', + sort: true, + newLine: true + ).set { ch_collated_versions } // // MODULE: MultiQC // - ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) - ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() - ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") - ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) - ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) - ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) - ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) - ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false)) + ch_multiqc_config = Channel.fromPath( + "$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = params.multiqc_config ? + Channel.fromPath(params.multiqc_config, checkIfExists: true) : + Channel.empty() + ch_multiqc_logo = params.multiqc_logo ? + Channel.fromPath(params.multiqc_logo, checkIfExists: true) : + Channel.empty() + + summary_params = paramsSummaryMap( + workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + + ch_multiqc_custom_methods_description = params.multiqc_methods_description ? + file(params.multiqc_methods_description, checkIfExists: true) : + file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_methods_description = Channel.value( + methodsDescriptionText(ch_multiqc_custom_methods_description)) + + ch_multiqc_files = ch_multiqc_files.mix( + ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_files = ch_multiqc_files.mix( + ch_methods_description.collectFile( + name: 'methods_description_mqc.yaml', + sort: false + ) + ) MULTIQC ( ch_multiqc_files.collect(), From 4dee9d90d1aa58b9a0c765155bfa7ec66a70936c Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 4 Apr 2024 12:06:16 +0000 Subject: [PATCH 159/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ad053f6df..3893b1860e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Add a cleanup action to `download_pipeline.yml` to fix failures caused by inadequate storage space on the runner ([#2849](https://github.com/nf-core/tools/pull/2849)) - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root +- Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) ### Linting From 9df6e378a8e024c9d334abf36bf2f26d0df2357f Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 16:19:38 +0200 Subject: [PATCH 160/270] fix snapshot test to handle updated fastp version --- tests/components/generate_snapshot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index 50024a8ebb..e42aeaa66e 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -1,6 +1,7 @@ """Test generate a snapshot""" import json +import shutil from pathlib import Path from unittest.mock import MagicMock @@ -120,9 +121,12 @@ def test_test_not_found(self): remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) + test_file = Path("modules", "nf-core-test", "fastp", "tests", "main.nf.test") + shutil.move(test_file, test_file.parent / "main.nf.test.bak") with pytest.raises(UserWarning) as e: snap_generator.run() assert "Test file 'main.nf.test' not found" in str(e.value) + shutil.move(test_file.parent / "main.nf.test.bak", test_file) def test_unstable_snapshot(self): From 70fac756f53bd74114caf24c6dd11817ad44658c Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 16:28:52 +0200 Subject: [PATCH 161/270] add linting tests for empty files in snapshots --- tests/components/generate_snapshot.py | 5 +- tests/modules/lint.py | 45 ++++++++++++ tests/subworkflows/lint.py | 100 ++++++++++++++++++++++++++ tests/test_modules.py | 21 +++++- tests/test_subworkflows.py | 20 +++++- 5 files changed, 182 insertions(+), 9 deletions(-) diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index e42aeaa66e..3176569ec8 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -1,7 +1,6 @@ """Test generate a snapshot""" import json -import shutil from pathlib import Path from unittest.mock import MagicMock @@ -122,11 +121,11 @@ def test_test_not_found(self): branch=GITLAB_NFTEST_BRANCH, ) test_file = Path("modules", "nf-core-test", "fastp", "tests", "main.nf.test") - shutil.move(test_file, test_file.parent / "main.nf.test.bak") + test_file.rename(test_file.parent / "main.nf.test.bak") with pytest.raises(UserWarning) as e: snap_generator.run() assert "Test file 'main.nf.test' not found" in str(e.value) - shutil.move(test_file.parent / "main.nf.test.bak", test_file) + Path(test_file.parent / "main.nf.test.bak").rename(test_file) def test_unstable_snapshot(self): diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 5e3bcfbd30..265caed379 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -1,3 +1,4 @@ +import json from pathlib import Path import pytest @@ -663,3 +664,47 @@ def test_modules_absent_version(self): assert len(module_lint.passed) >= 0 assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "test_snap_versions" + + +def test_modules_empty_file_in_snapshot(self): + """Test linting a nf-test module with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" + snap_file = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_snap_md5sum" + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_modules_empty_file_in_stub_snapshot(self): + """Test linting a nf-test module with an empty file sha sum in the stub test snapshot, which should make it not fail""" + snap_file = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert any(x.lint_test == "test_snap_md5sum" for x in module_lint.passed) + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 224278a5c8..28d29d2f9d 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -1,3 +1,5 @@ +import json +import shutil from pathlib import Path import pytest @@ -202,3 +204,101 @@ def test_subworkflows_absent_version(self): # cleanup with open(snap_file, "w") as fh: fh.write(content) + + +def test_subworkflows_missing_test_dir(self): + """Test linting a nf-test subworkflow if the tests directory is missing""" + test_dir = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests") + test_dir_copy = shutil.copytree(test_dir, test_dir.parent / "tests_copy") + shutil.rmtree(test_dir) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" + assert any([x.lint_test == "test_dir_versions" for x in subworkflow_lint.warned]) + + # cleanup + shutil.copytree(test_dir_copy, test_dir) + + +def test_subworkflows_missing_main_nf(self): + """Test linting a nf-test subworkflow if the main.nf file is missing""" + main_nf = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf") + main_nf_copy = shutil.copy(main_nf, main_nf.parent / "main_nf_copy") + main_nf.unlink() + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "main_nf_exists" + + # cleanup + shutil.copy(main_nf_copy, main_nf) + + +def test_subworkflows_empty_file_in_snapshot(self): + """Test linting a nf-test subworkflow with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" + snap_file = Path( + self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap" + ) + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "test_snap_md5sum" + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_empty_file_in_stub_snapshot(self): + """Test linting a nf-test subworkflow with an empty file sha sum in the stub test snapshot, which should make it not fail""" + snap_file = Path( + self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap" + ) + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert any(x.lint_test == "test_snap_md5sum" for x in subworkflow_lint.passed) + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_missing_tags_yml(self): + """Test linting a subworkflow with a missing tags.yml file""" + tags_path = Path(self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + tags_path.rename(tags_path.parent / "tags.yml.bak") + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) >= 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "test_tags_yml_exists" + + # cleanup + tags_path.rename(tags_path.parent / "tags.yml") diff --git a/tests/test_modules.py b/tests/test_modules.py index 354bd2bb28..d3d99abadd 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,5 +1,6 @@ """Tests covering the modules commands""" +import json import os import shutil import unittest @@ -53,10 +54,22 @@ def create_modules_repo_dummy(tmp_dir): yaml.dump(meta_yml, fh) # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") - test_snap_path.touch() + with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {},\n') - fh.write(' "versions": {}\n}') + json.dump( + { + "my test": { + "content": [ + { + "0": [], + "versions": {}, + } + ] + } + }, + fh, + indent=4, + ) # remove "TODO" statements from main.nf main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") @@ -181,6 +194,8 @@ def test_modulesrepo_class(self): ) from .modules.lint import ( # type: ignore[misc] test_modules_absent_version, + test_modules_empty_file_in_snapshot, + test_modules_empty_file_in_stub_snapshot, test_modules_environment_yml_file_doesnt_exists, test_modules_environment_yml_file_name_mismatch, test_modules_environment_yml_file_not_array, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 329156227d..6c00df9816 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -1,5 +1,6 @@ """Tests covering the subworkflows commands""" +import json import os import shutil import unittest @@ -34,10 +35,21 @@ def create_modules_repo_dummy(tmp_dir): # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") test_snap_path.parent.mkdir(parents=True, exist_ok=True) - test_snap_path.touch() with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {},\n') - fh.write(' "versions": {}\n}') + json.dump( + { + "my test": { + "content": [ + { + "0": [], + "versions": {}, + } + ] + } + }, + fh, + indent=4, + ) return root_dir @@ -121,6 +133,8 @@ def tearDown(self): ) from .subworkflows.lint import ( # type: ignore[misc] test_subworkflows_absent_version, + test_subworkflows_empty_file_in_snapshot, + test_subworkflows_empty_file_in_stub_snapshot, test_subworkflows_lint, test_subworkflows_lint_capitalization_fail, test_subworkflows_lint_empty, From 953a980e5c2ad38ca702304c0b2f4e05d2687450 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 17:56:07 +0200 Subject: [PATCH 162/270] add more tags.yml tests for subworkflows --- tests/modules/lint.py | 24 +++++++-------- tests/subworkflows/lint.py | 60 +++++++++++++++++++++++++++++++------- tests/test_subworkflows.py | 3 ++ 3 files changed, 64 insertions(+), 23 deletions(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 265caed379..de7ec203ef 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -564,30 +564,29 @@ def test_modules_missing_required_tag(self): def test_modules_missing_tags_yml(self): """Test linting a module with a missing tags.yml file""" - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml").rename( - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak") - ) + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + tags_path.rename(tags_path.parent / "tags.yml.bak") module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak").rename( - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") - ) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "test_tags_yml_exists" + # cleanup + Path(tags_path.parent / "tags.yml").rename(tags_path.parent / "tags.yml") def test_modules_incorrect_tags_yml_key(self): """Test linting a module with an incorrect key in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + with open(tags_path) as fh: content = fh.read() new_content = content.replace("bpipe/test:", "bpipe_test:") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(new_content) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=True, module="bpipe/test") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(content) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 @@ -597,14 +596,15 @@ def test_modules_incorrect_tags_yml_key(self): def test_modules_incorrect_tags_yml_values(self): """Test linting a module with an incorrect path in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + with open(tags_path) as fh: content = fh.read() new_content = content.replace("modules/nf-core/bpipe/test/**", "foo") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(new_content) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(content) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 28d29d2f9d..73d2452b37 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -242,9 +242,7 @@ def test_subworkflows_missing_main_nf(self): def test_subworkflows_empty_file_in_snapshot(self): """Test linting a nf-test subworkflow with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" - snap_file = Path( - self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap" - ) + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") snap = json.load(snap_file.open()) content = snap_file.read_text() snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" @@ -252,7 +250,7 @@ def test_subworkflows_empty_file_in_snapshot(self): with open(snap_file, "w") as fh: json.dump(snap, fh) - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 @@ -266,9 +264,7 @@ def test_subworkflows_empty_file_in_snapshot(self): def test_subworkflows_empty_file_in_stub_snapshot(self): """Test linting a nf-test subworkflow with an empty file sha sum in the stub test snapshot, which should make it not fail""" - snap_file = Path( - self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap" - ) + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") snap = json.load(snap_file.open()) content = snap_file.read_text() snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} @@ -276,7 +272,7 @@ def test_subworkflows_empty_file_in_stub_snapshot(self): with open(snap_file, "w") as fh: json.dump(snap, fh) - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 @@ -290,9 +286,9 @@ def test_subworkflows_empty_file_in_stub_snapshot(self): def test_subworkflows_missing_tags_yml(self): """Test linting a subworkflow with a missing tags.yml file""" - tags_path = Path(self.nfcore_subworkflows, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") tags_path.rename(tags_path.parent / "tags.yml.bak") - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_subworkflows) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" @@ -301,4 +297,46 @@ def test_subworkflows_missing_tags_yml(self): assert subworkflow_lint.failed[0].lint_test == "test_tags_yml_exists" # cleanup - tags_path.rename(tags_path.parent / "tags.yml") + Path(tags_path.parent / "tags.yml.bak").rename(tags_path.parent / "tags.yml") + + +def test_subworkflows_incorrect_tags_yml_key(self): + """Test linting a subworkflow with an incorrect key in tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + with open(tags_path) as fh: + content = fh.read() + new_content = content.replace("test_subworkflow:", "subworkflow:") + with open(tags_path, "w") as fh: + fh.write(new_content) + module_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + module_lint.lint(print_results=True, subworkflow="test_subworkflow") + with open(tags_path, "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + # cleanup + with open(tags_path, "w") as fh: + fh.write(content) + + +def test_subworkflows_incorrect_tags_yml_values(self): + """Test linting a subworkflow with an incorrect path in tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + with open(tags_path) as fh: + content = fh.read() + new_content = content.replace("subworkflows/nf-core/test_subworkflow/**", "foo") + with open(tags_path, "w") as fh: + fh.write(new_content) + module_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, subworkflow="test_subworkflow") + with open(tags_path, "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + # cleanup + with open(tags_path, "w") as fh: + fh.write(content) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 6c00df9816..0a9224002a 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -135,6 +135,8 @@ def tearDown(self): test_subworkflows_absent_version, test_subworkflows_empty_file_in_snapshot, test_subworkflows_empty_file_in_stub_snapshot, + test_subworkflows_incorrect_tags_yml_key, + test_subworkflows_incorrect_tags_yml_values, test_subworkflows_lint, test_subworkflows_lint_capitalization_fail, test_subworkflows_lint_empty, @@ -147,6 +149,7 @@ def tearDown(self): test_subworkflows_lint_snapshot_file, test_subworkflows_lint_snapshot_file_missing_fail, test_subworkflows_lint_snapshot_file_not_needed, + test_subworkflows_missing_tags_yml, ) from .subworkflows.list import ( # type: ignore[misc] test_subworkflows_install_and_list_subworkflows, From 53c7cda0b220ac5dd1170530f9b3658b6bea6844 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 4 Apr 2024 18:12:09 +0200 Subject: [PATCH 163/270] fix test --- tests/modules/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index de7ec203ef..595509de4e 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -573,7 +573,7 @@ def test_modules_missing_tags_yml(self): assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "test_tags_yml_exists" # cleanup - Path(tags_path.parent / "tags.yml").rename(tags_path.parent / "tags.yml") + Path(tags_path.parent / "tags.yml.bak").rename(tags_path.parent / "tags.yml") def test_modules_incorrect_tags_yml_key(self): From 986f9d07adb51b7ffaabd64e771ca23f9c2c50bc Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 4 Apr 2024 00:20:08 +0000 Subject: [PATCH 164/270] chore(deps): update codecov/codecov-action digest to 7afa10e --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 93a44aea5e..c6af0ed983 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -184,7 +184,7 @@ jobs: coverage report coverage xml - - uses: codecov/codecov-action@c16abc29c95fcf9174b58eb7e1abf4c866893bc8 # v4 + - uses: codecov/codecov-action@7afa10ed9b269c561c2336fd862446844e0cbf71 # v4 with: files: coverage.xml env: From b9e4d2fb5edea183db8d06ba9daab63e1cdf0958 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 5 Apr 2024 03:35:34 +0000 Subject: [PATCH 165/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3893b1860e..9c5a874df9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ - Get immediate parent path name for schema creation ([#2886](https://github.com/nf-core/tools/pull/2886)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 ([#2903](https://github.com/nf-core/tools/pull/2903)) - Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897)) +- Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 32aac4d96adab4edb147c43613a74c542381d918 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 04:44:50 +0000 Subject: [PATCH 166/270] Update codecov/codecov-action digest to 8450866 --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c6af0ed983..1303409c05 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -184,7 +184,7 @@ jobs: coverage report coverage xml - - uses: codecov/codecov-action@7afa10ed9b269c561c2336fd862446844e0cbf71 # v4 + - uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # v4 with: files: coverage.xml env: From b75416da9a493fa967289f845e0caa6b8c7146f4 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 11 Apr 2024 06:22:30 +0000 Subject: [PATCH 167/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 159a283daa..2da49c4829 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 ([#2903](https://github.com/nf-core/tools/pull/2903)) - Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897)) - Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) +- Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 513b5feb6f118535b29f084ca03b1df418077f0a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 09:51:03 +0000 Subject: [PATCH 168/270] chore(deps): update gitpod/workspace-base docker digest to 674315a --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index e476a7f140..ca69f74f71 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:450d764c8575345b32e3e22d197b5150152eff9f9a14482003f70f05d03f726a +FROM gitpod/workspace-base@sha256:674315a74b11e7b796040587d5e5ad000d755e0f14dd070704757e578be25fd9 USER root From b1c19fda298cadc7aa4b337dc349804364d8e918 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:43:44 +0000 Subject: [PATCH 169/270] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 915bbc345d..d7fed0449d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.5 + rev: v0.3.6 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 123ffcd8593e6807f93ef4892c83dfde0f375452 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 17:43:41 +0000 Subject: [PATCH 170/270] chore(deps): update python:3.12-slim docker digest to 541d45d --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f49001089..c801563667 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:5dc6f84b5e97bfb0c90abfb7c55f3cacc2cb6687c8f920b64a833a2219875997 +FROM python:3.12-slim@sha256:541d45d3d675fb8197f534525a671e2f8d66c882b89491f9dda271f4f94dcd06 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From 0932fd2624fb3aff33ecc07d00845865a4e24005 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 11 Apr 2024 17:44:36 +0000 Subject: [PATCH 171/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2da49c4829..7e758c6e60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ - Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897)) - Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) - Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 104d5b4c6909dc3e798897b90ea81203b50e62df Mon Sep 17 00:00:00 2001 From: Kyle Hazen <41054023+k1sauce@users.noreply.github.com> Date: Thu, 11 Apr 2024 16:47:04 -0700 Subject: [PATCH 172/270] Update subworkflow_tests.py fix typo --- nf_core/subworkflows/lint/subworkflow_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 02a429b661..cfae2d553c 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -20,7 +20,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): It verifies that the test directory exists and contains a ``main.nf.test`` a ``main.nf.test.snap`` and ``tags.yml``. - Additionally, hecks that all included components in test ``main.nf`` are specified in ``test.yml`` + Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ repo_dir = subworkflow.component_dir.parts[ From 941d087168007edfddfa927b3cec0594be57e0f9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 04:38:16 +0000 Subject: [PATCH 173/270] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d7fed0449d..9e227aa0b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.6 + rev: v0.3.7 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From c7361a0d24b7c5a4f5c063bba82d8f929cbc5a07 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 12 Apr 2024 04:39:06 +0000 Subject: [PATCH 174/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e758c6e60..e525ae4ecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ - Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) - Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From baf3519652d9256de4e13103de731d7c39c12a32 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 22:57:46 +0000 Subject: [PATCH 175/270] Update gitpod/workspace-base Docker digest to 0af257e --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index ca69f74f71..fd734381a5 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:674315a74b11e7b796040587d5e5ad000d755e0f14dd070704757e578be25fd9 +FROM gitpod/workspace-base@sha256:0af257e5e7f0340db76c905ca6acbe3d432567aafe53ffc0c91c369fc1183428 USER root From 3a7f8e6de2b74898844b75c592a9c03c0cab50af Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 17 Apr 2024 02:02:20 +0000 Subject: [PATCH 176/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e525ae4ecf..d7ba6a7a46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ - Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) +- Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 6fa38cbf6033554c4086fa491d83be8a63fe96c2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 19 Apr 2024 18:11:59 +0000 Subject: [PATCH 177/270] Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.1 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e227aa0b1..a90a732bfe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.7 + rev: v0.4.1 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 6313aa254715bafc97f001a8d0428ff2509cc6fd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 19 Apr 2024 02:07:21 +0000 Subject: [PATCH 178/270] Update GitHub Actions --- .github/actions/create-lint-wf/action.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/pytest.yml | 4 ++-- .github/workflows/sync.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 1741b934f2..27cb4dbc9f 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -77,7 +77,7 @@ runs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-lint-wf/log.txt diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 4ef448e5ec..3ee9686f94 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -158,7 +158,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: nf-core-log-file-${{ matrix.TEMPLATE }} path: create-test-lint-wf/artifact_files.tar diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 952181c044..88962a0bc7 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -75,7 +75,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-test-wf/log.txt diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 1303409c05..ea535394ea 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -143,7 +143,7 @@ jobs: fi - name: Upload coverage - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: coverage_${{ matrix.test }} path: .coverage @@ -177,7 +177,7 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4 + uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 2adb970f64..e54436af0e 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -86,7 +86,7 @@ jobs: - name: Upload sync log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: sync_log_${{ matrix.pipeline }} path: sync_log_${{ matrix.pipeline }}.txt diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index a8208b31ac..719689507f 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -60,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 with: name: linting-logs path: | From 360b044a656e39ab941b1645060a823c0b3d9cbc Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 20 Apr 2024 05:17:23 +0000 Subject: [PATCH 179/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7ba6a7a46..d4ae1e115d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) +- Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 50f180846d5792ae70975669220b591fbf7eeee5 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 08:45:39 +0200 Subject: [PATCH 180/270] Replace hardcoded URL to test data with parameter --- CHANGELOG.md | 1 + nf_core/pipeline-template/conf/test.config | 2 +- nf_core/pipeline-template/conf/test_full.config | 4 ++-- nf_core/pipeline-template/nextflow.config | 1 + nf_core/pipeline-template/nextflow_schema.json | 6 ++++++ 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ae1e115d..761aaf1863 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) +- Add a new hidden `--test_data_base_path` parameter to more easily switch locations of test data in test configs (#2900)[] ### Linting diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 32b9619ebb..f9f4218cb8 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -22,7 +22,7 @@ params { // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' + input = test_data_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' {% if igenomes -%} // Genome references diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index d92692fa94..93db73de26 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -17,13 +17,13 @@ params { // Input data for full size test // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA) // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' + input = test_data_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' {% if igenomes -%} // Genome references genome = 'R64-1-1' {%- else -%} // Fasta references - fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + fasta = test_data_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' {%- endif %} } diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2f9b349916..1da1e7a96b 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -37,6 +37,7 @@ params { hook_url = null help = false version = false + test_data_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' // Config options config_profile_name = null diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 77b1a7a070..56d07c764e 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -267,6 +267,12 @@ "description": "Validation of parameters in lenient more.", "hidden": true, "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." + }, + "test_data_base_path":{ + "type": "string", + "fa_icon": "far fa-check-circle", + "description": "Base URL to location of pipeline test dataset files", + "hidden": true, } } } From 8a781499192572072ce535e2b2346967ce911a4f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 13:32:25 +0000 Subject: [PATCH 181/270] chore(deps): update gitpod/workspace-base docker digest to 5aeb24f --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index fd734381a5..be4feab2c5 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:0af257e5e7f0340db76c905ca6acbe3d432567aafe53ffc0c91c369fc1183428 +FROM gitpod/workspace-base@sha256:5aeb24f9994e8226744a200b8316d5d429c0ac98e0b49d1c05ee7d376d8420c5 USER root From ced4255d9dcaebf0033bf2614676c16764d99d15 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 23 Apr 2024 16:07:39 +0000 Subject: [PATCH 182/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ae1e115d..0b4931a639 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) - Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) +- Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 8dca26f47d7b31b7abe8460703b105702dad68cc Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 08:57:32 +0200 Subject: [PATCH 183/270] Apply suggestions from code review --- nf_core/pipeline-template/nextflow_schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 56d07c764e..533989f3f8 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -268,10 +268,10 @@ "hidden": true, "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." }, - "test_data_base_path":{ + "test_data_base_path": { "type": "string", "fa_icon": "far fa-check-circle", - "description": "Base URL to location of pipeline test dataset files", + "description": "Base URL or local path to location of pipeline test dataset files", "hidden": true, } } From 9bfdcee731d7ea198e3631116a859db25625f9e1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 07:38:55 +0000 Subject: [PATCH 184/270] chore(deps): update github actions --- .github/actions/create-lint-wf/action.yml | 2 +- .github/workflows/changelog.yml | 2 +- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 4 ++-- .github/workflows/create-test-wf.yml | 4 ++-- .github/workflows/deploy-pypi.yml | 2 +- .github/workflows/fix-linting.yml | 2 +- .github/workflows/lint-code.yml | 2 +- .github/workflows/push_dockerhub_dev.yml | 2 +- .github/workflows/push_dockerhub_release.yml | 2 +- .github/workflows/pytest.yml | 10 +++++----- .github/workflows/rich-codex.yml | 2 +- .github/workflows/sync.yml | 6 +++--- .github/workflows/update_components_template.yml | 2 +- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- .../.github/workflows/fix-linting.yml | 2 +- .../pipeline-template/.github/workflows/linting.yml | 6 +++--- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 27cb4dbc9f..0bc5e432e7 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -77,7 +77,7 @@ runs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-lint-wf/log.txt diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index bed104e2d9..5318d2a0c4 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -19,7 +19,7 @@ jobs: ) steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 with: token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 996bdf2799..521f5d038f 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -48,7 +48,7 @@ jobs: export NXF_WORK=$(pwd) # Get the repo code - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository # Set up nf-core/tools diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 3ee9686f94..1634ade745 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -61,7 +61,7 @@ jobs: cd create-lint-wf-template export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: Set up Python 3.12 @@ -158,7 +158,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.TEMPLATE }} path: create-test-lint-wf/artifact_files.tar diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 88962a0bc7..d752e26713 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -48,7 +48,7 @@ jobs: cd create-test-wf export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: Set up Python 3.12 @@ -75,7 +75,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-test-wf/log.txt diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index ec94ddb88f..0afe47b751 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index b3f73c5a8c..b89dd44aa9 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 51a543c143..9a9eb31bc0 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -18,7 +18,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index af11a79990..360c863af6 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 0b3f381d86..1babf607d7 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index ea535394ea..0d51739bf9 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -61,7 +61,7 @@ jobs: name: Get test file matrix runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: List tests @@ -87,7 +87,7 @@ jobs: cd pytest export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out source-code repository - name: Set up Python ${{ needs.setup.outputs.python-version }} @@ -143,7 +143,7 @@ jobs: fi - name: Upload coverage - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: coverage_${{ matrix.test }} path: .coverage @@ -158,7 +158,7 @@ jobs: mkdir -p pytest cd pytest - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 env: @@ -177,7 +177,7 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@8caf195ad4b1dee92908e23f56eeb0696f1dd42d # v4 + uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index fc8af8a7bc..016f20ab0e 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index e54436af0e..43bab50bbf 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -44,10 +44,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out nf-core/tools - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} @@ -86,7 +86,7 @@ jobs: - name: Upload sync log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: sync_log_${{ matrix.pipeline }} path: sync_log_${{ matrix.pipeline }}.txt diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index c5413cd557..d3690af6e3 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 3880b2c4dc..9603a597a9 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index bc15ce42e8..1dc4be0113 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 719689507f..2e75daf1a9 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,7 +14,7 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 @@ -60,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: linting-logs path: | From cb4a775d79ea8156a4d3b8372a506ccd6350f686 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 12:50:32 +0200 Subject: [PATCH 185/270] Apply suggestions from code review Co-authored-by: Maxime U Garcia --- CHANGELOG.md | 2 +- nf_core/pipeline-template/conf/test.config | 2 +- nf_core/pipeline-template/conf/test_full.config | 4 ++-- nf_core/pipeline-template/nextflow.config | 2 +- nf_core/pipeline-template/nextflow_schema.json | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 761aaf1863..63d41a258c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) -- Add a new hidden `--test_data_base_path` parameter to more easily switch locations of test data in test configs (#2900)[] +- Add a new hidden `--pipelines_testdata_base_path` parameter to more easily switch locations of test data in test configs (#2931)[https://github.com/nf-core/tools/pull/2931] ### Linting diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index f9f4218cb8..cd12404c8e 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -22,7 +22,7 @@ params { // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = test_data_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' + input = pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' {% if igenomes -%} // Genome references diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index 93db73de26..dac47b9867 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -17,13 +17,13 @@ params { // Input data for full size test // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA) // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = test_data_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' + input = pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' {% if igenomes -%} // Genome references genome = 'R64-1-1' {%- else -%} // Fasta references - fasta = test_data_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + fasta = pipelines_testdata_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' {%- endif %} } diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 1da1e7a96b..377c2964b9 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -37,7 +37,7 @@ params { hook_url = null help = false version = false - test_data_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' + pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' // Config options config_profile_name = null diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 533989f3f8..8c687c74a1 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -268,7 +268,7 @@ "hidden": true, "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." }, - "test_data_base_path": { + "pipelines_testdata_base_path": { "type": "string", "fa_icon": "far fa-check-circle", "description": "Base URL or local path to location of pipeline test dataset files", From 60911aba63917b0900449254e4dbdc5ab13c3959 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 12:51:11 +0200 Subject: [PATCH 186/270] =?UTF-8?q?Harshil=20Align=E2=84=A2=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nf_core/pipeline-template/nextflow.config | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 377c2964b9..4452207ccc 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -28,16 +28,16 @@ params { multiqc_methods_description = null // Boilerplate options - outdir = null - publish_dir_mode = 'copy' - email = null - email_on_fail = null - plaintext_email = false - monochrome_logs = false - hook_url = null - help = false - version = false - pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' + outdir = null + publish_dir_mode = 'copy' + email = null + email_on_fail = null + plaintext_email = false + monochrome_logs = false + hook_url = null + help = false + version = false + pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' // Config options config_profile_name = null From 720852e76561dd0a3262f04bf8713ca15f0b76fe Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 24 Apr 2024 13:00:02 +0200 Subject: [PATCH 187/270] Fix JSON typo --- nf_core/pipeline-template/nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 8c687c74a1..01bff71f7d 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -272,7 +272,7 @@ "type": "string", "fa_icon": "far fa-check-circle", "description": "Base URL or local path to location of pipeline test dataset files", - "hidden": true, + "hidden": true } } } From 6f7a30982b2b563df1c870dd54becd0ccd5e4bed Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 10:28:34 +0000 Subject: [PATCH 188/270] Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a90a732bfe..5ff5afd864 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: alias: ec - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.9.0" + rev: "v1.10.0" hooks: - id: mypy additional_dependencies: From 8dc2ae6b56ab830d8154ac7735cf32e6cb8aacc3 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 25 Apr 2024 10:29:26 +0000 Subject: [PATCH 189/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b4931a639..08ef6ed61e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) - Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) +- Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From e77ed1b254bfacd063a5334d8ca47bdd18466a88 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 20:30:35 +0000 Subject: [PATCH 190/270] Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.2 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ff5afd864..d8f35e2cc5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.1 + rev: v0.4.2 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From aaabf5faee9097d9b9970597c8ad88d23a41e53e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 26 Apr 2024 08:51:13 +0200 Subject: [PATCH 191/270] Update nextflow.config --- nf_core/pipeline-template/nextflow.config | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2f9b349916..e7a9f8cf61 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -76,12 +76,11 @@ try { } // Load {{ name }} custom profiles from different institutions. -// Warning: Uncomment only if a pipeline-specific institutional config already exists on nf-core/configs! -// try { -// includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" -// } catch (Exception e) { -// System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") -// } +try { + includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" +} catch (Exception e) { + System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") +} {% endif -%} profiles { From f1ce816aa549aa4fbb48515030592fb549b28c4c Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 26 Apr 2024 08:57:55 +0200 Subject: [PATCH 192/270] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ef6ed61e..e5a9610453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) +- Permanently activated pipeline-specific institutional configs support for all pipelines ([#2936](https://github.com/nf-core/tools/pull/2936)) ### Linting From c83cdfa82ee526fbc4c46ce1ca9bf761b955e95e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 26 Apr 2024 08:59:17 +0200 Subject: [PATCH 193/270] Apply suggestions from code review --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5a9610453..b57381da11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) -- Permanently activated pipeline-specific institutional configs support for all pipelines ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) ### Linting From 91d264b0406c515edf478e02493d264a7430f3c0 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 26 Apr 2024 07:11:18 +0000 Subject: [PATCH 194/270] [automated] Fix code linting --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b57381da11..e0ba31484f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) -- Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) ### Linting From 2d029dbb86ed9098881991df0f0264c8a75aed0c Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 26 Apr 2024 14:13:44 +0200 Subject: [PATCH 195/270] fix ruff linting (UP031) --- nf_core/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index 9d4decc424..8ffecc0f55 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -925,7 +925,7 @@ def rectify_raw_container_matches(self, raw_findings): if bool(container_definition) and bool(container_definition.group(1)): pattern = re.escape(container_definition.group(1)) # extract the quoted string(s) following the variable assignment - container_names = re.findall(r"%s\s*=\s*[\"\']([^\"\']+)[\"\']" % pattern, search_space) + container_names = re.findall(rf"{pattern}\s*=\s*[\"\']([^\"\']+)[\"\']", search_space) if bool(container_names): if isinstance(container_names, str): From 352c02cb18000118218793aa995d61d62606c8aa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:28:12 +0000 Subject: [PATCH 196/270] Update GitHub Actions --- .github/workflows/changelog.yml | 2 +- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/deploy-pypi.yml | 2 +- .github/workflows/fix-linting.yml | 2 +- .github/workflows/lint-code.yml | 2 +- .github/workflows/push_dockerhub_dev.yml | 2 +- .github/workflows/push_dockerhub_release.yml | 2 +- .github/workflows/pytest.yml | 8 ++++---- .github/workflows/rich-codex.yml | 2 +- .github/workflows/sync.yml | 4 ++-- .github/workflows/update_components_template.yml | 2 +- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- .../pipeline-template/.github/workflows/fix-linting.yml | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 4 ++-- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 5318d2a0c4..cebcc854bc 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -19,7 +19,7 @@ jobs: ) steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 521f5d038f..03b9aa2411 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -48,7 +48,7 @@ jobs: export NXF_WORK=$(pwd) # Get the repo code - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository # Set up nf-core/tools diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 1634ade745..0de7287a57 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -61,7 +61,7 @@ jobs: cd create-lint-wf-template export NXF_WORK=$(pwd) - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index d752e26713..87cdf2e7bb 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -48,7 +48,7 @@ jobs: cd create-test-wf export NXF_WORK=$(pwd) - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 0afe47b751..1202891e4d 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index b89dd44aa9..4334871c4c 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 9a9eb31bc0..3bddd42d49 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -18,7 +18,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index 360c863af6..c613e13a2d 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 1babf607d7..5a076f6d3b 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0d51739bf9..03576fa310 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -61,7 +61,7 @@ jobs: name: Get test file matrix runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: List tests @@ -87,7 +87,7 @@ jobs: cd pytest export NXF_WORK=$(pwd) - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python ${{ needs.setup.outputs.python-version }} @@ -158,7 +158,7 @@ jobs: mkdir -p pytest cd pytest - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 env: @@ -177,7 +177,7 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4 + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 016f20ab0e..cd12b139d3 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 43bab50bbf..15802cdef2 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -44,10 +44,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out nf-core/tools - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index d3690af6e3..5ba513735e 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 9603a597a9..6b2547765d 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 1dc4be0113..18e6f9e158 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 2e75daf1a9..d1ecae12b7 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,7 +14,7 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python 3.12 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 From 025f51db4dabefbebe573bd144ff46337dfae4c6 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 26 Apr 2024 20:05:13 +0000 Subject: [PATCH 197/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ef6ed61e..b618f947ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ - Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) +- Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 952ae662321b8085e24ed87bead870a1980dce27 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 15:22:44 +0200 Subject: [PATCH 198/270] Template: Add config profile 'wave' --- nf_core/pipeline-template/docs/usage.md | 2 ++ nf_core/pipeline-template/nextflow.config | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index c908d3d38c..b8305bf67f 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -163,6 +163,8 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) - `apptainer` - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) +- `wave` + - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above. - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2f9b349916..ad22abb5ae 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -176,6 +176,11 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } + wave { + wave.enabled = true + wave.strategy = 'conda,container' + wave.freeze = true + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } From 8e9b3e6880f4b5fd14189b6dfe20458f9f05e472 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 15:37:07 +0200 Subject: [PATCH 199/270] Move Wave above GitPod --- nf_core/pipeline-template/nextflow.config | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ad22abb5ae..b95a741299 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -171,16 +171,16 @@ profiles { shifter.enabled = false charliecloud.enabled = false } - gitpod { - executor.name = 'local' - executor.cpus = 4 - executor.memory = 8.GB - } wave { wave.enabled = true wave.strategy = 'conda,container' wave.freeze = true } + gitpod { + executor.name = 'local' + executor.cpus = 4 + executor.memory = 8.GB + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } From 8875f1009db9b0815da68b6b63aad8542bc7d559 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 15:57:48 +0200 Subject: [PATCH 200/270] Add 'singularity.ociAutoPull = true' --- nf_core/pipeline-template/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index b95a741299..ebe7a34081 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -175,6 +175,7 @@ profiles { wave.enabled = true wave.strategy = 'conda,container' wave.freeze = true + singularity.ociAutoPull = true } gitpod { executor.name = 'local' From 6dc46f747f1ddc0b4020ce38ea6417e2b0581e54 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 16:18:07 +0200 Subject: [PATCH 201/270] Harshil aligned and Maxime sorted Co-authored-by: Maxime U Garcia --- nf_core/pipeline-template/nextflow.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ebe7a34081..ea4e3b44be 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -172,10 +172,10 @@ profiles { charliecloud.enabled = false } wave { - wave.enabled = true - wave.strategy = 'conda,container' - wave.freeze = true singularity.ociAutoPull = true + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' } gitpod { executor.name = 'local' From ac5d1e8bcce95ebc1d5601b05ea9d80dd12f5dac Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 17:12:19 +0200 Subject: [PATCH 202/270] Added 'apptainer.ociAutoPull' --- nf_core/pipeline-template/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index ea4e3b44be..2aa2c5914c 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -172,6 +172,7 @@ profiles { charliecloud.enabled = false } wave { + apptainer.ociAutoPull = true singularity.ociAutoPull = true wave.enabled = true wave.freeze = true From 76ed900bf28cdf237a24ffeeb82d17f7933ade99 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 30 Apr 2024 17:18:07 +0200 Subject: [PATCH 203/270] Add minimum Nextflow version to the docs --- nf_core/pipeline-template/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index b8305bf67f..d46dfca04c 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -164,7 +164,7 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - `apptainer` - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) - `wave` - - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above. + - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow ` 24.03.0-edge` or later). - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. From 90a107054112fecd156e2b7802d72f85c68be8d4 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Wed, 1 May 2024 14:08:22 -0400 Subject: [PATCH 204/270] Extend repo_full_name_from_remote function to accomodate paths that begin with ssh:// ftp[s]:// etc. --- nf_core/modules/modules_utils.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index ca8993483b..21169516bd 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -20,22 +20,20 @@ def repo_full_name_from_remote(remote_url: str) -> str: Extracts the path from the remote URL See https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS for the possible URL patterns """ - # Check whether we have a https or ssh url - if remote_url.startswith("https"): - path = urlparse(remote_url).path - # Remove the intial '/' - path = path[1:] - # Remove extension - path = os.path.splitext(path)[0] + + if remote_url.startswith(("https://", "http://", "ftps://", "ftp://", "ssh://")): + # Parse URL and remove the initial '/' + path = urlparse(remote_url).path.lstrip('/') + elif 'git@' in remote_url: + # Extract the part after 'git@' and parse it + path = urlparse(remote_url.split('git@')[-1]).path else: - # Remove the initial `git@`` - split_path: list = remote_url.split("@") - path = split_path[-1] if len(split_path) > 1 else split_path[0] - path = urlparse(path).path - # Remove extension - path = os.path.splitext(path)[0] - return path + path = urlparse(remote_url).path + + # Remove the file extension from the path + path, _ = os.path.splitext(path) + return path def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], List[NFCoreComponent]]: """ From bb4992090d2c2a97c5ccec16ebd850d7b4a34d0f Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Wed, 1 May 2024 14:17:26 -0400 Subject: [PATCH 205/270] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b618f947ba..0e38e319c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## v2.13.2dev +- Handle more complete list of possible git URL forms (ssh:// and ftp:// prefixes specifically) + ### Template - Remove fasta default from nextflow.config ([#2828](https://github.com/nf-core/tools/pull/2828)) From 453f9ea2b23d34780ef417e2ba4842e7f7b6e3d9 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Wed, 1 May 2024 14:24:06 -0400 Subject: [PATCH 206/270] Ruff formatting fix --- nf_core/modules/modules_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index 21169516bd..6796de41ec 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -23,10 +23,10 @@ def repo_full_name_from_remote(remote_url: str) -> str: if remote_url.startswith(("https://", "http://", "ftps://", "ftp://", "ssh://")): # Parse URL and remove the initial '/' - path = urlparse(remote_url).path.lstrip('/') - elif 'git@' in remote_url: + path = urlparse(remote_url).path.lstrip("/") + elif "git@" in remote_url: # Extract the part after 'git@' and parse it - path = urlparse(remote_url.split('git@')[-1]).path + path = urlparse(remote_url.split("git@")[-1]).path else: path = urlparse(remote_url).path @@ -35,6 +35,7 @@ def repo_full_name_from_remote(remote_url: str) -> str: return path + def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], List[NFCoreComponent]]: """ Make a list of all modules installed in this repository From ca79d543b3f4dac20b9480a67be31b727848d8b3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 00:22:59 +0000 Subject: [PATCH 207/270] chore(deps): update codecov/codecov-action digest to 5ecb98a --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 03576fa310..70b9cfd0a8 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -184,7 +184,7 @@ jobs: coverage report coverage xml - - uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # v4 + - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be # v4 with: files: coverage.xml env: From 3ad567e960f32c281c8630d002b1e350c60c8298 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 00:58:39 +0000 Subject: [PATCH 208/270] Update gitpod/workspace-base Docker digest to 124f2b8 --- nf_core/gitpod/gitpod.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index be4feab2c5..6afca0e479 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:5aeb24f9994e8226744a200b8316d5d429c0ac98e0b49d1c05ee7d376d8420c5 +FROM gitpod/workspace-base@sha256:124f2b8cbefe9b4abbb6a14538da8846770dde20b93f038d9551b6230aec1d1c USER root From 01d70a99101c1609d7934463b34dc646bee4d43e Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 1 May 2024 04:20:00 +0000 Subject: [PATCH 209/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b618f947ba..50c585879e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) +- Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From cc4e4165c1cca3fd842cc845d3d2963ad787f5f6 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 2 May 2024 11:09:33 +0200 Subject: [PATCH 210/270] More @drpatelh alignment to satisfy @mashehu --- nf_core/pipeline-template/nextflow.config | 136 +++++++++++----------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2aa2c5914c..931ff013a7 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -86,90 +86,90 @@ try { profiles { debug { - dumpHashes = true - process.beforeScript = 'echo $HOSTNAME' - cleanup = false + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + cleanup = false nextflow.enable.configProcessNamesValidation = true } conda { - conda.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - channels = ['conda-forge', 'bioconda', 'defaults'] - apptainer.enabled = false + conda.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + channels = ['conda-forge', 'bioconda', 'defaults'] + apptainer.enabled = false } mamba { - conda.enabled = true - conda.useMamba = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + conda.enabled = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } docker { - docker.enabled = true - conda.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false - docker.runOptions = '-u $(id -u):$(id -g)' + docker.enabled = true + conda.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + docker.runOptions = '-u $(id -u):$(id -g)' } arm { - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { - singularity.enabled = true - singularity.autoMounts = true - conda.enabled = false - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + singularity.enabled = true + singularity.autoMounts = true + conda.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } podman { - podman.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + podman.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } shifter { - shifter.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + shifter.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } charliecloud { - charliecloud.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - apptainer.enabled = false + charliecloud.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + apptainer.enabled = false } apptainer { - apptainer.enabled = true - apptainer.autoMounts = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + apptainer.enabled = true + apptainer.autoMounts = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false } wave { apptainer.ociAutoPull = true @@ -179,9 +179,9 @@ profiles { wave.strategy = 'conda,container' } gitpod { - executor.name = 'local' - executor.cpus = 4 - executor.memory = 8.GB + executor.name = 'local' + executor.cpus = 4 + executor.memory = 8.GB } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } From b63087e4bf960a032b67b0a9d382f69a451c4f3f Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 2 May 2024 11:52:44 +0200 Subject: [PATCH 211/270] fix(template): fix handling of multiple DOIs in manifest --- .../local/utils_nfcore_pipeline_pipeline/main.nf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index e02a113755..07cbb0bc56 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -243,8 +243,14 @@ def methodsDescriptionText(mqc_methods_yaml) { meta["manifest_map"] = workflow.manifest.toMap() // Pipeline DOI - meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" - meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + if (meta.manifest_map.doi) { + def temp_doi_ref = "" + String[] manifest_doi = meta.manifest_map.doi.split(",") + for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref}), " + meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length()-2) + } else { + meta["nodoi_text"] = "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + } // Tool references meta["tool_citations"] = "" From 13e5170296524e03f7b21543841b29848fc1e600 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 2 May 2024 12:04:46 +0200 Subject: [PATCH 212/270] Update nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf --- .../local/utils_nfcore_pipeline_pipeline/main.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 07cbb0bc56..479900d5a9 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -248,9 +248,8 @@ def methodsDescriptionText(mqc_methods_yaml) { String[] manifest_doi = meta.manifest_map.doi.split(",") for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref}), " meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length()-2) - } else { - meta["nodoi_text"] = "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " - } + } else meta["doi_text"] = "" + meta["nodoi_text"] = meta.manifest_map.doi ? "" : "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " // Tool references meta["tool_citations"] = "" From d6e1b102175feaf96dfaa6cda0cfb74629cb0195 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 2 May 2024 12:17:57 +0200 Subject: [PATCH 213/270] fix(collectfile): sort true for methods_description_mqc.yaml --- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 8cdff1baa3..de0f21fe38 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -75,7 +75,7 @@ workflow {{ short_name|upper }} { ch_multiqc_files = ch_multiqc_files.mix( ch_methods_description.collectFile( name: 'methods_description_mqc.yaml', - sort: false + sort: true ) ) From 7786e255ea99d34c34f0d34b63f0b1a039f0655a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 2 May 2024 10:19:31 +0000 Subject: [PATCH 214/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0cca60b0d..9b27f8b8c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) +- fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From f382f980a1be496e96726a8293ae9c031247f925 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 2 May 2024 18:23:42 +0200 Subject: [PATCH 215/270] handles doi.org resolver + comments --- .../local/utils_nfcore_pipeline_pipeline/main.nf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index 479900d5a9..a4bfb9f8be 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -244,10 +244,13 @@ def methodsDescriptionText(mqc_methods_yaml) { // Pipeline DOI if (meta.manifest_map.doi) { + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list def temp_doi_ref = "" - String[] manifest_doi = meta.manifest_map.doi.split(",") - for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref}), " - meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length()-2) + String[] manifest_doi = meta.manifest_map.doi.tokenize(",") + for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref.replace("https://doi.org/", "").replace(" ", "")}), " + meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length() - 2) } else meta["doi_text"] = "" meta["nodoi_text"] = meta.manifest_map.doi ? "" : "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " From fa0b60d642b964d7a32249d0838f6d39a0b8d9af Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 2 May 2024 18:24:41 +0200 Subject: [PATCH 216/270] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0cca60b0d..d2a2bcf042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) - Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Handles multiple DOIs + doi.org resolver from manifest.doi ([#2946](https://github.com/nf-core/tools/pull/2946)) ### Linting From c5d55b8f4a5b4c2107ef6c4be0f660cd24e4796a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 03:48:04 +0000 Subject: [PATCH 217/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b618f947ba..439c38769d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) +- Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 80fcd09575dc2b4972c792622543d2f1e9f39d30 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 3 May 2024 08:03:13 +0200 Subject: [PATCH 218/270] update components to fix CI testing --- nf_core/pipeline-template/modules.json | 4 ++-- nf_core/pipeline-template/modules/nf-core/fastqc/main.nf | 6 ++++++ .../subworkflows/nf-core/utils_nfcore_pipeline/main.nf | 8 +++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 38aa622063..9137c59675 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,7 +7,7 @@ "nf-core": { "fastqc": { "branch": "master", - "git_sha": "f4ae1d942bd50c5c0b9bd2de1393ce38315ba57c", + "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", "installed_by": ["modules"] }, "multiqc": { @@ -26,7 +26,7 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", "installed_by": ["subworkflows"] }, "utils_nfvalidation_plugin": { diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 9e19a74c56..d79f1c862d 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -25,6 +25,11 @@ process FASTQC { def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + + def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') + // FastQC memory value allowed range (100 - 10000) + def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) + """ printf "%s %s\\n" $rename_to | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name @@ -33,6 +38,7 @@ process FASTQC { fastqc \\ $args \\ --threads $task.cpus \\ + --memory $fastqc_memory \\ $renamed_files cat <<-END_VERSIONS > versions.yml diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index a8b55d6fe1..14558c3927 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -65,9 +65,15 @@ def checkProfileProvided(nextflow_cli_args) { // Citation string for pipeline // def workflowCitation() { + def temp_doi_ref = "" + String[] manifest_doi = workflow.manifest.doi.tokenize(",") + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + for (String doi_ref: manifest_doi) temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + "* The pipeline\n" + - " ${workflow.manifest.doi}\n\n" + + temp_doi_ref + "\n" + "* The nf-core framework\n" + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + "* Software dependencies\n" + From 13b1f10f796729acd8c680ca7d16516ac56827a1 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 06:04:58 +0000 Subject: [PATCH 219/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7ba6a7a46..015a1d0825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) +- update components to fix CI testing ([#2949](https://github.com/nf-core/tools/pull/2949)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 32ac3837e94d52f1ce0331cead78f86ef5761dbf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 06:14:02 +0000 Subject: [PATCH 220/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e38d8bb39..5e10a63824 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,13 +58,13 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) - Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) -- update components to fix CI testing ([#2949](https://github.com/nf-core/tools/pull/2949)) - Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) - Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) +- Template: Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 0adbe37e19ae36e3488b0d592e08b08fbaf62d1f Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 3 May 2024 08:16:48 +0200 Subject: [PATCH 221/270] fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e10a63824..40450bc51d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) - Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) ### Linting @@ -64,7 +65,6 @@ - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) - Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) -- Template: Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From e674d3cd5ac416d74a39b7f67eced22922f6516f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 3 May 2024 09:39:00 +0200 Subject: [PATCH 222/270] Template config: `conda.channels`, not `channels` --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index fb3f08c62e..936e5e80d7 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -97,7 +97,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false - channels = ['conda-forge', 'bioconda', 'defaults'] + conda.channels = ['conda-forge', 'bioconda', 'defaults'] apptainer.enabled = false } mamba { From c7ea164792157e04b2ef55a01a672d1f8eb97c38 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 07:40:04 +0000 Subject: [PATCH 223/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e832326fb..d43c0d8838 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Remove `pyproject.toml` from template root - Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) - Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Template config: `conda.channels`, not `channels` ([#2950](https://github.com/nf-core/tools/pull/2950)) ### Linting From d0c17ca718c942c75c67bc7f08e0d61a97b11b57 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 3 May 2024 10:39:53 +0000 Subject: [PATCH 224/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e38e319c0..6ee191a4b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ - Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) - Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) +- Components: Handle extended set of git path prefixes ([#2945](https://github.com/nf-core/tools/pull/2945)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From aa6310939a7edd81a23e894182c2fa0ee9a902fa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 23:27:47 +0000 Subject: [PATCH 225/270] Update python:3.12-slim Docker digest to 2be8dad --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c801563667..ae3a4e1a3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:541d45d3d675fb8197f534525a671e2f8d66c882b89491f9dda271f4f94dcd06 +FROM python:3.12-slim@sha256:2be8daddbb82756f7d1f2c7ece706aadcb284bf6ab6d769ea695cc3ed6016743 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From ffa64950ea26064ac9f0cbc8a839ddc74d53373b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 00:58:27 +0000 Subject: [PATCH 226/270] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d8f35e2cc5..887cbe027c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.2 + rev: v0.4.3 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 7cfcdd69eaf6a23180d9e4e45ae88e6989dbdf2e Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 4 May 2024 00:59:14 +0000 Subject: [PATCH 227/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a02e9a929..00f0c0ed0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ - Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) - fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 ([#2951](https://github.com/nf-core/tools/pull/2951)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 6ab6c8ddb3a8a347e506e3a7833d7baec55bb044 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 23 Apr 2024 14:57:22 +0200 Subject: [PATCH 228/270] Update Docstring of DownloadWorkflow class. --- nf_core/download.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 8ffecc0f55..2978dd08ad 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -88,10 +88,17 @@ class DownloadWorkflow: Args: pipeline (str): A nf-core pipeline name. - revision (List[str]): The workflow revision to download, like `1.0`. Defaults to None. - container (bool): Flag, if the Singularity container should be downloaded as well. Defaults to False. - platform (bool): Flag, to customize the download for Seqera Platform (convert to git bare repo). Defaults to False. + revision (List[str]): The workflow revision(s) to download, like `1.0` or `dev` . Defaults to None. outdir (str): Path to the local download directory. Defaults to None. + compress_type (str): Type of compression for the downloaded files. Defaults to None. + force (bool): Flag to force download even if files already exist (overwrite existing files). Defaults to False. + platform (bool): Flag to customize the download for Seqera Platform (convert to git bare repo). Defaults to False. + download_configuration (str): Download the configuration files from nf-core/configs. Defaults to None. + container_system (str): The container system to use (e.g., "singularity"). Defaults to None. + container_library (str): The container libraries (registries) to use. Defaults to None. + container_cache_utilisation (str): If a local or remote cache of already existing container images should be considered. Defaults to None. + container_cache_index (str): An index for the remote container cache. Defaults to None. + parallel_downloads (int): The number of parallel downloads to use. Defaults to 4. """ def __init__( From d2f8525c7abb89e658ce747e11a5f0483a3a805b Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 23 Apr 2024 17:22:16 +0200 Subject: [PATCH 229/270] Include the '--additional_tags' argument in the CLI for 'nf-core download' --- nf_core/__main__.py | 10 +++++++++- tests/test_cli.py | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 807bc776bb..f3212200d7 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -372,7 +372,7 @@ def create_params_file(pipeline, revision, output, force, show_hidden): is_flag=True, default=False, hidden=True, - help="Download for Seqera Platform. DEPRECATED: Please use --platform instead.", + help="Download for Seqera Platform. DEPRECATED: Please use `--platform` instead.", ) @click.option( "-t", @@ -388,6 +388,12 @@ def create_params_file(pipeline, revision, output, force, show_hidden): default=False, help="Include configuration profiles in download. Not available with `--platform`", ) +@click.option( + "-a", + "--additional-tags", + multiple=True, + help="Add custom alias tags to `--platform` downloads. For example, '-a \"3.10=validated\"' adds the custom 'validated' tag to the 3.10 release.", +) # -c changed to -s for consistency with other --container arguments, where it is always the first letter of the last word. # Also -c might be used instead of -d for config in a later release, but reusing params for different options in two subsequent releases might be too error-prone. @click.option( @@ -430,6 +436,7 @@ def download( tower, platform, download_configuration, + additional_tags, container_system, container_library, container_cache_utilisation, @@ -452,6 +459,7 @@ def download( force, tower or platform, # True if either specified download_configuration, + additional_tags, container_system, container_library, container_cache_utilisation, diff --git a/tests/test_cli.py b/tests/test_cli.py index 913a4aac1d..826008f033 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -168,6 +168,7 @@ def test_cli_download(self, mock_dl): "force": None, "platform": None, "download-configuration": None, + "additional-tags": "3.12=testing", "container-system": "singularity", "container-library": "quay.io", "container-cache-utilisation": "copy", @@ -188,6 +189,7 @@ def test_cli_download(self, mock_dl): "force" in params, "platform" in params, "download-configuration" in params, + (params["additional-tags"],), params["container-system"], (params["container-library"],), params["container-cache-utilisation"], From 550da8650fd0a893f1882b7e6c9cc6cb13c77615 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 23 Apr 2024 18:15:04 +0200 Subject: [PATCH 230/270] Started with additional_tags handling in the level of the DownloadWorkflow class. ToDo for WorkflowRepo class. --- nf_core/download.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index 2978dd08ad..d12b498b29 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -94,8 +94,9 @@ class DownloadWorkflow: force (bool): Flag to force download even if files already exist (overwrite existing files). Defaults to False. platform (bool): Flag to customize the download for Seqera Platform (convert to git bare repo). Defaults to False. download_configuration (str): Download the configuration files from nf-core/configs. Defaults to None. + additional_tags (List[str]): Specify additional tags to add to the downloaded pipeline. Defaults to None. container_system (str): The container system to use (e.g., "singularity"). Defaults to None. - container_library (str): The container libraries (registries) to use. Defaults to None. + container_library (List[str]): The container libraries (registries) to use. Defaults to None. container_cache_utilisation (str): If a local or remote cache of already existing container images should be considered. Defaults to None. container_cache_index (str): An index for the remote container cache. Defaults to None. parallel_downloads (int): The number of parallel downloads to use. Defaults to 4. @@ -110,6 +111,7 @@ def __init__( force=False, platform=False, download_configuration=None, + additional_tags=None, container_system=None, container_library=None, container_cache_utilisation=None, @@ -132,6 +134,15 @@ def __init__( # this implies that non-interactive "no" choice is only possible implicitly (e.g. with --platform or if prompt is suppressed by !stderr.is_interactive). # only alternative would have been to make it a parameter with argument, e.g. -d="yes" or -d="no". self.include_configs = True if download_configuration else False if bool(platform) else None + # Additional tags to add to the downloaded pipeline. This enables to mark particular commits or revisions with + # additional tags, e.g. "stable", "testing", "validated", "production" etc. Since this requires a git-repo, it is only + # available for the bare / Seqera Platform download. + if isinstance(additional_tags, str) and bool(len(additional_tags)) and self.platform: + self.additional_tags = [additional_tags] + elif isinstance(additional_tags, tuple) and bool(len(additional_tags)) and self.platform: + self.additional_tags = [*additional_tags] + else: + self.additional_tags = None # Specifying a cache index or container library implies that containers should be downloaded. self.container_system = "singularity" if container_cache_index or bool(container_library) else container_system # Manually specified container library (registry) @@ -289,6 +300,7 @@ def download_workflow_platform(self, location=None): remote_url=f"https://github.com/{self.pipeline}.git", revision=self.revision if self.revision else None, commit=self.wf_sha.values() if bool(self.wf_sha) else None, + additional_tags=self.additional_tags if self.additional_tags else None, location=(location if location else None), # manual location is required for the tests to work in_cache=False, ) @@ -1496,6 +1508,7 @@ def __init__( remote_url, revision, commit, + additional_tags, location=None, hide_progress=False, in_cache=True, @@ -1533,6 +1546,9 @@ def __init__( # expose some instance attributes self.tags = self.repo.tags + # additional tags to be added to the repository + self.additional_tags = additional_tags + def __repr__(self): """Called by print, creates representation of object""" return f"" From 620ca82da9c375a872f656c05f098c7bbd42000c Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 24 Apr 2024 17:21:56 +0200 Subject: [PATCH 231/270] Implement a private __add_additional_tags() method for the WorkflowRepo class in 'nf-core download'. --- nf_core/download.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index d12b498b29..184a02d32b 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -308,6 +308,8 @@ def download_workflow_platform(self, location=None): # Remove tags for those revisions that had not been selected self.workflow_repo.tidy_tags_and_branches() + import pdb; pdb.set_trace() + # create a bare clone of the modified repository needed for Seqera Platform self.workflow_repo.bare_clone(os.path.join(self.outdir, self.output_filename)) @@ -1543,16 +1545,21 @@ def __init__( self.setup_local_repo(remote=remote_url, location=location, in_cache=in_cache) - # expose some instance attributes - self.tags = self.repo.tags - # additional tags to be added to the repository - self.additional_tags = additional_tags + self.additional_tags = additional_tags if additional_tags else None def __repr__(self): """Called by print, creates representation of object""" return f"" + @property + def heads(self): + return self.repo.heads + + @property + def tags(self): + return self.repo.tags + def access(self): if os.path.exists(self.local_repo_dir): return self.local_repo_dir @@ -1663,7 +1670,6 @@ def tidy_tags_and_branches(self): # delete unwanted tags from repository for tag in tags_to_remove: self.repo.delete_tag(tag) - self.tags = self.repo.tags # switch to a revision that should be kept, because deleting heads fails, if they are checked out (e.g. "master") self.checkout(self.revision[0]) @@ -1700,7 +1706,8 @@ def tidy_tags_and_branches(self): if self.repo.head.is_detached: self.repo.head.reset(index=True, working_tree=True) - self.heads = self.repo.heads + # Apply the custom additional tags to the repository + self.__add_additional_tags() # get all tags and available remote_branches completed_revisions = {revision.name for revision in self.repo.heads + self.repo.tags} @@ -1718,6 +1725,25 @@ def tidy_tags_and_branches(self): self.retry_setup_local_repo(skip_confirm=True) raise DownloadError(e) from e + # "Private" method to add the additional custom tags to the repository. + def __add_additional_tags(self) -> None: + if self.additional_tags: + for additional_tag in self.additional_tags: + # A valid git branch or tag name can contain alphanumeric characters, underscores, hyphens, and dots. + # But it must not start with a dot, hyphen or underscore and also cannot contain two consecutive dots. + if re.match(r"^\w[\w_.-]+={1}\w[\w_.-]+$",additional_tag) and '..' not in additional_tag: + anchor, tag = additional_tag.split("=") + if self.repo.is_valid_object(anchor) and not self.repo.is_valid_object(tag): + try: + self.repo.create_tag(tag,ref=anchor,message=f"Synonynmous tag to {anchor}; added by 'nf-core download'.") + self.repo.create_head(tag,anchor) + except (GitCommandError, InvalidGitRepositoryError) as e: + log.error(f"[red]Additional tag(s) could not be applied:[/]\n{e}\n") + else: + log.error(f"[red]Adding the additional tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{anchor}' must be a valid git reference that resolves to a commit, while '{tag}' must not exist hitherto.") + else: + log.error(f"[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: '{additional_tag}'") + def bare_clone(self, destination): if self.repo: try: From da0eab10ecf079ea44f85e7b4d96cd8bd5a06721 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 25 Apr 2024 21:02:54 +0200 Subject: [PATCH 232/270] Experiment with capturing log messages in tests. --- nf_core/download.py | 4 +-- tests/test_download.py | 61 +++++++++++++++++++++++++++++++++++++++++- tests/utils.py | 52 ++++++++++++++++++++++++++++++++++- 3 files changed, 112 insertions(+), 5 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 184a02d32b..62fe08e7d5 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -308,8 +308,6 @@ def download_workflow_platform(self, location=None): # Remove tags for those revisions that had not been selected self.workflow_repo.tidy_tags_and_branches() - import pdb; pdb.set_trace() - # create a bare clone of the modified repository needed for Seqera Platform self.workflow_repo.bare_clone(os.path.join(self.outdir, self.output_filename)) @@ -1736,7 +1734,7 @@ def __add_additional_tags(self) -> None: if self.repo.is_valid_object(anchor) and not self.repo.is_valid_object(tag): try: self.repo.create_tag(tag,ref=anchor,message=f"Synonynmous tag to {anchor}; added by 'nf-core download'.") - self.repo.create_head(tag,anchor) + self.repo.create_head(tag,anchor) # should heads be created as well? except (GitCommandError, InvalidGitRepositoryError) as e: log.error(f"[red]Additional tag(s) could not be applied:[/]\n{e}\n") else: diff --git a/tests/test_download.py b/tests/test_download.py index 14a96be26f..0fc3abae60 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -16,7 +16,7 @@ from nf_core.synced_repo import SyncedRepo from nf_core.utils import run_cmd -from .utils import with_temporary_folder +from .utils import with_logger, with_temporary_folder class DownloadTest(unittest.TestCase): @@ -623,3 +623,62 @@ def test_download_workflow_for_platform(self, tmp_dir, _): "https://depot.galaxyproject.org/singularity/mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:59cdd445419f14abac76b31dd0d71217994cbcc9-0" in download_obj.containers ) # indirect definition via $container variable. + + + # + # Test adding custom tags to Seqera Platform download + # + @with_temporary_folder + @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") + @with_logger + def test_download_workflow_for_platform_with_custom_tags(self, tmp_dir, _, logger): + + from git.refs.head import Head + from git.refs.tag import TagReference + + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.7", "3.9"), + compress_type="none", + platform=True, + container_system=None, + additional_tags=("3.7=a.tad.outdated", "3.9=cool_revision", "3.9=invalid tag","3.14.0=not_included","What is this?"), + ) + + download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. + + assert isinstance(download_obj.revision, list) and len(download_obj.revision) == 2 + assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 0 + assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 + assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 5 + + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(download_obj.pipeline, wfs) + + download_obj.get_revision_hash() + download_obj.output_filename = f"{download_obj.outdir}.git" + download_obj.download_workflow_platform(location=tmp_dir) + + assert download_obj.workflow_repo + assert isinstance(download_obj.workflow_repo, WorkflowRepo) + assert issubclass(type(download_obj.workflow_repo), SyncedRepo) + + # assert that every additional tag has been passed on to the WorkflowRepo instance + assert download_obj.additional_tags == download_obj.workflow_repo.additional_tags + + # assert that the additional tags are all TagReference objects + assert all(isinstance(tag, TagReference) for tag in download_obj.workflow_repo.tags) + + workflow_repo_tags = {tag.name for tag in download_obj.workflow_repo.tags} + assert len(workflow_repo_tags) == 4 + # the invalid/malformed additional_tags should not have been added. + assert all(tag in workflow_repo_tags for tag in {'3.7', 'a.tad.outdated', 'cool_revision', '3.9'}) + assert not any(tag in workflow_repo_tags for tag in {'invalid tag','not_included','What is this?'}) + + import pdb; pdb.set_trace() + diff --git a/tests/utils.py b/tests/utils.py index 89c1328818..c87e2c5ecd 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,7 +1,7 @@ """ Helper functions for tests """ - +from logging import getLogger, Logger, LogRecord, Handler import functools import os import tempfile @@ -56,6 +56,56 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return wrapper +class FirstHandlerProxy: + """ + A proxy so the in operator can be used directly on the logger for convenience in asserting the test log messages. + """ + def __init__(self, logger: Logger) -> None: + self.logger = logger + + def __contains__(self, message: str) -> bool: + return message in self.logger.handlers[0] + + def get_log_messages(self) -> list[str]: + return self.logger.handlers[0].log_messages + +class LogToList(Handler): + """ + A logging handler that appends log messages to a list for simple log message capturing. + """ + def __init__(self) -> None: + super().__init__() + self.log_messages = [] + + def __repr__(self) -> str: + return str(self.log_messages) + + # enable using the in operator directly on the handler + def __contains__(self, message: str) -> bool: + return message in self.log_messages + + def emit(self, record: LogRecord) -> None: + self.log_messages.append(self.format(record)) + +def with_logger(func: Callable[..., Any]) -> Callable[..., Any]: + """ + Captures the log messages from a decorated test into a dictionary. + Pass the logger to the decorated function and assert with the _in_ operator. + + assert 'This is a log message' in test_logger + """ + + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + logger = getLogger(func.__name__) + list_handler = LogToList() + logger.addHandler(list_handler) + try: + return func(*args, FirstHandlerProxy(logger), **kwargs) + finally: + logger.removeHandler(list_handler) + + return wrapper def mock_anaconda_api_calls(rsps: responses.RequestsMock, module: str, version: str) -> None: """Mock anaconda api calls for module""" From 113c9d03990aca415e0e7cb315f03a72723e84b9 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 26 Apr 2024 19:00:40 +0200 Subject: [PATCH 233/270] Finally managed to assert over log messages, thanks @adamrtalbot . --- tests/test_download.py | 107 ++++++++++++++++++++++++----------------- tests/utils.py | 53 +------------------- 2 files changed, 64 insertions(+), 96 deletions(-) diff --git a/tests/test_download.py b/tests/test_download.py index 0fc3abae60..6928a9087a 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -1,5 +1,6 @@ """Tests for the download subcommand of nf-core tools""" +import logging import os import re import shutil @@ -16,10 +17,29 @@ from nf_core.synced_repo import SyncedRepo from nf_core.utils import run_cmd -from .utils import with_logger, with_temporary_folder +from .utils import with_temporary_folder class DownloadTest(unittest.TestCase): + + @pytest.fixture(autouse=True) + def use_caplog(self, caplog): + self._caplog = caplog + + @property + def logged_levels(self) -> list[str]: + return [record.levelname for record in self._caplog.records] + + @property + def logged_messages(self) -> list[str]: + return [record.message for record in self._caplog.records] + + def __contains__(self, item: str) -> bool: + """Allows to check for log messages easily using the in operator inside a test: + assert 'my log message' in self + """ + return any(record.message == item for record in self._caplog.records if self._caplog) + # # Tests for 'get_release_hash' # @@ -628,57 +648,56 @@ def test_download_workflow_for_platform(self, tmp_dir, _): # # Test adding custom tags to Seqera Platform download # - @with_temporary_folder @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") - @with_logger - def test_download_workflow_for_platform_with_custom_tags(self, tmp_dir, _, logger): - - from git.refs.head import Head - from git.refs.tag import TagReference - - download_obj = DownloadWorkflow( - pipeline="nf-core/rnaseq", - revision=("3.7", "3.9"), - compress_type="none", - platform=True, - container_system=None, - additional_tags=("3.7=a.tad.outdated", "3.9=cool_revision", "3.9=invalid tag","3.14.0=not_included","What is this?"), - ) + @with_temporary_folder + def test_download_workflow_for_platform_with_custom_tags(self,_, tmp_dir): + with self._caplog.at_level(logging.INFO): + from git.refs.head import Head + from git.refs.tag import TagReference + + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.7", "3.9"), + compress_type="none", + platform=True, + container_system=None, + additional_tags=("3.7=a.tad.outdated", "3.9=cool_revision", "3.9=invalid tag","3.14.0=not_included","What is this?"), + ) - download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. + download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. - assert isinstance(download_obj.revision, list) and len(download_obj.revision) == 2 - assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 0 - assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 - assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 5 + assert isinstance(download_obj.revision, list) and len(download_obj.revision) == 2 + assert isinstance(download_obj.wf_sha, dict) and len(download_obj.wf_sha) == 0 + assert isinstance(download_obj.wf_download_url, dict) and len(download_obj.wf_download_url) == 0 + assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 5 - wfs = nf_core.list.Workflows() - wfs.get_remote_workflows() - ( - download_obj.pipeline, - download_obj.wf_revisions, - download_obj.wf_branches, - ) = nf_core.utils.get_repo_releases_branches(download_obj.pipeline, wfs) + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + ( + download_obj.pipeline, + download_obj.wf_revisions, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(download_obj.pipeline, wfs) - download_obj.get_revision_hash() - download_obj.output_filename = f"{download_obj.outdir}.git" - download_obj.download_workflow_platform(location=tmp_dir) + download_obj.get_revision_hash() + download_obj.output_filename = f"{download_obj.outdir}.git" + download_obj.download_workflow_platform(location=tmp_dir) - assert download_obj.workflow_repo - assert isinstance(download_obj.workflow_repo, WorkflowRepo) - assert issubclass(type(download_obj.workflow_repo), SyncedRepo) + assert download_obj.workflow_repo + assert isinstance(download_obj.workflow_repo, WorkflowRepo) + assert issubclass(type(download_obj.workflow_repo), SyncedRepo) - # assert that every additional tag has been passed on to the WorkflowRepo instance - assert download_obj.additional_tags == download_obj.workflow_repo.additional_tags + # assert that every additional tag has been passed on to the WorkflowRepo instance + assert download_obj.additional_tags == download_obj.workflow_repo.additional_tags - # assert that the additional tags are all TagReference objects - assert all(isinstance(tag, TagReference) for tag in download_obj.workflow_repo.tags) + # assert that the additional tags are all TagReference objects + assert all(isinstance(tag, TagReference) for tag in download_obj.workflow_repo.tags) - workflow_repo_tags = {tag.name for tag in download_obj.workflow_repo.tags} - assert len(workflow_repo_tags) == 4 - # the invalid/malformed additional_tags should not have been added. - assert all(tag in workflow_repo_tags for tag in {'3.7', 'a.tad.outdated', 'cool_revision', '3.9'}) - assert not any(tag in workflow_repo_tags for tag in {'invalid tag','not_included','What is this?'}) + workflow_repo_tags = {tag.name for tag in download_obj.workflow_repo.tags} + assert len(workflow_repo_tags) == 4 + # the invalid/malformed additional_tags should not have been added. + assert all(tag in workflow_repo_tags for tag in {'3.7', 'a.tad.outdated', 'cool_revision', '3.9'}) + assert not any(tag in workflow_repo_tags for tag in {'invalid tag','not_included','What is this?'}) - import pdb; pdb.set_trace() + assert all(log in self.logged_messages for log in {"[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: '3.9=invalid tag'", "[red]Adding the additional tag 'not_included' to '3.14.0' failed.[/]\n Mind that '3.14.0' must be a valid git reference that resolves to a commit, while 'not_included' must not exist hitherto.", "[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: 'What is this?'"}) diff --git a/tests/utils.py b/tests/utils.py index c87e2c5ecd..4635c53b7b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,7 +1,7 @@ """ Helper functions for tests """ -from logging import getLogger, Logger, LogRecord, Handler + import functools import os import tempfile @@ -56,57 +56,6 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return wrapper -class FirstHandlerProxy: - """ - A proxy so the in operator can be used directly on the logger for convenience in asserting the test log messages. - """ - def __init__(self, logger: Logger) -> None: - self.logger = logger - - def __contains__(self, message: str) -> bool: - return message in self.logger.handlers[0] - - def get_log_messages(self) -> list[str]: - return self.logger.handlers[0].log_messages - -class LogToList(Handler): - """ - A logging handler that appends log messages to a list for simple log message capturing. - """ - def __init__(self) -> None: - super().__init__() - self.log_messages = [] - - def __repr__(self) -> str: - return str(self.log_messages) - - # enable using the in operator directly on the handler - def __contains__(self, message: str) -> bool: - return message in self.log_messages - - def emit(self, record: LogRecord) -> None: - self.log_messages.append(self.format(record)) - -def with_logger(func: Callable[..., Any]) -> Callable[..., Any]: - """ - Captures the log messages from a decorated test into a dictionary. - Pass the logger to the decorated function and assert with the _in_ operator. - - assert 'This is a log message' in test_logger - """ - - @functools.wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: - logger = getLogger(func.__name__) - list_handler = LogToList() - logger.addHandler(list_handler) - try: - return func(*args, FirstHandlerProxy(logger), **kwargs) - finally: - logger.removeHandler(list_handler) - - return wrapper - def mock_anaconda_api_calls(rsps: responses.RequestsMock, module: str, version: str) -> None: """Mock anaconda api calls for module""" anaconda_api_url = f"https://api.anaconda.org/package/bioconda/{module}" From 35b4fd66e03e36411924e17b7864ed3ceeb56f6f Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 26 Apr 2024 19:40:56 +0200 Subject: [PATCH 234/270] Update the README.md to reflect the '-a' / '--additional-tags' argument of 'nf-core download'. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a5e6799861..65cfc21b3a 100644 --- a/README.md +++ b/README.md @@ -437,6 +437,8 @@ Subsequently, the `*.git` folder can be moved to it's final destination and link > [!TIP] > Also without access to Seqera Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. +Facilities and those who are setting up pipelines for others to use may find the '-a' / '--additional-tag' argument helpful. It must be followed by a string in a "key=value" format and can be provided multiple times. The left-hand side must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. The '-a' / '--additional-tag' argument allows to customize the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: '-a "3.12.0=testing" -a "3.9.0=validated"' so their staff can easily ensure that the correct version of the pipeline is run in production. + ## Pipeline software licences Sometimes it's useful to see the software licences of the tools used in a pipeline. From 3901f2de4e4167e38e7aa05d99774ee24c42d309 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 26 Apr 2024 21:23:22 +0200 Subject: [PATCH 235/270] Update CHANGELOG: Minor version jump, because CLI arguments changed. --- CHANGELOG.md | 5 +++-- README.md | 2 +- tests/utils.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88ce77a3b8..5648603280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v2.13.2dev +## v2.14.0dev ### Template @@ -25,8 +25,9 @@ ### Download -- Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. +- Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. ([#2853](https://github.com/nf-core/tools/pull/2853)) - Better error message when GITHUB_TOKEN exists but is wrong/outdated +- New `-a` / `--additional-tag` argument to add custom tags during a pipeline download ([#2938](https://github.com/nf-core/tools/pull/2938)) ### Components diff --git a/README.md b/README.md index 65cfc21b3a..b28be6efa3 100644 --- a/README.md +++ b/README.md @@ -437,7 +437,7 @@ Subsequently, the `*.git` folder can be moved to it's final destination and link > [!TIP] > Also without access to Seqera Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. -Facilities and those who are setting up pipelines for others to use may find the '-a' / '--additional-tag' argument helpful. It must be followed by a string in a "key=value" format and can be provided multiple times. The left-hand side must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. The '-a' / '--additional-tag' argument allows to customize the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: '-a "3.12.0=testing" -a "3.9.0=validated"' so their staff can easily ensure that the correct version of the pipeline is run in production. +Facilities and those who are setting up pipelines for others to use may find the `-a` / `--additional-tag` argument helpful. It must be followed by a string in a "key=value" format and can be provided multiple times. The left-hand side must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. The `-a` / `--additional-tag` argument allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `-a "3.12.0=testing" -a "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production. ## Pipeline software licences diff --git a/tests/utils.py b/tests/utils.py index 4635c53b7b..89c1328818 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -56,6 +56,7 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return wrapper + def mock_anaconda_api_calls(rsps: responses.RequestsMock, module: str, version: str) -> None: """Mock anaconda api calls for module""" anaconda_api_url = f"https://api.anaconda.org/package/bioconda/{module}" From 00cb57eecc9c831d6f10f95ad86c83e33d840f0d Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 26 Apr 2024 21:42:01 +0200 Subject: [PATCH 236/270] Ensure a user is configured in Git config. Required for Github Actions, but may also affect normal usage. --- nf_core/download.py | 18 +++++++++++++----- nf_core/synced_repo.py | 16 ++++++++++++++++ tests/test_download.py | 35 ++++++++++++++++++++++------------- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 62fe08e7d5..89a2e39d71 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1726,21 +1726,29 @@ def tidy_tags_and_branches(self): # "Private" method to add the additional custom tags to the repository. def __add_additional_tags(self) -> None: if self.additional_tags: + self.ensure_git_user_config(f"nf-core download v{nf_core.__version__}", "core@nf-co.re") + for additional_tag in self.additional_tags: # A valid git branch or tag name can contain alphanumeric characters, underscores, hyphens, and dots. # But it must not start with a dot, hyphen or underscore and also cannot contain two consecutive dots. - if re.match(r"^\w[\w_.-]+={1}\w[\w_.-]+$",additional_tag) and '..' not in additional_tag: + if re.match(r"^\w[\w_.-]+={1}\w[\w_.-]+$", additional_tag) and ".." not in additional_tag: anchor, tag = additional_tag.split("=") if self.repo.is_valid_object(anchor) and not self.repo.is_valid_object(tag): try: - self.repo.create_tag(tag,ref=anchor,message=f"Synonynmous tag to {anchor}; added by 'nf-core download'.") - self.repo.create_head(tag,anchor) # should heads be created as well? + self.repo.create_tag( + tag, ref=anchor, message=f"Synonynmous tag to {anchor}; added by 'nf-core download'." + ) + self.repo.create_head(tag, anchor) # should heads be created as well? except (GitCommandError, InvalidGitRepositoryError) as e: log.error(f"[red]Additional tag(s) could not be applied:[/]\n{e}\n") else: - log.error(f"[red]Adding the additional tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{anchor}' must be a valid git reference that resolves to a commit, while '{tag}' must not exist hitherto.") + log.error( + f"[red]Adding the additional tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{anchor}' must be a valid git reference that resolves to a commit, while '{tag}' must not exist hitherto." + ) else: - log.error(f"[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: '{additional_tag}'") + log.error( + f"[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: '{additional_tag}'" + ) def bare_clone(self, destination): if self.repo: diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 5c31e96911..5e1587baef 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -116,6 +116,10 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url + self.repo = None + # ToDo: SyncedRepo doesn't have this method and both the ModulesRepo and + # the WorkflowRepo define their own including custom init methods. This needs + # fixing, but is beyond the scope of this PR. self.setup_local_repo(remote_url, branch, hide_progress) config_fn, repo_config = load_tools_config(self.local_repo_dir) @@ -326,6 +330,18 @@ def component_files_identical(self, component_name, base_path, commit, component self.checkout_branch() return files_identical + def ensure_git_user_config(self, default_name: str, default_email: str) -> None: + with self.repo.config_reader() as git_config: + user_name = git_config.get_value("user", "name", default=None) + user_email = git_config.get_value("user", "email", default=None) + + if not user_name or not user_email: + with self.repo.config_writer() as git_config: + if not user_name: + git_config.set_value("user", "name", default_name) + if not user_email: + git_config.set_value("user", "email", default_email) + def get_component_git_log(self, component_name, component_type, depth=None): """ Fetches the commit history the of requested module/subworkflow since a given date. The default value is diff --git a/tests/test_download.py b/tests/test_download.py index 6928a9087a..9794151abe 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -21,22 +21,21 @@ class DownloadTest(unittest.TestCase): - @pytest.fixture(autouse=True) def use_caplog(self, caplog): self._caplog = caplog @property - def logged_levels(self) -> list[str]: + def logged_levels(self) -> list: return [record.levelname for record in self._caplog.records] @property - def logged_messages(self) -> list[str]: + def logged_messages(self) -> list: return [record.message for record in self._caplog.records] def __contains__(self, item: str) -> bool: """Allows to check for log messages easily using the in operator inside a test: - assert 'my log message' in self + assert 'my log message' in self """ return any(record.message == item for record in self._caplog.records if self._caplog) @@ -644,15 +643,13 @@ def test_download_workflow_for_platform(self, tmp_dir, _): in download_obj.containers ) # indirect definition via $container variable. - # # Test adding custom tags to Seqera Platform download # @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") @with_temporary_folder - def test_download_workflow_for_platform_with_custom_tags(self,_, tmp_dir): + def test_download_workflow_for_platform_with_custom_tags(self, _, tmp_dir): with self._caplog.at_level(logging.INFO): - from git.refs.head import Head from git.refs.tag import TagReference download_obj = DownloadWorkflow( @@ -661,7 +658,13 @@ def test_download_workflow_for_platform_with_custom_tags(self,_, tmp_dir): compress_type="none", platform=True, container_system=None, - additional_tags=("3.7=a.tad.outdated", "3.9=cool_revision", "3.9=invalid tag","3.14.0=not_included","What is this?"), + additional_tags=( + "3.7=a.tad.outdated", + "3.9=cool_revision", + "3.9=invalid tag", + "3.14.0=not_included", + "What is this?", + ), ) download_obj.include_configs = False # suppress prompt, because stderr.is_interactive doesn't. @@ -696,8 +699,14 @@ def test_download_workflow_for_platform_with_custom_tags(self,_, tmp_dir): workflow_repo_tags = {tag.name for tag in download_obj.workflow_repo.tags} assert len(workflow_repo_tags) == 4 # the invalid/malformed additional_tags should not have been added. - assert all(tag in workflow_repo_tags for tag in {'3.7', 'a.tad.outdated', 'cool_revision', '3.9'}) - assert not any(tag in workflow_repo_tags for tag in {'invalid tag','not_included','What is this?'}) - - assert all(log in self.logged_messages for log in {"[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: '3.9=invalid tag'", "[red]Adding the additional tag 'not_included' to '3.14.0' failed.[/]\n Mind that '3.14.0' must be a valid git reference that resolves to a commit, while 'not_included' must not exist hitherto.", "[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: 'What is this?'"}) - + assert all(tag in workflow_repo_tags for tag in {"3.7", "a.tad.outdated", "cool_revision", "3.9"}) + assert not any(tag in workflow_repo_tags for tag in {"invalid tag", "not_included", "What is this?"}) + + assert all( + log in self.logged_messages + for log in { + "[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: '3.9=invalid tag'", + "[red]Adding the additional tag 'not_included' to '3.14.0' failed.[/]\n Mind that '3.14.0' must be a valid git reference that resolves to a commit, while 'not_included' must not exist hitherto.", + "[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: 'What is this?'", + } + ) From e089676df548596dc6162115759e47e0e6cbf064 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Mon, 29 Apr 2024 16:23:37 +0200 Subject: [PATCH 237/270] Catch exceptions if section is missing in the Git config. --- nf_core/synced_repo.py | 10 +++++++--- tests/test_download.py | 22 ++++++++++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 5e1587baef..f72309780d 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -2,6 +2,7 @@ import logging import os import shutil +from configparser import NoOptionError, NoSectionError from pathlib import Path from typing import Dict @@ -331,9 +332,12 @@ def component_files_identical(self, component_name, base_path, commit, component return files_identical def ensure_git_user_config(self, default_name: str, default_email: str) -> None: - with self.repo.config_reader() as git_config: - user_name = git_config.get_value("user", "name", default=None) - user_email = git_config.get_value("user", "email", default=None) + try: + with self.repo.config_reader() as git_config: + user_name = git_config.get_value("user", "name", default=None) + user_email = git_config.get_value("user", "email", default=None) + except (NoOptionError, NoSectionError): + user_name = user_email = None if not user_name or not user_email: with self.repo.config_writer() as git_config: diff --git a/tests/test_download.py b/tests/test_download.py index 9794151abe..8213b980d9 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -7,6 +7,7 @@ import tempfile import unittest from pathlib import Path +from typing import List from unittest import mock import pytest @@ -26,11 +27,11 @@ def use_caplog(self, caplog): self._caplog = caplog @property - def logged_levels(self) -> list: + def logged_levels(self) -> List[str]: return [record.levelname for record in self._caplog.records] @property - def logged_messages(self) -> list: + def logged_messages(self) -> List[str]: return [record.message for record in self._caplog.records] def __contains__(self, item: str) -> bool: @@ -689,6 +690,7 @@ def test_download_workflow_for_platform_with_custom_tags(self, _, tmp_dir): assert download_obj.workflow_repo assert isinstance(download_obj.workflow_repo, WorkflowRepo) assert issubclass(type(download_obj.workflow_repo), SyncedRepo) + assert "Locally cached repository: nf-core/rnaseq, revisions 3.7, 3.9" in repr(download_obj.workflow_repo) # assert that every additional tag has been passed on to the WorkflowRepo instance assert download_obj.additional_tags == download_obj.workflow_repo.additional_tags @@ -710,3 +712,19 @@ def test_download_workflow_for_platform_with_custom_tags(self, _, tmp_dir): "[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: 'What is this?'", } ) + + # + # Test adding a single custom tags to Seqera Platform download so CodeCov is happy. + # + @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") + @with_temporary_folder + def test_download_workflow_for_platform_with_one_custom_tag(self, _, tmp_dir): + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.9"), + compress_type="none", + platform=True, + container_system=None, + additional_tags=("3.9=cool_revision",), + ) + assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 1 From 6688079dc4bf891847a6d99c30b9fcf2a6d4a038 Mon Sep 17 00:00:00 2001 From: Matthias Zepper <6963520+MatthiasZepper@users.noreply.github.com> Date: Fri, 3 May 2024 18:37:01 +0200 Subject: [PATCH 238/270] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- README.md | 2 +- nf_core/download.py | 2 +- nf_core/synced_repo.py | 4 ++-- tests/test_download.py | 3 --- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b28be6efa3..b63514b44a 100644 --- a/README.md +++ b/README.md @@ -437,7 +437,7 @@ Subsequently, the `*.git` folder can be moved to it's final destination and link > [!TIP] > Also without access to Seqera Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. -Facilities and those who are setting up pipelines for others to use may find the `-a` / `--additional-tag` argument helpful. It must be followed by a string in a "key=value" format and can be provided multiple times. The left-hand side must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. The `-a` / `--additional-tag` argument allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `-a "3.12.0=testing" -a "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production. +Facilities and those who are setting up pipelines for others to use may find the `-a` / `--additional-tag` argument helpful. It must be followed by a string in a `key=value` format and can be provided multiple times. The `key` must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. The `-a` / `--additional-tag` argument allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `-a "3.12.0=testing" -a "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production. ## Pipeline software licences diff --git a/nf_core/download.py b/nf_core/download.py index 89a2e39d71..26e48958fd 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -300,7 +300,7 @@ def download_workflow_platform(self, location=None): remote_url=f"https://github.com/{self.pipeline}.git", revision=self.revision if self.revision else None, commit=self.wf_sha.values() if bool(self.wf_sha) else None, - additional_tags=self.additional_tags if self.additional_tags else None, + additional_tags=self.additional_tags, location=(location if location else None), # manual location is required for the tests to work in_cache=False, ) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index f72309780d..d4f302254f 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -118,9 +118,9 @@ def __init__(self, remote_url=None, branch=None, no_pull=False, hide_progress=Fa self.remote_url = remote_url self.repo = None - # ToDo: SyncedRepo doesn't have this method and both the ModulesRepo and + # TODO: SyncedRepo doesn't have this method and both the ModulesRepo and # the WorkflowRepo define their own including custom init methods. This needs - # fixing, but is beyond the scope of this PR. + # fixing. self.setup_local_repo(remote_url, branch, hide_progress) config_fn, repo_config = load_tools_config(self.local_repo_dir) diff --git a/tests/test_download.py b/tests/test_download.py index 8213b980d9..589fe6edab 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -713,9 +713,6 @@ def test_download_workflow_for_platform_with_custom_tags(self, _, tmp_dir): } ) - # - # Test adding a single custom tags to Seqera Platform download so CodeCov is happy. - # @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") @with_temporary_folder def test_download_workflow_for_platform_with_one_custom_tag(self, _, tmp_dir): From 4d2e68148f75f876b429d03206cc0af98fa822c8 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Fri, 3 May 2024 18:59:19 +0200 Subject: [PATCH 239/270] Update --additional-tag to --tag. --- CHANGELOG.md | 2 +- README.md | 2 +- nf_core/__main__.py | 11 +++++------ nf_core/download.py | 19 +++++++++++-------- tests/test_cli.py | 4 ++-- tests/test_download.py | 37 ++++++++++++++++++++----------------- 6 files changed, 40 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5648603280..d6218f7d29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ - Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. ([#2853](https://github.com/nf-core/tools/pull/2853)) - Better error message when GITHUB_TOKEN exists but is wrong/outdated -- New `-a` / `--additional-tag` argument to add custom tags during a pipeline download ([#2938](https://github.com/nf-core/tools/pull/2938)) +- New `--tag` argument to add custom tags during a pipeline download ([#2938](https://github.com/nf-core/tools/pull/2938)) ### Components diff --git a/README.md b/README.md index b63514b44a..fa909804c4 100644 --- a/README.md +++ b/README.md @@ -437,7 +437,7 @@ Subsequently, the `*.git` folder can be moved to it's final destination and link > [!TIP] > Also without access to Seqera Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. -Facilities and those who are setting up pipelines for others to use may find the `-a` / `--additional-tag` argument helpful. It must be followed by a string in a `key=value` format and can be provided multiple times. The `key` must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. The `-a` / `--additional-tag` argument allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `-a "3.12.0=testing" -a "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production. +Facilities and those who are setting up pipelines for others to use may find the `--tag` argument helpful. It must be followed by a string in a `key=value` format and can be provided multiple times. The `key` must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. The `--tag` argument allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `--tag "3.12.0=testing" --tag "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production. ## Pipeline software licences diff --git a/nf_core/__main__.py b/nf_core/__main__.py index f3212200d7..ead74aa67c 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -368,6 +368,7 @@ def create_params_file(pipeline, revision, output, force, show_hidden): @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") # TODO: Remove this in a future release. Deprecated in March 2024. @click.option( + "-t", "--tower", is_flag=True, default=False, @@ -375,7 +376,6 @@ def create_params_file(pipeline, revision, output, force, show_hidden): help="Download for Seqera Platform. DEPRECATED: Please use `--platform` instead.", ) @click.option( - "-t", "--platform", is_flag=True, default=False, @@ -389,10 +389,9 @@ def create_params_file(pipeline, revision, output, force, show_hidden): help="Include configuration profiles in download. Not available with `--platform`", ) @click.option( - "-a", - "--additional-tags", + "--tag", multiple=True, - help="Add custom alias tags to `--platform` downloads. For example, '-a \"3.10=validated\"' adds the custom 'validated' tag to the 3.10 release.", + help="Add custom alias tags to `--platform` downloads. For example, `--tag \"3.10=validated\"` adds the custom 'validated' tag to the 3.10 release.", ) # -c changed to -s for consistency with other --container arguments, where it is always the first letter of the last word. # Also -c might be used instead of -d for config in a later release, but reusing params for different options in two subsequent releases might be too error-prone. @@ -436,7 +435,7 @@ def download( tower, platform, download_configuration, - additional_tags, + tag, container_system, container_library, container_cache_utilisation, @@ -459,7 +458,7 @@ def download( force, tower or platform, # True if either specified download_configuration, - additional_tags, + tag, container_system, container_library, container_cache_utilisation, diff --git a/nf_core/download.py b/nf_core/download.py index 26e48958fd..5edba513fd 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -94,7 +94,7 @@ class DownloadWorkflow: force (bool): Flag to force download even if files already exist (overwrite existing files). Defaults to False. platform (bool): Flag to customize the download for Seqera Platform (convert to git bare repo). Defaults to False. download_configuration (str): Download the configuration files from nf-core/configs. Defaults to None. - additional_tags (List[str]): Specify additional tags to add to the downloaded pipeline. Defaults to None. + tag (List[str]): Specify additional tags to add to the downloaded pipeline. Defaults to None. container_system (str): The container system to use (e.g., "singularity"). Defaults to None. container_library (List[str]): The container libraries (registries) to use. Defaults to None. container_cache_utilisation (str): If a local or remote cache of already existing container images should be considered. Defaults to None. @@ -1736,19 +1736,22 @@ def __add_additional_tags(self) -> None: if self.repo.is_valid_object(anchor) and not self.repo.is_valid_object(tag): try: self.repo.create_tag( - tag, ref=anchor, message=f"Synonynmous tag to {anchor}; added by 'nf-core download'." + tag, ref=anchor, message=f"Synonynmous tag to {anchor}; added by `nf-core download`." ) self.repo.create_head(tag, anchor) # should heads be created as well? except (GitCommandError, InvalidGitRepositoryError) as e: log.error(f"[red]Additional tag(s) could not be applied:[/]\n{e}\n") else: - log.error( - f"[red]Adding the additional tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{anchor}' must be a valid git reference that resolves to a commit, while '{tag}' must not exist hitherto." - ) + if not self.repo.is_valid_object(anchor): + log.error( + f"[red]Adding tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{anchor}' must be a valid git reference that resolves to a commit." + ) + if self.repo.is_valid_object(tag): + log.error( + f"[red]Adding tag '{tag}' to '{anchor}' failed.[/]\n Mind that '{tag}' must not exist hitherto." + ) else: - log.error( - f"[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: '{additional_tag}'" - ) + log.error(f"[red]Could not apply invalid `--tag` specification[/]: '{additional_tag}'") def bare_clone(self, destination): if self.repo: diff --git a/tests/test_cli.py b/tests/test_cli.py index 826008f033..5df0a32418 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -168,7 +168,7 @@ def test_cli_download(self, mock_dl): "force": None, "platform": None, "download-configuration": None, - "additional-tags": "3.12=testing", + "tag": "3.12=testing", "container-system": "singularity", "container-library": "quay.io", "container-cache-utilisation": "copy", @@ -189,7 +189,7 @@ def test_cli_download(self, mock_dl): "force" in params, "platform" in params, "download-configuration" in params, - (params["additional-tags"],), + (params["tag"],), params["container-system"], (params["container-library"],), params["container-cache-utilisation"], diff --git a/tests/test_download.py b/tests/test_download.py index 589fe6edab..3e0f11d579 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -645,7 +645,23 @@ def test_download_workflow_for_platform(self, tmp_dir, _): ) # indirect definition via $container variable. # - # Test adding custom tags to Seqera Platform download + # Brief test adding a single custom tag to Seqera Platform download + # + @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") + @with_temporary_folder + def test_download_workflow_for_platform_with_one_custom_tag(self, _, tmp_dir): + download_obj = DownloadWorkflow( + pipeline="nf-core/rnaseq", + revision=("3.9"), + compress_type="none", + platform=True, + container_system=None, + additional_tags=("3.9=cool_revision",), + ) + assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 1 + + # + # Test adding custom tags to Seqera Platform download (full test) # @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") @with_temporary_folder @@ -707,21 +723,8 @@ def test_download_workflow_for_platform_with_custom_tags(self, _, tmp_dir): assert all( log in self.logged_messages for log in { - "[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: '3.9=invalid tag'", - "[red]Adding the additional tag 'not_included' to '3.14.0' failed.[/]\n Mind that '3.14.0' must be a valid git reference that resolves to a commit, while 'not_included' must not exist hitherto.", - "[red]Could not apply invalid '-a' / '--additional-tag' specification[/]: 'What is this?'", + "[red]Could not apply invalid `--tag` specification[/]: '3.9=invalid tag'", + "[red]Adding tag 'not_included' to '3.14.0' failed.[/]\n Mind that '3.14.0' must be a valid git reference that resolves to a commit.", + "[red]Could not apply invalid `--tag` specification[/]: 'What is this?'", } ) - - @mock.patch("nf_core.download.DownloadWorkflow.get_singularity_images") - @with_temporary_folder - def test_download_workflow_for_platform_with_one_custom_tag(self, _, tmp_dir): - download_obj = DownloadWorkflow( - pipeline="nf-core/rnaseq", - revision=("3.9"), - compress_type="none", - platform=True, - container_system=None, - additional_tags=("3.9=cool_revision",), - ) - assert isinstance(download_obj.additional_tags, list) and len(download_obj.additional_tags) == 1 From 0b8e4adf5cce24b8b55859ac8e467f2c2172f796 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Mon, 6 May 2024 15:30:07 +0200 Subject: [PATCH 240/270] Change the e-mail from the nf-core bot e-mail to a sinkhole address. --- nf_core/download.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index 5edba513fd..ee786325f3 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1726,7 +1726,11 @@ def tidy_tags_and_branches(self): # "Private" method to add the additional custom tags to the repository. def __add_additional_tags(self) -> None: if self.additional_tags: - self.ensure_git_user_config(f"nf-core download v{nf_core.__version__}", "core@nf-co.re") + # example.com is reserved by the Internet Assigned Numbers Authority (IANA) as special-use domain names for documentation purposes. + # Although "dev-null" is a syntactically-valid local-part that is equally valid for delivery, + # and only the receiving MTA can decide whether to accept it, it is to my best knowledge configured with + # a Postfix discard mail delivery agent (https://www.postfix.org/discard.8.html), so incoming mails should be sinkholed. + self.ensure_git_user_config(f"nf-core download v{nf_core.__version__}", "dev-null@example.com") for additional_tag in self.additional_tags: # A valid git branch or tag name can contain alphanumeric characters, underscores, hyphens, and dots. From 20b83e824cb205536de2a94c22bb0236059e7dc4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 6 May 2024 16:17:31 +0200 Subject: [PATCH 241/270] set correct sha before running component lint tests --- nf_core/modules/lint/__init__.py | 4 ++++ nf_core/subworkflows/lint/__init__.py | 4 ++++ nf_core/synced_repo.py | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index b2816dab6a..94c6eb3b60 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -213,6 +213,10 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals # Otherwise run all the lint tests else: + # Set correct sha + version = self.modules_json.get_module_version(mod.component_name, mod.repo_url, mod.org) + mod.git_sha = version + for test_name in self.lint_tests: if test_name == "main_nf": getattr(self, test_name)(mod, fix_version, self.registry, progress_bar) diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index 96d2e0ab92..df02e3376b 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -207,6 +207,10 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Otherwise run all the lint tests else: + # Set correct sha + version = self.modules_json.get_subworkflow_version(swf.component_name, swf.repo_url, swf.org) + swf.git_sha = version + for test_name in self.lint_tests: getattr(self, test_name)(swf) diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 5c31e96911..28e017840a 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -319,9 +319,9 @@ def component_files_identical(self, component_name, base_path, commit, component component_dir = self.get_component_dir(component_name, component_type) for file in component_files: try: - files_identical[file] = filecmp.cmp(os.path.join(component_dir, file), os.path.join(base_path, file)) + files_identical[file] = filecmp.cmp(Path(component_dir, file), Path(base_path, file)) except FileNotFoundError: - log.debug(f"Could not open file: {os.path.join(component_dir, file)}") + log.debug(f"Could not open file: {Path(component_dir, file)}") continue self.checkout_branch() return files_identical From 83d1f1de79ea95700dbb47661484fed8e568bbba Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 6 May 2024 14:22:14 +0000 Subject: [PATCH 242/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88ce77a3b8..69cda3587a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Only match assignments of params in `main.nf` and not references like `params.aligner == ` ([#2833](https://github.com/nf-core/tools/pull/2833)) - Include test for presence of versions in snapshot ([#2888](https://github.com/nf-core/tools/pull/2888)) +- Components: set correct sha before running component lint tests ([#2952](https://github.com/nf-core/tools/pull/2952)) ### Download From f69fe301b29f54237157bb6c68365d99e7d7c96e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 6 May 2024 16:49:59 +0200 Subject: [PATCH 243/270] only get component version when linting a pipeline repo --- nf_core/modules/lint/__init__.py | 7 ++++--- nf_core/subworkflows/lint/__init__.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 94c6eb3b60..02ea700ea4 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -213,9 +213,10 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals # Otherwise run all the lint tests else: - # Set correct sha - version = self.modules_json.get_module_version(mod.component_name, mod.repo_url, mod.org) - mod.git_sha = version + if self.repo_type == "pipeline": + # Set correct sha + version = self.modules_json.get_module_version(mod.component_name, mod.repo_url, mod.org) + mod.git_sha = version for test_name in self.lint_tests: if test_name == "main_nf": diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index df02e3376b..b7627cb3f7 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -207,9 +207,10 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Otherwise run all the lint tests else: - # Set correct sha - version = self.modules_json.get_subworkflow_version(swf.component_name, swf.repo_url, swf.org) - swf.git_sha = version + if self.repo_type == "pipeline": + # Set correct sha + version = self.modules_json.get_subworkflow_version(swf.component_name, swf.repo_url, swf.org) + swf.git_sha = version for test_name in self.lint_tests: getattr(self, test_name)(swf) From 65f7647503ef875649ae3cc02ad4c4f554758c47 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 6 May 2024 17:19:22 +0200 Subject: [PATCH 244/270] fix pytest by checking if modules_json exists --- nf_core/modules/lint/__init__.py | 2 +- nf_core/subworkflows/lint/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 02ea700ea4..f96683089c 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -213,7 +213,7 @@ def lint_module(self, mod, progress_bar, registry, local=False, fix_version=Fals # Otherwise run all the lint tests else: - if self.repo_type == "pipeline": + if self.repo_type == "pipeline" and self.modules_json: # Set correct sha version = self.modules_json.get_module_version(mod.component_name, mod.repo_url, mod.org) mod.git_sha = version diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index b7627cb3f7..cc79ed8639 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -207,7 +207,7 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Otherwise run all the lint tests else: - if self.repo_type == "pipeline": + if self.repo_type == "pipeline" and self.modules_json: # Set correct sha version = self.modules_json.get_subworkflow_version(swf.component_name, swf.repo_url, swf.org) swf.git_sha = version From 3a60a17a30f0fefc9b918dfaadb8fc22795529ef Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Mon, 6 May 2024 18:46:00 +0200 Subject: [PATCH 245/270] Update .editorconfig I think we don't need to ignore README.md as it's covered by the md on the following line. --- nf_core/pipeline-template/.editorconfig | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/.editorconfig b/nf_core/pipeline-template/.editorconfig index dd9ffa5387..72dda289a0 100644 --- a/nf_core/pipeline-template/.editorconfig +++ b/nf_core/pipeline-template/.editorconfig @@ -28,10 +28,6 @@ indent_style = unset [/assets/email*] indent_size = unset -# ignore Readme -[README.md] -indent_style = unset - -# ignore python +# ignore python and markdown [*.{py,md}] indent_style = unset From f7ccdf4f0eef1066f695f288f2c1868d54b68cc4 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 6 May 2024 16:47:13 +0000 Subject: [PATCH 246/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69cda3587a..6e1fd6f30b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Template config: `conda.channels`, not `channels` ([#2950](https://github.com/nf-core/tools/pull/2950)) - Handles multiple DOIs + doi.org resolver from manifest.doi ([#2946](https://github.com/nf-core/tools/pull/2946)) - Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) +- Template - Update .editorconfig ([#2953](https://github.com/nf-core/tools/pull/2953)) ### Linting From 3277b7cdcfcaf1c4d21ec70988c6029d56560125 Mon Sep 17 00:00:00 2001 From: Matthias Zepper <6963520+MatthiasZepper@users.noreply.github.com> Date: Mon, 6 May 2024 17:34:53 +0200 Subject: [PATCH 247/270] Rearranging the sentences in README. Thanks @mashehu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fa909804c4..5222854084 100644 --- a/README.md +++ b/README.md @@ -437,7 +437,8 @@ Subsequently, the `*.git` folder can be moved to it's final destination and link > [!TIP] > Also without access to Seqera Platform, pipelines downloaded with the `--platform` flag can be run if the _absolute_ path is specified: `nextflow run -r 2.5 file:/path/to/pipelinedownload.git`. Downloads in this format allow you to include multiple revisions of a pipeline in a single file, but require that the revision (e.g. `-r 2.5`) is always explicitly specified. -Facilities and those who are setting up pipelines for others to use may find the `--tag` argument helpful. It must be followed by a string in a `key=value` format and can be provided multiple times. The `key` must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. The `--tag` argument allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `--tag "3.12.0=testing" --tag "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production. +Facilities and those who are setting up pipelines for others to use may find the `--tag` argument helpful. It allows customizing the downloaded pipeline with additional tags that can be used to select particular revisions in the Seqera Platform interface. For example, an accredited facility may opt to tag particular revisions according to their structured release management process: `--tag "3.12.0=testing" --tag "3.9.0=validated"` so their staff can easily ensure that the correct version of the pipeline is run in production. +The `--tag` argument must be followed by a string in a `key=value` format and can be provided multiple times. The `key` must refer to a valid branch, tag or commit SHA. The right-hand side must comply with the naming conventions for Git tags and may not yet exist in the repository. ## Pipeline software licences From 1e632c2d82239f9053af64b3644d7fdb829576bb Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 7 May 2024 07:26:37 +0200 Subject: [PATCH 248/270] Add deprecation warning for -t / --tower flag. --- nf_core/__main__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ead74aa67c..39a1afbf63 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -450,6 +450,9 @@ def download( """ from nf_core.download import DownloadWorkflow + if tower: + log.warning("[red]The `-t` / `--tower` flag is deprecated. Please use `--platform` instead.[/]") + dl = DownloadWorkflow( pipeline, revision, From c95880f25ec5e62a822daa8a26479b90f0c7056e Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 7 May 2024 07:44:47 +0200 Subject: [PATCH 249/270] Omit creating heads for additional tags due to UI bug in Platform. --- nf_core/download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index ee786325f3..f5ab3a0f58 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -1742,7 +1742,6 @@ def __add_additional_tags(self) -> None: self.repo.create_tag( tag, ref=anchor, message=f"Synonynmous tag to {anchor}; added by `nf-core download`." ) - self.repo.create_head(tag, anchor) # should heads be created as well? except (GitCommandError, InvalidGitRepositoryError) as e: log.error(f"[red]Additional tag(s) could not be applied:[/]\n{e}\n") else: From 9f84850a0fc5f1a2e34f6f74ccdf4b65892a4742 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 May 2024 10:11:42 +0200 Subject: [PATCH 250/270] fix line break with jinja --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 936e5e80d7..ea4ca47de5 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -18,7 +18,7 @@ params { genome = null igenomes_base = 's3://ngi-igenomes/igenomes/' igenomes_ignore = false - {%- endif -%} + {%- endif %} // MultiQC options multiqc_config = null From 61a175cac02f51a6a25de0fc9393398dfc090aef Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 May 2024 10:24:18 +0200 Subject: [PATCH 251/270] remove duplicates from CHANGELOG and restructure to the correct sections --- CHANGELOG.md | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1fd6f30b..591d0231ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Template - Remove fasta default from nextflow.config ([#2828](https://github.com/nf-core/tools/pull/2828)) -- Update templates to use nf-core/setup-nextflow v2 +- Update templates to use nf-core/setup-nextflow v2 ([#2818](https://github.com/nf-core/tools/pull/2818)) - Link to troubleshooting docs when pipeline fails ([#2845](https://github.com/nf-core/tools/pull/2845)) - Add fallback to `download_pipeline.yml` in case the pipeline does not support stub runs ([#2846](https://github.com/nf-core/tools/pull/2846)) - Set topic variable correctly in the mastodon announcement ([#2848](https://github.com/nf-core/tools/pull/2848)) @@ -17,7 +17,7 @@ - Template config: `conda.channels`, not `channels` ([#2950](https://github.com/nf-core/tools/pull/2950)) - Handles multiple DOIs + doi.org resolver from manifest.doi ([#2946](https://github.com/nf-core/tools/pull/2946)) - Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) -- Template - Update .editorconfig ([#2953](https://github.com/nf-core/tools/pull/2953)) +- Update .editorconfig ([#2953](https://github.com/nf-core/tools/pull/2953)) ### Linting @@ -33,42 +33,27 @@ ### Components - Handle more complete list of possible git URL forms (ssh:// and ftp:// prefixes specifically) ([#2945](https://github.com/nf-core/tools/pull/2945)) +- Fix path in component update script ([#2823](https://github.com/nf-core/tools/pull/2823)) ### General -- Update CI to use nf-core/setup-nextflow v2 +- Update CI to use nf-core/setup-nextflow v2 ([#2819](https://github.com/nf-core/tools/pull/2819)) - Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) -- Fix path in component update script ([#2823](https://github.com/nf-core/tools/pull/2823)) - Update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) -- Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) +- Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)), ([#2902](https://github.com/nf-core/tools/pull/2902)), ([#2927](https://github.com/nf-core/tools/pull/2927)), ([#2939](https://github.com/nf-core/tools/pull/2939)) - Switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.2 ([#2836](https://github.com/nf-core/tools/pull/2836)) - Add tests for assignment and referencing of params in main.nf ([#2841](https://github.com/nf-core/tools/pull/2841)) - Optimize layers in dockerfile ([#2842](https://github.com/nf-core/tools/pull/2842)) -- Update gitpod/workspace-base Docker digest to 1e133e5 ([#2843](https://github.com/nf-core/tools/pull/2843)) - Update python:3.11-slim Docker digest to a2eb07f ([#2847](https://github.com/nf-core/tools/pull/2847)) - Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 ([#2850](https://github.com/nf-core/tools/pull/2850)) - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) - fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) - Fix incorrect assertions for called_with on mocks ([#2891](https://github.com/nf-core/tools/pull/2891)) - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) -- Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) -- Update GitHub Actions ([#2902](https://github.com/nf-core/tools/pull/2902)) - Get immediate parent path name for schema creation ([#2886](https://github.com/nf-core/tools/pull/2886)) -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 ([#2903](https://github.com/nf-core/tools/pull/2903)) - Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897)) -- Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) -- Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) -- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) -- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) -- Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) -- Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) -- Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) - Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) -- Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) - Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) - fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) From 40089c2f4f87ce6f507c479ec43adc56856199a7 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 7 May 2024 08:27:32 +0000 Subject: [PATCH 252/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 591d0231ab..dd2e58436b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,7 @@ - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) - fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 ([#2951](https://github.com/nf-core/tools/pull/2951)) +- Restructure CHANGELOG.md ([#2954](https://github.com/nf-core/tools/pull/2954)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From abb20d304e3addf8c6bf0783c65bbdd4d32eee36 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 May 2024 10:32:20 +0200 Subject: [PATCH 253/270] Remove premature mention of nf-test on PR template --- CHANGELOG.md | 1 + nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1fd6f30b..3dee6c5d31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Handles multiple DOIs + doi.org resolver from manifest.doi ([#2946](https://github.com/nf-core/tools/pull/2946)) - Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) - Template - Update .editorconfig ([#2953](https://github.com/nf-core/tools/pull/2953)) +- Remove premature mention of nf-test on PR template ([#2955](https://github.com/nf-core/tools/pull/2955)) ### Linting diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index ca0687733e..4f01a97993 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -20,7 +20,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. {%- endif %} - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nf-test test tests/ --verbose --profile +docker`). +- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. From 050d240d097ed6b360d632ae7438968d636d196e Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 7 May 2024 08:33:35 +0000 Subject: [PATCH 254/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dee6c5d31..a4958a1a4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,6 @@ - Handles multiple DOIs + doi.org resolver from manifest.doi ([#2946](https://github.com/nf-core/tools/pull/2946)) - Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) - Template - Update .editorconfig ([#2953](https://github.com/nf-core/tools/pull/2953)) -- Remove premature mention of nf-test on PR template ([#2955](https://github.com/nf-core/tools/pull/2955)) ### Linting @@ -74,6 +73,7 @@ - Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) - fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 ([#2951](https://github.com/nf-core/tools/pull/2951)) +- Remove-premature-mention-nftest ([#2955](https://github.com/nf-core/tools/pull/2955)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 4c66dbc5a1511a2c875c937559a91a4d64e393bc Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 7 May 2024 14:36:55 +0200 Subject: [PATCH 255/270] be more libaral with logo comparisons --- nf_core/lint/files_unchanged.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 1cd1f7fdb1..014c2b5f09 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -160,10 +160,8 @@ def _tf(file_path: Union[str, Path]) -> Path: if filecmp.cmp(_pf(f), _tf(f), shallow=True): passed.append(f"`{f}` matches the template") else: - if ( - f.name.endswith(".png") - and os.stat(_pf(f)).st_mode == os.stat(_tf(f)).st_mode - and int(os.stat(_pf(f)).st_size / 100) == int(os.stat(_tf(f)).st_size / 100) + if f.name.endswith(".png") and int(os.stat(_pf(f)).st_size / 500) == int( + os.stat(_tf(f)).st_size / 500 ): # almost the same file, good enough for the logo log.debug(f"Files are almost the same. Will pass: {f}") From c0b23bb7fba4962df22b227697d353f2e2d30eca Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 7 May 2024 12:40:21 +0000 Subject: [PATCH 256/270] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb416ddc8c..55702b0f21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 ([#2951](https://github.com/nf-core/tools/pull/2951)) - Restructure CHANGELOG.md ([#2954](https://github.com/nf-core/tools/pull/2954)) - Remove-premature-mention-nftest ([#2955](https://github.com/nf-core/tools/pull/2955)) +- Less strict logo comparison ([#2956](https://github.com/nf-core/tools/pull/2956)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From c392ab51d9db8ea159434a7511d77025961be94a Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 7 May 2024 14:41:58 +0200 Subject: [PATCH 257/270] fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55702b0f21..70c778664d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Only match assignments of params in `main.nf` and not references like `params.aligner == ` ([#2833](https://github.com/nf-core/tools/pull/2833)) - Include test for presence of versions in snapshot ([#2888](https://github.com/nf-core/tools/pull/2888)) - Components: set correct sha before running component lint tests ([#2952](https://github.com/nf-core/tools/pull/2952)) +- Less strict logo comparison ([#2956](https://github.com/nf-core/tools/pull/2956)) ### Download @@ -61,7 +62,6 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 ([#2951](https://github.com/nf-core/tools/pull/2951)) - Restructure CHANGELOG.md ([#2954](https://github.com/nf-core/tools/pull/2954)) - Remove-premature-mention-nftest ([#2955](https://github.com/nf-core/tools/pull/2955)) -- Less strict logo comparison ([#2956](https://github.com/nf-core/tools/pull/2956)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 5c1f9eb49904e0ccb8cfce3ab9ad5b0983b44fc6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 7 May 2024 14:53:01 +0200 Subject: [PATCH 258/270] correct changelog bot --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70c778664d..7e8823c390 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,6 @@ - fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 ([#2951](https://github.com/nf-core/tools/pull/2951)) - Restructure CHANGELOG.md ([#2954](https://github.com/nf-core/tools/pull/2954)) -- Remove-premature-mention-nftest ([#2955](https://github.com/nf-core/tools/pull/2955)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] From 7067afe3fed0f667f094a23a69cf6e60619ee76e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 7 May 2024 14:53:53 +0200 Subject: [PATCH 259/270] fix another changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e8823c390..4d4710976d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,7 +50,7 @@ - Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible - Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) - Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) -- fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) +- Ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) - Fix incorrect assertions for called_with on mocks ([#2891](https://github.com/nf-core/tools/pull/2891)) - Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) - Get immediate parent path name for schema creation ([#2886](https://github.com/nf-core/tools/pull/2886)) From f08f0e49296105b4b4a201e37723eb691a57d45a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 7 May 2024 15:16:02 +0200 Subject: [PATCH 260/270] Apply suggestions from code review Co-authored-by: Maxime U Garcia --- nf_core/pipeline-template/conf/test.config | 2 +- nf_core/pipeline-template/conf/test_full.config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index cd12404c8e..827e21b7b7 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -22,7 +22,7 @@ params { // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' + input = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv' {% if igenomes -%} // Genome references diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index dac47b9867..aa3b70c1e5 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -17,13 +17,13 @@ params { // Input data for full size test // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA) // TODO nf-core: Give any required params for the test so that command line flags are not needed - input = pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' + input = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv' {% if igenomes -%} // Genome references genome = 'R64-1-1' {%- else -%} // Fasta references - fasta = pipelines_testdata_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + fasta = params.pipelines_testdata_base_path + 'viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' {%- endif %} } From d0de101cb5ee2be0662a22dddfc5dd7776e15f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 7 May 2024 15:25:33 +0200 Subject: [PATCH 261/270] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 786b3535fc..9a41f95e37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,8 +42,6 @@ - Update CI to use nf-core/setup-nextflow v2 ([#2819](https://github.com/nf-core/tools/pull/2819)) - Changelog bot: handle also patch version before dev suffix ([#2820](https://github.com/nf-core/tools/pull/2820)) - Add `force_pr` flag to sync, to force a PR even though there are no changes committed ([#2822](https://github.com/nf-core/tools/pull/2822)) -- update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) -- Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)) - Update prettier to 3.2.5 ([#2830](https://github.com/nf-core/tools/pull/2830)) - Update GitHub Actions ([#2827](https://github.com/nf-core/tools/pull/2827)), ([#2902](https://github.com/nf-core/tools/pull/2902)), ([#2927](https://github.com/nf-core/tools/pull/2927)), ([#2939](https://github.com/nf-core/tools/pull/2939)) - Switch to setup-nf-test ([#2834](https://github.com/nf-core/tools/pull/2834)) From d4c8f3620493b5384e0f199cb6fc12e612cd864e Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 7 May 2024 15:28:30 +0200 Subject: [PATCH 262/270] add default value to schema --- nf_core/pipeline-template/nextflow_schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 01bff71f7d..ae7c0b715f 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -272,6 +272,7 @@ "type": "string", "fa_icon": "far fa-check-circle", "description": "Base URL or local path to location of pipeline test dataset files", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/", "hidden": true } } From a6702602c6f54537698d83db3230f30ec03a974f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 7 May 2024 16:00:24 +0200 Subject: [PATCH 263/270] bump version to 2.14.0 --- .gitpod.yml | 2 +- CHANGELOG.md | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index b2fbb73133..445cb35706 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:dev +image: nfcore/gitpod:latest tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index a14a0ea544..6ba5d53f3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # nf-core/tools: Changelog -## v2.14.0dev +## [v2.14.0 - Tantalum Toad](https://github.com/nf-core/tools/releases/tag/2.14) - [2024-05-07] ### Template diff --git a/setup.py b/setup.py index fc7b69ac1e..1ebf4a6b38 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.13.2dev" +version = "2.14.0" with open("README.md") as f: readme = f.read() From 24a2d482b797e2b84b16ae4fe6f9d4a86320d4a8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 7 May 2024 14:38:40 +0000 Subject: [PATCH 264/270] Generate new screengrabs with rich-codex --- docs/images/nf-core-bump-version.svg | 148 ++++----- docs/images/nf-core-create-logo.svg | 68 ++-- docs/images/nf-core-create.svg | 126 +++---- docs/images/nf-core-download.svg | 100 +++--- docs/images/nf-core-launch-rnaseq.svg | 82 ++--- docs/images/nf-core-licences.svg | 135 +++++--- docs/images/nf-core-lint.svg | 174 +++++----- docs/images/nf-core-list-rna.svg | 144 ++++---- docs/images/nf-core-list-stars.svg | 104 +++--- docs/images/nf-core-list.svg | 108 +++--- docs/images/nf-core-modules-bump-version.svg | 106 +++--- docs/images/nf-core-modules-create.svg | 130 +++++--- docs/images/nf-core-modules-info.svg | 206 ++++++------ docs/images/nf-core-modules-install.svg | 86 ++--- docs/images/nf-core-modules-lint.svg | 115 ++++--- docs/images/nf-core-modules-list-local.svg | 122 ++++--- docs/images/nf-core-modules-list-remote.svg | 132 ++++---- docs/images/nf-core-modules-patch.svg | 156 ++++----- docs/images/nf-core-modules-remove.svg | 70 ++-- docs/images/nf-core-modules-test.svg | 70 ++-- docs/images/nf-core-modules-update.svg | 82 ++--- docs/images/nf-core-schema-build.svg | 84 ++--- docs/images/nf-core-schema-lint.svg | 78 ++--- docs/images/nf-core-schema-validate.svg | 82 ++--- docs/images/nf-core-subworkflows-create.svg | 110 +++---- docs/images/nf-core-subworkflows-info.svg | 138 ++++---- docs/images/nf-core-subworkflows-install.svg | 92 +++--- docs/images/nf-core-subworkflows-lint.svg | 310 +++--------------- .../nf-core-subworkflows-list-local.svg | 130 ++++---- .../nf-core-subworkflows-list-remote.svg | 132 ++++---- docs/images/nf-core-subworkflows-remove.svg | 127 +++---- docs/images/nf-core-subworkflows-test.svg | 73 +++-- docs/images/nf-core-subworkflows-update.svg | 93 +++--- docs/images/nf-core-sync.svg | 114 +++---- 34 files changed, 1894 insertions(+), 2133 deletions(-) diff --git a/docs/images/nf-core-bump-version.svg b/docs/images/nf-core-bump-version.svg index 70171475d9..8a7ad902f4 100644 --- a/docs/images/nf-core-bump-version.svg +++ b/docs/images/nf-core-bump-version.svg @@ -19,122 +19,122 @@ font-weight: 700; } - .terminal-3430704182-matrix { + .terminal-3430835254-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3430704182-title { + .terminal-3430835254-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3430704182-r1 { fill: #c5c8c6 } -.terminal-3430704182-r2 { fill: #98a84b } -.terminal-3430704182-r3 { fill: #9a9b99 } -.terminal-3430704182-r4 { fill: #608ab1 } -.terminal-3430704182-r5 { fill: #d0b344 } -.terminal-3430704182-r6 { fill: #cc555a } + .terminal-3430835254-r1 { fill: #c5c8c6 } +.terminal-3430835254-r2 { fill: #98a84b } +.terminal-3430835254-r3 { fill: #9a9b99 } +.terminal-3430835254-r4 { fill: #608ab1 } +.terminal-3430835254-r5 { fill: #d0b344 } +.terminal-3430835254-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -146,41 +146,41 @@ - + - - $ nf-core bump-version 1.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     Changing version number from '1.0dev' to '1.1' -INFO     Updated version in 'nextflow.config' - - version         = '1.0dev' - + version         = '1.1' - - -INFO     Updated version in 'assets/multiqc_config.yml' - - This report has been generated by the <a  -href="https://github.com/nf-core/nextbigthing/tree/dev" target="_blank">nf-core/nextbigthing</a> - + This report has been generated by the <a  -href="https://github.com/nf-core/nextbigthing/releases/tag/1.1"  -target="_blank">nf-core/nextbigthing</a> - - -INFO     Updated version in 'assets/multiqc_config.yml' - - <a href="https://nf-co.re/nextbigthing/dev/docs/output"  -target="_blank">documentation</a>. - + <a href="https://nf-co.re/nextbigthing/1.1/docs/output"  -target="_blank">documentation</a>. - - + + $ nf-core bump-version 1.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     Changing version number from '1.0dev' to '1.1' +INFO     Updated version in 'nextflow.config' + - version         = '1.0dev' + + version         = '1.1' + + +INFO     Updated version in 'assets/multiqc_config.yml' + - This report has been generated by the <a  +href="https://github.com/nf-core/nextbigthing/tree/dev" target="_blank">nf-core/nextbigthing</a> + + This report has been generated by the <a  +href="https://github.com/nf-core/nextbigthing/releases/tag/1.1"  +target="_blank">nf-core/nextbigthing</a> + + +INFO     Updated version in 'assets/multiqc_config.yml' + - <a href="https://nf-co.re/nextbigthing/dev/docs/output"  +target="_blank">documentation</a>. + + <a href="https://nf-co.re/nextbigthing/1.1/docs/output"  +target="_blank">documentation</a>. + + diff --git a/docs/images/nf-core-create-logo.svg b/docs/images/nf-core-create-logo.svg index c6e7269363..50340ff44f 100644 --- a/docs/images/nf-core-create-logo.svg +++ b/docs/images/nf-core-create-logo.svg @@ -19,62 +19,62 @@ font-weight: 700; } - .terminal-2584676837-matrix { + .terminal-2584807909-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2584676837-title { + .terminal-2584807909-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2584676837-r1 { fill: #c5c8c6 } -.terminal-2584676837-r2 { fill: #98a84b } -.terminal-2584676837-r3 { fill: #9a9b99 } -.terminal-2584676837-r4 { fill: #608ab1 } -.terminal-2584676837-r5 { fill: #d0b344 } -.terminal-2584676837-r6 { fill: #98729f } + .terminal-2584807909-r1 { fill: #c5c8c6 } +.terminal-2584807909-r2 { fill: #98a84b } +.terminal-2584807909-r3 { fill: #9a9b99 } +.terminal-2584807909-r4 { fill: #608ab1 } +.terminal-2584807909-r5 { fill: #d0b344 } +.terminal-2584807909-r6 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + @@ -86,21 +86,21 @@ - + - - $ nf-core create-logo nextbigthing - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     Created logo: nf-core-nextbigthing_logo_light.png + + $ nf-core create-logo nextbigthing + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     Created logo: nf-core-nextbigthing_logo_light.png diff --git a/docs/images/nf-core-create.svg b/docs/images/nf-core-create.svg index 422e6734ce..70b281cd1c 100644 --- a/docs/images/nf-core-create.svg +++ b/docs/images/nf-core-create.svg @@ -19,104 +19,104 @@ font-weight: 700; } - .terminal-165925560-matrix { + .terminal-166056632-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-165925560-title { + .terminal-166056632-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-165925560-r1 { fill: #c5c8c6 } -.terminal-165925560-r2 { fill: #98a84b } -.terminal-165925560-r3 { fill: #9a9b99 } -.terminal-165925560-r4 { fill: #608ab1 } -.terminal-165925560-r5 { fill: #d0b344 } -.terminal-165925560-r6 { fill: #98729f } -.terminal-165925560-r7 { fill: #ff2c7a } -.terminal-165925560-r8 { fill: #98a84b;font-weight: bold } -.terminal-165925560-r9 { fill: #1984e9;text-decoration: underline; } + .terminal-166056632-r1 { fill: #c5c8c6 } +.terminal-166056632-r2 { fill: #98a84b } +.terminal-166056632-r3 { fill: #9a9b99 } +.terminal-166056632-r4 { fill: #608ab1 } +.terminal-166056632-r5 { fill: #d0b344 } +.terminal-166056632-r6 { fill: #98729f } +.terminal-166056632-r7 { fill: #ff2c7a } +.terminal-166056632-r8 { fill: #98a84b;font-weight: bold } +.terminal-166056632-r9 { fill: #1984e9;text-decoration: underline; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -128,34 +128,34 @@ - + - - $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  --a "Big Steve" --plain - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' -INFO     Initialising pipeline git repository                                                        -INFO     Done. Remember to add a remote and push to GitHub:                                          - cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing - git remote add origin git@github.com:USERNAME/REPO_NAME.git  - git push --all origin                                        -INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      -INFO    !!!!!! IMPORTANT !!!!!! - -If you are interested in adding your pipeline to the nf-core community, -PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! - -Please read: https://nf-co.re/developers/adding_pipelines#join-the-community + + $ nf-core create -n nextbigthing -d "This pipeline analyses data from the next big omics technique"  +-a "Big Steve" --plain + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     Creating new nf-core pipeline: 'nf-core/nextbigthing' +INFO     Initialising pipeline git repository                                                        +INFO     Done. Remember to add a remote and push to GitHub:                                          + cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing + git remote add origin git@github.com:USERNAME/REPO_NAME.git  + git push --all origin                                        +INFO     This will also push your newly created dev branch and the TEMPLATE branch for syncing.      +INFO    !!!!!! IMPORTANT !!!!!! + +If you are interested in adding your pipeline to the nf-core community, +PLEASE COME AND TALK TO US IN THE NF-CORE SLACK BEFORE WRITING ANY CODE! + +Please read: https://nf-co.re/developers/adding_pipelines#join-the-community diff --git a/docs/images/nf-core-download.svg b/docs/images/nf-core-download.svg index 5594930fa7..e7511c8f54 100644 --- a/docs/images/nf-core-download.svg +++ b/docs/images/nf-core-download.svg @@ -19,86 +19,86 @@ font-weight: 700; } - .terminal-2899766736-matrix { + .terminal-2899897808-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2899766736-title { + .terminal-2899897808-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2899766736-r1 { fill: #c5c8c6 } -.terminal-2899766736-r2 { fill: #98a84b } -.terminal-2899766736-r3 { fill: #9a9b99 } -.terminal-2899766736-r4 { fill: #608ab1 } -.terminal-2899766736-r5 { fill: #d0b344 } -.terminal-2899766736-r6 { fill: #cc555a } + .terminal-2899897808-r1 { fill: #c5c8c6 } +.terminal-2899897808-r2 { fill: #98a84b } +.terminal-2899897808-r3 { fill: #9a9b99 } +.terminal-2899897808-r4 { fill: #608ab1 } +.terminal-2899897808-r5 { fill: #d0b344 } +.terminal-2899897808-r6 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -110,29 +110,29 @@ - + - - $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -s none -d - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -WARNING  Could not find GitHub authentication token. Some API requests may fail.                     -INFO     Saving 'nf-core/rnaseq' -          Pipeline revision: '3.8' -          Use containers: 'none' -          Container library: 'quay.io' -          Output directory: 'nf-core-rnaseq' -          Include default institutional configuration: 'True' -INFO     Downloading centralised configs from GitHub                                                 -INFO     Downloading workflow files from GitHub                                                      + + $ nf-core download rnaseq -r 3.8 --outdir nf-core-rnaseq -x none -s none -d + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +WARNING  Could not find GitHub authentication token. Some API requests may fail.                     +INFO     Saving 'nf-core/rnaseq' +          Pipeline revision: '3.8' +          Use containers: 'none' +          Container library: 'quay.io' +          Output directory: 'nf-core-rnaseq' +          Include default institutional configuration: 'True' +INFO     Downloading centralised configs from GitHub                                                 +INFO     Downloading workflow files from GitHub                                                      diff --git a/docs/images/nf-core-launch-rnaseq.svg b/docs/images/nf-core-launch-rnaseq.svg index f2608fe76f..ef0cfc2ba8 100644 --- a/docs/images/nf-core-launch-rnaseq.svg +++ b/docs/images/nf-core-launch-rnaseq.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-3860049459-matrix { + .terminal-3860180531-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3860049459-title { + .terminal-3860180531-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3860049459-r1 { fill: #c5c8c6 } -.terminal-3860049459-r2 { fill: #98a84b } -.terminal-3860049459-r3 { fill: #9a9b99 } -.terminal-3860049459-r4 { fill: #608ab1 } -.terminal-3860049459-r5 { fill: #d0b344 } -.terminal-3860049459-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3860049459-r7 { fill: #68a0b3;font-weight: bold } + .terminal-3860180531-r1 { fill: #c5c8c6 } +.terminal-3860180531-r2 { fill: #98a84b } +.terminal-3860180531-r3 { fill: #9a9b99 } +.terminal-3860180531-r4 { fill: #608ab1 } +.terminal-3860180531-r5 { fill: #d0b344 } +.terminal-3860180531-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3860180531-r7 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,24 +96,24 @@ - + - - $ nf-core launch rnaseq -r 3.8.1 - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      -         files or profiles                                                                           - -INFO     Downloading workflow: nf-core/rnaseq (3.8.1) + + $ nf-core launch rnaseq -r 3.8.1 + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config      +         files or profiles                                                                           + +INFO     Downloading workflow: nf-core/rnaseq (3.8.1) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index 8cc00c351b..4cbed577c1 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    +INFO     Warning: This tool only prints licence information for the software tools packaged using    +         conda.                                                                                      +INFO     The pipeline may use other software and dependencies not described here.                    +┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ +┃Package Name┃Version┃Licence┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ +│ lbzip2       │ 2.5     │ GPL v3  │ +│ deepvariant  │ 0.7.0   │ MIT     │ +│ htslib       │ 1.9     │ MIT     │ +│ picard       │ 2.18.7  │ MIT     │ +│ pip          │ 10.0.1  │ MIT     │ +│ samtools     │ 1.9     │ MIT     │ +│ python       │ 2.7.15  │ PSF     │ +│ bzip2        │ 1.0.6   │ bzip2   │ +└──────────────┴─────────┴─────────┘ diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index 2e55a7e11e..d5a6fb8488 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -19,139 +19,139 @@ font-weight: 700; } - .terminal-2900416932-matrix { + .terminal-2655508897-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2900416932-title { + .terminal-2655508897-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2900416932-r1 { fill: #c5c8c6 } -.terminal-2900416932-r2 { fill: #98a84b } -.terminal-2900416932-r3 { fill: #9a9b99 } -.terminal-2900416932-r4 { fill: #608ab1 } -.terminal-2900416932-r5 { fill: #d0b344 } -.terminal-2900416932-r6 { fill: #98729f } -.terminal-2900416932-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2900416932-r8 { fill: #608ab1;text-decoration: underline; } -.terminal-2900416932-r9 { fill: #d0b344;font-weight: bold } -.terminal-2900416932-r10 { fill: #68a0b3;font-weight: bold } -.terminal-2900416932-r11 { fill: #cc555a } + .terminal-2655508897-r1 { fill: #c5c8c6 } +.terminal-2655508897-r2 { fill: #98a84b } +.terminal-2655508897-r3 { fill: #9a9b99 } +.terminal-2655508897-r4 { fill: #608ab1 } +.terminal-2655508897-r5 { fill: #d0b344 } +.terminal-2655508897-r6 { fill: #98729f } +.terminal-2655508897-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2655508897-r8 { fill: #608ab1;text-decoration: underline; } +.terminal-2655508897-r9 { fill: #d0b344;font-weight: bold } +.terminal-2655508897-r10 { fill: #68a0b3;font-weight: bold } +.terminal-2655508897-r11 { fill: #cc555a } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -163,45 +163,45 @@ - + - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Testing pipeline: . - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos                                                                  │ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ -││ -│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ -│doi (after the first release).                                                                  │ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - - -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔] 188 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Testing pipeline: . + + +╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ +││ +│pipeline_todos: pipeline_todos                                                                  │ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ +││ +│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ +│doi (after the first release).                                                                  │ +││ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +╭──────────────────────╮ +│LINT RESULTS SUMMARY│ +├──────────────────────┤ +│[✔] 194 Tests Passed│ +│[?]   1 Test Ignored│ +│[!]   1 Test Warning│ +│[✗]   0 Tests Failed│ +╰──────────────────────╯ diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index 643545c6fc..d9df6fe641 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name       ┃Stars┃Latest Release┃    Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq               │   742 │         3.14.0 │ 2 months ago │           - │ -                   │ -│ differentialabundan… │    38 │          1.4.0 │ 3 months ago │           - │ -                   │ -│ smrnaseq             │    64 │          2.3.0 │   6 days ago │           - │ -                   │ -│ rnasplice            │    25 │          1.0.3 │   6 days ago │           - │ -                   │ -│ circrna              │    34 │            dev │  1 weeks ago │           - │ -                   │ -│ scrnaseq             │   125 │          2.5.1 │ 1 months ago │           - │ -                   │ -│ rnafusion            │   126 │          3.0.1 │ 3 months ago │           - │ -                   │ -│ spatialtranscriptom… │    36 │            dev │  4 weeks ago │           - │ -                   │ -│ dualrnaseq           │    16 │          1.0.0 │  3 years ago │           - │ -                   │ -│ marsseq              │     5 │          1.0.3 │ 8 months ago │           - │ -                   │ -│ lncpipe              │    28 │            dev │  1 years ago │           - │ -                   │ -│ scflow               │    24 │            dev │  3 years ago │           - │ -                   │ -└──────────────────────┴───────┴────────────────┴──────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ +│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ +│ spatialvi           │    39 │            dev │     yesterday │           - │ -                   │ +│ scnanoseq           │     5 │            dev │    6 days ago │           - │ -                   │ +│ scrnaseq            │   169 │          2.5.1 │  4 months ago │           - │ -                   │ +│ differentialabunda… │    45 │          1.4.0 │  5 months ago │           - │ -                   │ +│ circrna             │    40 │            dev │   2 weeks ago │           - │ -                   │ +│ rnafusion           │   129 │          3.0.2 │   4 weeks ago │           - │ -                   │ +│ rnasplice           │    33 │          1.0.3 │  2 months ago │           - │ -                   │ +│ dualrnaseq          │    16 │          1.0.0 │   3 years ago │           - │ -                   │ +│ marsseq             │     5 │          1.0.3 │ 10 months ago │           - │ -                   │ +│ lncpipe             │    30 │            dev │   2 years ago │           - │ -                   │ +│ scflow              │    24 │            dev │   3 years ago │           - │ -                   │ +└─────────────────────┴───────┴────────────────┴───────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 8ea120599f..f033a14092 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-3700160896-matrix { + .terminal-3593730463-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3700160896-title { + .terminal-3593730463-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3700160896-r1 { fill: #c5c8c6 } -.terminal-3700160896-r2 { fill: #98a84b } -.terminal-3700160896-r3 { fill: #9a9b99 } -.terminal-3700160896-r4 { fill: #608ab1 } -.terminal-3700160896-r5 { fill: #d0b344 } -.terminal-3700160896-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3700160896-r7 { fill: #868887 } -.terminal-3700160896-r8 { fill: #868887;font-style: italic; } + .terminal-3593730463-r1 { fill: #c5c8c6 } +.terminal-3593730463-r2 { fill: #98a84b } +.terminal-3593730463-r3 { fill: #9a9b99 } +.terminal-3593730463-r4 { fill: #608ab1 } +.terminal-3593730463-r5 { fill: #d0b344 } +.terminal-3593730463-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3593730463-r7 { fill: #868887 } +.terminal-3593730463-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq              │   742 │         3.14.0 │  2 months ago │           - │ -                   │ -│ sarek               │   320 │          3.4.0 │  4 months ago │           - │ -                   │ -│ mag                 │   175 │          2.5.4 │   2 weeks ago │           - │ -                   │ -│ chipseq             │   161 │          2.0.0 │   1 years ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ +│ sarek               │   340 │          3.4.2 │   3 hours ago │           - │ -                   │ +│ mag                 │   182 │          2.5.4 │  3 months ago │           - │ -                   │ +│ chipseq             │   172 │          2.0.0 │   2 years ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 5e4939f746..79e73ed0e7 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-3505815442-matrix { + .terminal-1180468483-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3505815442-title { + .terminal-1180468483-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3505815442-r1 { fill: #c5c8c6 } -.terminal-3505815442-r2 { fill: #98a84b } -.terminal-3505815442-r3 { fill: #9a9b99 } -.terminal-3505815442-r4 { fill: #608ab1 } -.terminal-3505815442-r5 { fill: #d0b344 } -.terminal-3505815442-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3505815442-r7 { fill: #868887 } -.terminal-3505815442-r8 { fill: #868887;font-style: italic; } + .terminal-1180468483-r1 { fill: #c5c8c6 } +.terminal-1180468483-r2 { fill: #98a84b } +.terminal-1180468483-r3 { fill: #9a9b99 } +.terminal-1180468483-r4 { fill: #608ab1 } +.terminal-1180468483-r5 { fill: #d0b344 } +.terminal-1180468483-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1180468483-r7 { fill: #868887 } +.terminal-1180468483-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ raredisease         │    65 │          1.1.1 │  7 months ago │           - │ -                   │ -│ fetchngs            │   101 │         1.11.0 │  4 months ago │           - │ -                   │ -│ sarek               │   320 │          3.4.0 │  4 months ago │           - │ -                   │ -│ methylseq           │   126 │          2.6.0 │  2 months ago │           - │ -                   │ -│ rnaseq              │   742 │         3.14.0 │  2 months ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ +│ sarek               │   340 │          3.4.2 │   3 hours ago │           - │ -                   │ +│ variantbenchmarking │     4 │            dev │   an hour ago │           - │ -                   │ +│ meerpipe            │     0 │            dev │   8 hours ago │           - │ -                   │ +│ mcmicro             │     4 │            dev │  20 hours ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-bump-version.svg b/docs/images/nf-core-modules-bump-version.svg index 093a0cebc6..4c342507a3 100644 --- a/docs/images/nf-core-modules-bump-version.svg +++ b/docs/images/nf-core-modules-bump-version.svg @@ -19,90 +19,90 @@ font-weight: 700; } - .terminal-1669068868-matrix { + .terminal-1669199940-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1669068868-title { + .terminal-1669199940-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1669068868-r1 { fill: #c5c8c6 } -.terminal-1669068868-r2 { fill: #98a84b } -.terminal-1669068868-r3 { fill: #9a9b99 } -.terminal-1669068868-r4 { fill: #608ab1 } -.terminal-1669068868-r5 { fill: #d0b344 } -.terminal-1669068868-r6 { fill: #98a84b;font-weight: bold } -.terminal-1669068868-r7 { fill: #c5c8c6;font-weight: bold } + .terminal-1669199940-r1 { fill: #c5c8c6 } +.terminal-1669199940-r2 { fill: #98a84b } +.terminal-1669199940-r3 { fill: #9a9b99 } +.terminal-1669199940-r4 { fill: #608ab1 } +.terminal-1669199940-r5 { fill: #d0b344 } +.terminal-1669199940-r6 { fill: #98a84b;font-weight: bold } +.terminal-1669199940-r7 { fill: #c5c8c6;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -114,30 +114,30 @@ - + - - $ nf-core modules bump-versions fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - - -╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│[!] 1 Module version up to date.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ -│Module name                             │Update Message                                       │ -├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ -│ fastqc                                   │ Module version up to date: fastqc                     │ -╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ + + $ nf-core modules bump-versions fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + + +╭──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│[!] 1 Module version up to date.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭──────────────────────────────────────────┬───────────────────────────────────────────────────────╮ +│Module name                             │Update Message                                       │ +├──────────────────────────────────────────┼───────────────────────────────────────────────────────┤ +│ fastqc                                   │ Module version up to date: fastqc                     │ +╰──────────────────────────────────────────┴───────────────────────────────────────────────────────╯ diff --git a/docs/images/nf-core-modules-create.svg b/docs/images/nf-core-modules-create.svg index caa9c389f0..6cd7e15b66 100644 --- a/docs/images/nf-core-modules-create.svg +++ b/docs/images/nf-core-modules-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' + + $ nf-core modules create fastqc --author @nf-core-bot  --label process_low --meta --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Using Bioconda package: 'bioconda::fastqc=0.12.1' +INFO     Using Docker container: 'biocontainers/fastqc:0.12.1--hdfd78af_0' +INFO     Using Singularity container:                                                                +'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' +INFO     Created component template: 'fastqc' +INFO     Created following files:                                                                    +           modules/nf-core/fastqc/main.nf                                                            +           modules/nf-core/fastqc/meta.yml                                                           +           modules/nf-core/fastqc/environment.yml                                                    +           modules/nf-core/fastqc/tests/tags.yml                                                     +           modules/nf-core/fastqc/tests/main.nf.test                                                 diff --git a/docs/images/nf-core-modules-info.svg b/docs/images/nf-core-modules-info.svg index 255c595e7c..92d19331ab 100644 --- a/docs/images/nf-core-modules-info.svg +++ b/docs/images/nf-core-modules-info.svg @@ -19,163 +19,163 @@ font-weight: 700; } - .terminal-1309602353-matrix { + .terminal-1309733425-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1309602353-title { + .terminal-1309733425-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1309602353-r1 { fill: #c5c8c6 } -.terminal-1309602353-r2 { fill: #98a84b } -.terminal-1309602353-r3 { fill: #9a9b99 } -.terminal-1309602353-r4 { fill: #608ab1 } -.terminal-1309602353-r5 { fill: #d0b344 } -.terminal-1309602353-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1309602353-r7 { fill: #98a84b;font-weight: bold } -.terminal-1309602353-r8 { fill: #868887 } -.terminal-1309602353-r9 { fill: #d08442 } -.terminal-1309602353-r10 { fill: #868887;font-style: italic; } -.terminal-1309602353-r11 { fill: #98729f } + .terminal-1309733425-r1 { fill: #c5c8c6 } +.terminal-1309733425-r2 { fill: #98a84b } +.terminal-1309733425-r3 { fill: #9a9b99 } +.terminal-1309733425-r4 { fill: #608ab1 } +.terminal-1309733425-r5 { fill: #d0b344 } +.terminal-1309733425-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1309733425-r7 { fill: #98a84b;font-weight: bold } +.terminal-1309733425-r8 { fill: #868887 } +.terminal-1309733425-r9 { fill: #d08442 } +.terminal-1309733425-r10 { fill: #868887;font-style: italic; } +.terminal-1309733425-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -187,53 +187,53 @@ - + - - $ nf-core modules info abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 🔧 Tools: abacas                                                                                 │ -│ 📖 Description: contiguate draft genome assembly                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -                  ╷                                                                   ╷              -📥 Inputs        │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} -                  ╵                                                                   ╵              -                  ╷                                                                   ╷              -📤 Outputs       │Description                                                        │     Pattern -╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ - meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ -                  │single_end:false ]                                                 │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - results  (file) │List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* -                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ -                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ -                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ -                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ -╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ - versions  (file)│File containing software versions                                  │versions.yml -                  ╵                                                                   ╵              - - 💻  Installation command: nf-core modules install abacas - + + $ nf-core modules info abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +╭─ Module: abacas  ────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 🔧 Tools: abacas                                                                                 │ +│ 📖 Description: contiguate draft genome assembly                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +                  ╷                                                                   ╷              +📥 Inputs        │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + scaffold  (file)│Fasta file containing scaffold                                     │*.{fasta,fa} +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + fasta  (file)   │FASTA reference file                                               │*.{fasta,fa} +                  ╵                                                                   ╵              +                  ╷                                                                   ╷              +📤 Outputs       │Description                                                        │     Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━╸ + meta  (map)     │Groovy Map containing sample information e.g. [ id:'test',         │ +                  │single_end:false ]                                                 │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + results  (file) │List containing abacas output files [ 'test.abacas.bin',           │ *.{abacas}* +                  │'test.abacas.fasta', 'test.abacas.gaps', 'test.abacas.gaps.tab',   │ +                  │'test.abacas.nucmer.delta', 'test.abacas.nucmer.filtered.delta',   │ +                  │'test.abacas.nucmer.tiling', 'test.abacas.tab',                    │ +                  │'test.abacas.unused.contigs.out', 'test.abacas.MULTIFASTA.fa' ]    │ +╶─────────────────┼───────────────────────────────────────────────────────────────────┼────────────╴ + versions  (file)│File containing software versions                                  │versions.yml +                  ╵                                                                   ╵              + + 💻  Installation command: nf-core modules install abacas + diff --git a/docs/images/nf-core-modules-install.svg b/docs/images/nf-core-modules-install.svg index 2b0331bcaa..51487ef6a5 100644 --- a/docs/images/nf-core-modules-install.svg +++ b/docs/images/nf-core-modules-install.svg @@ -19,76 +19,76 @@ font-weight: 700; } - .terminal-310938767-matrix { + .terminal-311069839-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-310938767-title { + .terminal-311069839-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-310938767-r1 { fill: #c5c8c6 } -.terminal-310938767-r2 { fill: #98a84b } -.terminal-310938767-r3 { fill: #9a9b99 } -.terminal-310938767-r4 { fill: #608ab1 } -.terminal-310938767-r5 { fill: #d0b344 } + .terminal-311069839-r1 { fill: #c5c8c6 } +.terminal-311069839-r2 { fill: #98a84b } +.terminal-311069839-r3 { fill: #9a9b99 } +.terminal-311069839-r4 { fill: #608ab1 } +.terminal-311069839-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -100,26 +100,26 @@ - + - - $ nf-core modules install abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Installing 'abacas' -INFO     Use the following statement to include this module:                                         - - include { ABACAS } from '../modules/nf-core/abacas/main'                                            - + + $ nf-core modules install abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Installing 'abacas' +INFO     Use the following statement to include this module:                                         + + include { ABACAS } from '../modules/nf-core/abacas/main'                                            + diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 2809eeee22..11c1c4b091 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' + +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  60 Tests Passed │ +│[!]   0 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-modules-list-local.svg b/docs/images/nf-core-modules-list-local.svg index ecb9ed499a..4956396bcf 100644 --- a/docs/images/nf-core-modules-list-local.svg +++ b/docs/images/nf-core-modules-list-local.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - + - + - - $ nf-core modules list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Modules installed in '.':                                                                   - -self.repo_type='pipeline' -┏━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Module Name┃Repository           ┃Version SHA          ┃Message              ┃Date      ┃ -┡━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ fastqc      │ https://github.com/n… │ f4ae1d942bd50c5c0b9b… │ Update FASTQC to use  │ 2024-01-31 │ -│             │                       │                       │ unique names for      │            │ -│             │                       │                       │ snapshots (#4825)     │            │ -│ multiqc     │ https://github.com/n… │ b7ebe95761cd389603f9… │ Update MQC container  │ 2024-02-29 │ -│             │                       │                       │ (#5006)               │            │ -└─────────────┴───────────────────────┴───────────────────────┴───────────────────────┴────────────┘ + + $ nf-core modules list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Repository type: pipeline +INFO     Modules installed in '.':                                                                   + +┏━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Module Name┃Repository     ┃Version SHA┃Message                             ┃Date      ┃ +┡━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ fastqc      │ nf-core/modules │ 285a505     │ Fix FastQC memory allocation (#5432) │ 2024-04-05 │ +│ multiqc     │ nf-core/modules │ b7ebe95     │ Update MQC container (#5006)         │ 2024-02-29 │ +└─────────────┴─────────────────┴─────────────┴──────────────────────────────────────┴────────────┘ diff --git a/docs/images/nf-core-modules-list-remote.svg b/docs/images/nf-core-modules-list-remote.svg index 138be73068..b6e6fb3a8e 100644 --- a/docs/images/nf-core-modules-list-remote.svg +++ b/docs/images/nf-core-modules-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2384550328-matrix { + .terminal-2384681400-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2384550328-title { + .terminal-2384681400-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2384550328-r1 { fill: #c5c8c6 } -.terminal-2384550328-r2 { fill: #98a84b } -.terminal-2384550328-r3 { fill: #9a9b99 } -.terminal-2384550328-r4 { fill: #608ab1 } -.terminal-2384550328-r5 { fill: #d0b344 } -.terminal-2384550328-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-2384550328-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2384550328-r8 { fill: #868887;font-style: italic; } + .terminal-2384681400-r1 { fill: #c5c8c6 } +.terminal-2384681400-r2 { fill: #98a84b } +.terminal-2384681400-r3 { fill: #9a9b99 } +.terminal-2384681400-r4 { fill: #608ab1 } +.terminal-2384681400-r5 { fill: #d0b344 } +.terminal-2384681400-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-2384681400-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-2384681400-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core modules list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Modules available from https://github.com/nf-core/modules.git(master):                     - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Module Name                                          ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ abacas                                                │ -│ abricate/run                                          │ -│ abricate/summary                                      │ -│ abritamr/run                                          │ -│ adapterremoval                                        │ -│ adapterremovalfixprefix                               │ -│ admixture                                             │ -│ affy/justrma                                          │ -│ agat/convertspgff2gtf                                 │ -[..truncated..] + + $ nf-core modules list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Modules available from https://github.com/nf-core/modules.git(master):                     + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Module Name                                          ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ abacas                                                │ +│ abricate/run                                          │ +│ abricate/summary                                      │ +│ abritamr/run                                          │ +│ adapterremoval                                        │ +│ adapterremovalfixprefix                               │ +│ admixture                                             │ +│ affy/justrma                                          │ +│ agat/convertspgff2gtf                                 │ +[..truncated..] diff --git a/docs/images/nf-core-modules-patch.svg b/docs/images/nf-core-modules-patch.svg index 5ed5e2f4d1..34d9c8dfb7 100644 --- a/docs/images/nf-core-modules-patch.svg +++ b/docs/images/nf-core-modules-patch.svg @@ -19,127 +19,127 @@ font-weight: 700; } - .terminal-1838296137-matrix { + .terminal-2649828425-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1838296137-title { + .terminal-2649828425-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1838296137-r1 { fill: #c5c8c6 } -.terminal-1838296137-r2 { fill: #98a84b } -.terminal-1838296137-r3 { fill: #9a9b99 } -.terminal-1838296137-r4 { fill: #608ab1 } -.terminal-1838296137-r5 { fill: #d0b344 } -.terminal-1838296137-r6 { fill: #ff2627 } -.terminal-1838296137-r7 { fill: #00823d } -.terminal-1838296137-r8 { fill: #ff2c7a;font-weight: bold } + .terminal-2649828425-r1 { fill: #c5c8c6 } +.terminal-2649828425-r2 { fill: #98a84b } +.terminal-2649828425-r3 { fill: #9a9b99 } +.terminal-2649828425-r4 { fill: #608ab1 } +.terminal-2649828425-r5 { fill: #d0b344 } +.terminal-2649828425-r6 { fill: #ff2627 } +.terminal-2649828425-r7 { fill: #00823d } +.terminal-2649828425-r8 { fill: #ff2c7a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -151,42 +151,42 @@ - + - - $ nf-core modules patch fastqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Changes in module 'nf-core/fastqc' -INFO    'modules/nf-core/fastqc/environment.yml' is unchanged                                       -INFO    'modules/nf-core/fastqc/meta.yml' is unchanged                                              -INFO     Changes in 'fastqc/main.nf':                                                                - ---- modules/nf-core/fastqc/main.nf -+++ modules/nf-core/fastqc/main.nf -@@ -1,6 +1,6 @@ -process FASTQC {                                                                                   -    tag "$meta.id"                                                                                 --    label 'process_medium' -+    label 'process_low' - -    conda "${moduleDir}/environment.yml"                                                           -    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  - - -INFO    'modules/nf-core/fastqc/tests/tags.yml' is unchanged                                        -INFO    'modules/nf-core/fastqc/tests/main.nf.test' is unchanged                                    -INFO    'modules/nf-core/fastqc/tests/main.nf.test.snap' is unchanged                               -INFO     Patch file of 'modules/nf-core/fastqc' written to 'modules/nf-core/fastqc/fastqc.diff' + + $ nf-core modules patch fastqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Changes in module 'nf-core/fastqc' +INFO     Changes in 'fastqc/main.nf':                                                                + +--- modules/nf-core/fastqc/main.nf ++++ modules/nf-core/fastqc/main.nf +@@ -1,6 +1,6 @@ +process FASTQC {                                                                                   +    tag "$meta.id"                                                                                 +-    label 'process_medium' ++    label 'process_low' + +    conda "${moduleDir}/environment.yml"                                                           +    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_  + + +INFO    'modules/nf-core/fastqc/meta.yml' is unchanged                                              +INFO    'modules/nf-core/fastqc/environment.yml' is unchanged                                       +INFO    'modules/nf-core/fastqc/tests/tags.yml' is unchanged                                        +INFO    'modules/nf-core/fastqc/tests/main.nf.test.snap' is unchanged                               +INFO    'modules/nf-core/fastqc/tests/main.nf.test' is unchanged                                    +INFO     Patch file of 'modules/nf-core/fastqc' written to 'modules/nf-core/fastqc/fastqc.diff' diff --git a/docs/images/nf-core-modules-remove.svg b/docs/images/nf-core-modules-remove.svg index 91f7020043..af35fe2d31 100644 --- a/docs/images/nf-core-modules-remove.svg +++ b/docs/images/nf-core-modules-remove.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-2115907041-matrix { + .terminal-2116038113-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2115907041-title { + .terminal-2116038113-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2115907041-r1 { fill: #c5c8c6 } -.terminal-2115907041-r2 { fill: #98a84b } -.terminal-2115907041-r3 { fill: #9a9b99 } -.terminal-2115907041-r4 { fill: #608ab1 } -.terminal-2115907041-r5 { fill: #d0b344 } + .terminal-2116038113-r1 { fill: #c5c8c6 } +.terminal-2116038113-r2 { fill: #98a84b } +.terminal-2116038113-r3 { fill: #9a9b99 } +.terminal-2116038113-r4 { fill: #608ab1 } +.terminal-2116038113-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules remove abacas - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Removed files for 'abacas' and its dependencies 'abacas'.                                   + + $ nf-core modules remove abacas + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Removed files for 'abacas' and its dependencies 'abacas'.                                   diff --git a/docs/images/nf-core-modules-test.svg b/docs/images/nf-core-modules-test.svg index 52e86ba826..cb1304e879 100644 --- a/docs/images/nf-core-modules-test.svg +++ b/docs/images/nf-core-modules-test.svg @@ -19,64 +19,64 @@ font-weight: 700; } - .terminal-9171199-matrix { + .terminal-9302271-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-9171199-title { + .terminal-9302271-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-9171199-r1 { fill: #c5c8c6 } -.terminal-9171199-r2 { fill: #98a84b } -.terminal-9171199-r3 { fill: #9a9b99 } -.terminal-9171199-r4 { fill: #608ab1 } -.terminal-9171199-r5 { fill: #d0b344 } + .terminal-9302271-r1 { fill: #c5c8c6 } +.terminal-9302271-r2 { fill: #98a84b } +.terminal-9302271-r3 { fill: #9a9b99 } +.terminal-9302271-r4 { fill: #608ab1 } +.terminal-9302271-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +88,22 @@ - + - - $ nf-core modules test fastqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Generating nf-test snapshot                                                                 + + $ nf-core modules test fastqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Generating nf-test snapshot                                                                 diff --git a/docs/images/nf-core-modules-update.svg b/docs/images/nf-core-modules-update.svg index 23d9931ce2..46983bf5ea 100644 --- a/docs/images/nf-core-modules-update.svg +++ b/docs/images/nf-core-modules-update.svg @@ -19,73 +19,73 @@ font-weight: 700; } - .terminal-2984179218-matrix { + .terminal-2984310290-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2984179218-title { + .terminal-2984310290-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2984179218-r1 { fill: #c5c8c6 } -.terminal-2984179218-r2 { fill: #98a84b } -.terminal-2984179218-r3 { fill: #9a9b99 } -.terminal-2984179218-r4 { fill: #608ab1 } -.terminal-2984179218-r5 { fill: #d0b344 } + .terminal-2984310290-r1 { fill: #c5c8c6 } +.terminal-2984310290-r2 { fill: #98a84b } +.terminal-2984310290-r3 { fill: #9a9b99 } +.terminal-2984310290-r4 { fill: #608ab1 } +.terminal-2984310290-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -97,25 +97,25 @@ - + - - $ nf-core modules update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO    'modules/nf-core/abacas' is already up to date                                              -INFO    'modules/nf-core/fastqc' is already up to date                                              -INFO    'modules/nf-core/multiqc' is already up to date                                             -INFO     Updates complete ✨                                                                         + + $ nf-core modules update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO    'modules/nf-core/abacas' is already up to date                                              +INFO    'modules/nf-core/fastqc' is already up to date                                              +INFO    'modules/nf-core/multiqc' is already up to date                                             +INFO     Updates complete ✨                                                                         diff --git a/docs/images/nf-core-schema-build.svg b/docs/images/nf-core-schema-build.svg index 7236440a0d..6d3e448b04 100644 --- a/docs/images/nf-core-schema-build.svg +++ b/docs/images/nf-core-schema-build.svg @@ -19,72 +19,72 @@ font-weight: 700; } - .terminal-978569984-matrix { + .terminal-1038338818-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-978569984-title { + .terminal-1038338818-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-978569984-r1 { fill: #c5c8c6 } -.terminal-978569984-r2 { fill: #98a84b } -.terminal-978569984-r3 { fill: #9a9b99 } -.terminal-978569984-r4 { fill: #608ab1 } -.terminal-978569984-r5 { fill: #d0b344 } -.terminal-978569984-r6 { fill: #98a84b;font-weight: bold } -.terminal-978569984-r7 { fill: #868887;font-weight: bold } -.terminal-978569984-r8 { fill: #868887 } -.terminal-978569984-r9 { fill: #4e707b;font-weight: bold } -.terminal-978569984-r10 { fill: #68a0b3;font-weight: bold } + .terminal-1038338818-r1 { fill: #c5c8c6 } +.terminal-1038338818-r2 { fill: #98a84b } +.terminal-1038338818-r3 { fill: #9a9b99 } +.terminal-1038338818-r4 { fill: #608ab1 } +.terminal-1038338818-r5 { fill: #d0b344 } +.terminal-1038338818-r6 { fill: #98a84b;font-weight: bold } +.terminal-1038338818-r7 { fill: #868887;font-weight: bold } +.terminal-1038338818-r8 { fill: #868887 } +.terminal-1038338818-r9 { fill: #4e707b;font-weight: bold } +.terminal-1038338818-r10 { fill: #68a0b3;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -96,23 +96,23 @@ - + - - $ nf-core schema build --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 30 params) -INFO     Writing schema with 31 params: 'nextflow_schema.json' + + $ nf-core schema build --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 31 params) +INFO     Writing schema with 32 params: 'nextflow_schema.json' diff --git a/docs/images/nf-core-schema-lint.svg b/docs/images/nf-core-schema-lint.svg index d08a01144b..1b1a836556 100644 --- a/docs/images/nf-core-schema-lint.svg +++ b/docs/images/nf-core-schema-lint.svg @@ -19,68 +19,68 @@ font-weight: 700; } - .terminal-3566947207-matrix { + .terminal-3582544776-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3566947207-title { + .terminal-3582544776-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3566947207-r1 { fill: #c5c8c6 } -.terminal-3566947207-r2 { fill: #98a84b } -.terminal-3566947207-r3 { fill: #9a9b99 } -.terminal-3566947207-r4 { fill: #608ab1 } -.terminal-3566947207-r5 { fill: #d0b344 } -.terminal-3566947207-r6 { fill: #98a84b;font-weight: bold } -.terminal-3566947207-r7 { fill: #868887;font-weight: bold } -.terminal-3566947207-r8 { fill: #868887 } -.terminal-3566947207-r9 { fill: #4e707b;font-weight: bold } + .terminal-3582544776-r1 { fill: #c5c8c6 } +.terminal-3582544776-r2 { fill: #98a84b } +.terminal-3582544776-r3 { fill: #9a9b99 } +.terminal-3582544776-r4 { fill: #608ab1 } +.terminal-3582544776-r5 { fill: #d0b344 } +.terminal-3582544776-r6 { fill: #98a84b;font-weight: bold } +.terminal-3582544776-r7 { fill: #868887;font-weight: bold } +.terminal-3582544776-r8 { fill: #868887 } +.terminal-3582544776-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -92,22 +92,22 @@ - + - - $ nf-core schema lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 31 params) + + $ nf-core schema lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 32 params) diff --git a/docs/images/nf-core-schema-validate.svg b/docs/images/nf-core-schema-validate.svg index 33984ccbeb..94663e3812 100644 --- a/docs/images/nf-core-schema-validate.svg +++ b/docs/images/nf-core-schema-validate.svg @@ -19,71 +19,71 @@ font-weight: 700; } - .terminal-2456222967-matrix { + .terminal-2456354039-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2456222967-title { + .terminal-2456354039-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2456222967-r1 { fill: #c5c8c6 } -.terminal-2456222967-r2 { fill: #98a84b } -.terminal-2456222967-r3 { fill: #9a9b99 } -.terminal-2456222967-r4 { fill: #608ab1 } -.terminal-2456222967-r5 { fill: #d0b344 } -.terminal-2456222967-r6 { fill: #98a84b;font-weight: bold } -.terminal-2456222967-r7 { fill: #868887;font-weight: bold } -.terminal-2456222967-r8 { fill: #868887 } -.terminal-2456222967-r9 { fill: #4e707b;font-weight: bold } + .terminal-2456354039-r1 { fill: #c5c8c6 } +.terminal-2456354039-r2 { fill: #98a84b } +.terminal-2456354039-r3 { fill: #9a9b99 } +.terminal-2456354039-r4 { fill: #608ab1 } +.terminal-2456354039-r5 { fill: #d0b344 } +.terminal-2456354039-r6 { fill: #98a84b;font-weight: bold } +.terminal-2456354039-r7 { fill: #868887;font-weight: bold } +.terminal-2456354039-r8 { fill: #868887 } +.terminal-2456354039-r9 { fill: #4e707b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,23 +95,23 @@ - + - - $ nf-core schema validate nf-core-rnaseq/3_8 nf-params.json - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -INFO    [✓] Default parameters match schema validation -INFO    [✓] Pipeline schema looks valid(found 93 params) -INFO    [✓] Input parameters look valid + + $ nf-core schema validate nf-core-rnaseq/3_8 nf-params.json + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO    [✓] Default parameters match schema validation +INFO    [✓] Pipeline schema looks valid(found 93 params) +INFO    [✓] Input parameters look valid diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index fd20914cfe..f5b3212fa7 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Repository type: modules -INFO    Press enter to use default values (shown in brackets)or type your own responses.  -ctrl+click underlined text to open links. -INFO     Created component template: 'bam_stats_samtools' -INFO     Created following files:                                                                    -           subworkflows/nf-core/bam_stats_samtools/main.nf                                           -           subworkflows/nf-core/bam_stats_samtools/meta.yml                                          -           subworkflows/nf-core/bam_stats_samtools/tests/tags.yml                                    -           subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test                                + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-info.svg b/docs/images/nf-core-subworkflows-info.svg index b3f4f38ed3..e2c828102c 100644 --- a/docs/images/nf-core-subworkflows-info.svg +++ b/docs/images/nf-core-subworkflows-info.svg @@ -19,112 +19,112 @@ font-weight: 700; } - .terminal-1309193345-matrix { + .terminal-1309324417-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1309193345-title { + .terminal-1309324417-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1309193345-r1 { fill: #c5c8c6 } -.terminal-1309193345-r2 { fill: #98a84b } -.terminal-1309193345-r3 { fill: #9a9b99 } -.terminal-1309193345-r4 { fill: #608ab1 } -.terminal-1309193345-r5 { fill: #d0b344 } -.terminal-1309193345-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1309193345-r7 { fill: #98a84b;font-weight: bold } -.terminal-1309193345-r8 { fill: #868887 } -.terminal-1309193345-r9 { fill: #868887;font-style: italic; } -.terminal-1309193345-r10 { fill: #d08442 } -.terminal-1309193345-r11 { fill: #98729f } + .terminal-1309324417-r1 { fill: #c5c8c6 } +.terminal-1309324417-r2 { fill: #98a84b } +.terminal-1309324417-r3 { fill: #9a9b99 } +.terminal-1309324417-r4 { fill: #608ab1 } +.terminal-1309324417-r5 { fill: #d0b344 } +.terminal-1309324417-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1309324417-r7 { fill: #98a84b;font-weight: bold } +.terminal-1309324417-r8 { fill: #868887 } +.terminal-1309324417-r9 { fill: #868887;font-style: italic; } +.terminal-1309324417-r10 { fill: #d08442 } +.terminal-1309324417-r11 { fill: #98729f } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -136,36 +136,36 @@ - + - - $ nf-core subworkflows info bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -╭─ Subworkflow: bam_rseqc  ────────────────────────────────────────────────────────────────────────╮ -│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ -│ 📖 Description: Subworkflow to run multiple commands in the RSeqC package                        │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -[..truncated..] - readduplication_rscript  (file)    │script to reproduce the plot       │                      *.R -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - tin_txt  (file)                    │TXT file containing tin.py results │                    *.txt -                                     │summary                            │ -╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ - versions  (file)                   │File containing software versions  │             versions.yml -                                     ╵                                   ╵                           - - 💻  Installation command: nf-core subworkflows install bam_rseqc - + + $ nf-core subworkflows info bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +╭─ Subworkflow: bam_rseqc  ────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: https://github.com/nf-core/modules.git                                            │ +│ 📖 Description: Subworkflow to run multiple commands in the RSeqC package                        │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +[..truncated..] + readduplication_rscript  (file)    │script to reproduce the plot       │                      *.R +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + tin_txt  (file)                    │TXT file containing tin.py results │                    *.txt +                                     │summary                            │ +╶────────────────────────────────────┼───────────────────────────────────┼─────────────────────────╴ + versions  (file)                   │File containing software versions  │             versions.yml +                                     ╵                                   ╵                           + + 💻  Installation command: nf-core subworkflows install bam_rseqc + diff --git a/docs/images/nf-core-subworkflows-install.svg b/docs/images/nf-core-subworkflows-install.svg index 8c86c3a7e4..628ed9fef0 100644 --- a/docs/images/nf-core-subworkflows-install.svg +++ b/docs/images/nf-core-subworkflows-install.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - + - + - - $ nf-core subworkflows install bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Installing 'bam_rseqc' -INFO     Use the following statement to include this subworkflow:                                    - - include { BAM_RSEQC } from '../subworkflows/nf-core/bam_rseqc/main'                                 - + + $ nf-core subworkflows install bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Installing 'bam_rseqc' diff --git a/docs/images/nf-core-subworkflows-lint.svg b/docs/images/nf-core-subworkflows-lint.svg index 7e827661ac..8250d22683 100644 --- a/docs/images/nf-core-subworkflows-lint.svg +++ b/docs/images/nf-core-subworkflows-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core subworkflows lint bam_stats_samtools - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting subworkflow: 'bam_stats_samtools' - -╭─[!] 14 Subworkflow Test Warnings───────────────────────────────────────────────────────────────╮ -│                     ╷                                     ╷                                      │ -│Subworkflow name   │File path                          │Test message                        │ -│╶────────────────────┼─────────────────────────────────────┼─────────────────────────────────────╴│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: //Add │ -│││all required assertions to verify │ -│││the test output.│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Add │ -│││tags for all modules used within │ -│││this subworkflow. Example:│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Change │ -│││the test name preferably indicating │ -│││the test-data and file-format used│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Once │ -│││you have added the required tests, │ -│││please run the following command to │ -│││build this file:│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: define │ -│││inputs of the workflow here. │ -│││Example:│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: A │ -│││subworkflow SHOULD import at least │ -│││two modules│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: If in doubt │ -│││look at other nf-core/subworkflows │ -│││to see how we are doing things! :)│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit emitted│ -│││channels│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit input │ -│││(take) channels│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: substitute │ -│││modules here for the modules of your│ -│││subworkflow│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a │ -│││description of the subworkflow and │ -│││list keywords│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a list│ -│││of the modules and/or subworkflows │ -│││used in the subworkflow│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ -│││of the channels used as input with a│ -│││description and their structure│ -│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ -│││of the channels used as output with │ -│││a descriptions and their structure│ -│                     ╵                                     ╵                                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  42 Tests Passed │ -│[!]  14 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + + + $ nf-core subworkflows lint bam_stats_samtools + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-list-local.svg b/docs/images/nf-core-subworkflows-list-local.svg index 6bec883e1f..50072fb6df 100644 --- a/docs/images/nf-core-subworkflows-list-local.svg +++ b/docs/images/nf-core-subworkflows-list-local.svg @@ -19,108 +19,108 @@ font-weight: 700; } - .terminal-2930702375-matrix { + .terminal-1791436705-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2930702375-title { + .terminal-1791436705-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2930702375-r1 { fill: #c5c8c6 } -.terminal-2930702375-r2 { fill: #98a84b } -.terminal-2930702375-r3 { fill: #9a9b99 } -.terminal-2930702375-r4 { fill: #608ab1 } -.terminal-2930702375-r5 { fill: #d0b344 } -.terminal-2930702375-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2930702375-r7 { fill: #868887;font-style: italic; } + .terminal-1791436705-r1 { fill: #c5c8c6 } +.terminal-1791436705-r2 { fill: #98a84b } +.terminal-1791436705-r3 { fill: #9a9b99 } +.terminal-1791436705-r4 { fill: #608ab1 } +.terminal-1791436705-r5 { fill: #d0b344 } +.terminal-1791436705-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-1791436705-r7 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -132,36 +132,36 @@ - + - - $ nf-core subworkflows list local - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Subworkflows installed in '.':                                                              - -self.repo_type='pipeline' -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ -┃Subworkflow Name   ┃Repository        ┃Version SHA        ┃Message           ┃Date      ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ -│ utils_nextflow_pip… │ https://github.co… │ 5caf7640a9ef1d18d7… │ remove             │ 2024-02-28 │ -│                     │                    │                     │ params.outdir from │            │ -│                     │                    │                     │ modules and        │            │ -│                     │                    │                     │ subworfklows tests │            │ -│                     │                    │                     │ (#5007)            │            │ -│ utils_nfcore_pipel… │ https://github.co… │ 5caf7640a9ef1d18d7… │ remove             │ 2024-02-28 │ -│                     │                    │                     │ params.outdir from │            │ -│                     │                    │                     │ modules and        │            │ -[..truncated..] + + $ nf-core subworkflows list local + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Repository type: pipeline +INFO     Subworkflows installed in '.':                                                              + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓ +┃Subworkflow Name        ┃Repository     ┃Version SHA┃Message                 ┃Date      ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩ +│ utils_nextflow_pipeline  │ nf-core/modules │ 5caf764     │ remove params.outdir     │ 2024-02-28 │ +│                          │                 │             │ from modules and         │            │ +│                          │                 │             │ subworfklows tests       │            │ +│                          │                 │             │ (#5007)                  │            │ +│ utils_nfcore_pipeline    │ nf-core/modules │ 92de218     │ handle multiple DOIs     │ 2024-05-02 │ +│                          │                 │             │ (#5556)                  │            │ +│ utils_nfvalidation_plug… │ nf-core/modules │ 5caf764     │ remove params.outdir     │ 2024-02-28 │ +│                          │                 │             │ from modules and         │            │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-list-remote.svg b/docs/images/nf-core-subworkflows-list-remote.svg index 704c2e9a45..b2c9fcb327 100644 --- a/docs/images/nf-core-subworkflows-list-remote.svg +++ b/docs/images/nf-core-subworkflows-list-remote.svg @@ -19,109 +19,109 @@ font-weight: 700; } - .terminal-2294237964-matrix { + .terminal-1181242108-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2294237964-title { + .terminal-1181242108-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2294237964-r1 { fill: #c5c8c6 } -.terminal-2294237964-r2 { fill: #98a84b } -.terminal-2294237964-r3 { fill: #9a9b99 } -.terminal-2294237964-r4 { fill: #608ab1 } -.terminal-2294237964-r5 { fill: #d0b344 } -.terminal-2294237964-r6 { fill: #1984e9;text-decoration: underline; } -.terminal-2294237964-r7 { fill: #c5c8c6;font-weight: bold } -.terminal-2294237964-r8 { fill: #868887;font-style: italic; } + .terminal-1181242108-r1 { fill: #c5c8c6 } +.terminal-1181242108-r2 { fill: #98a84b } +.terminal-1181242108-r3 { fill: #9a9b99 } +.terminal-1181242108-r4 { fill: #608ab1 } +.terminal-1181242108-r5 { fill: #d0b344 } +.terminal-1181242108-r6 { fill: #1984e9;text-decoration: underline; } +.terminal-1181242108-r7 { fill: #c5c8c6;font-weight: bold } +.terminal-1181242108-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,36 +133,36 @@ - + - - $ nf-core subworkflows list remote - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                - -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃Subworkflow Name                             ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ bam_cnv_wisecondorx                           │ -│ bam_create_som_pon_gatk                       │ -│ bam_dedup_stats_samtools_umitools             │ -│ bam_docounts_contamination_angsd              │ -│ bam_markduplicates_picard                     │ -│ bam_markduplicates_samtools                   │ -│ bam_ngscheckmate                              │ -│ bam_qc_picard                                 │ -│ bam_rseqc                                     │ -[..truncated..] + + $ nf-core subworkflows list remote + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Subworkflows available from https://github.com/nf-core/modules.git(master):                + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Subworkflow Name                             ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ bam_cnv_wisecondorx                           │ +│ bam_create_som_pon_gatk                       │ +│ bam_dedup_stats_samtools_umicollapse          │ +│ bam_dedup_stats_samtools_umitools             │ +│ bam_docounts_contamination_angsd              │ +│ bam_markduplicates_picard                     │ +│ bam_markduplicates_samtools                   │ +│ bam_ngscheckmate                              │ +│ bam_qc_picard                                 │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 42c576f4c5..61e782532d 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Removed files for 'rseqc/bamstat' and its dependencies 'rseqc/bamstat'.                     -INFO     Removed files for 'rseqc/inferexperiment' and its dependencies 'rseqc/inferexperiment'.     -INFO     Removed files for 'rseqc/innerdistance' and its dependencies 'rseqc/innerdistance'.         -INFO     Removed files for 'rseqc/junctionannotation' and its dependencies                           -'rseqc/junctionannotation'.                                                                 -INFO     Removed files for 'rseqc/junctionsaturation' and its dependencies                           -'rseqc/junctionsaturation'.                                                                 -INFO     Removed files for 'rseqc/readdistribution' and its dependencies 'rseqc/readdistribution'.   -INFO     Removed files for 'rseqc/readduplication' and its dependencies 'rseqc/readduplication'.     -INFO     Removed files for 'rseqc/tin' and its dependencies 'rseqc/tin'.                             -INFO     Removed files for 'bam_rseqc' and its dependencies 'bam_rseqc, rseqc_bamstat,  -rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation,  -rseqc_junctionsaturation, rseqc_readdistribution, rseqc_readduplication, rseqc_tin'.        + + + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index 9cda6beda6..447fb6e5ec 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -19,64 +19,65 @@ font-weight: 700; } - .terminal-1767175273-matrix { + .terminal-3677110743-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1767175273-title { + .terminal-3677110743-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1767175273-r1 { fill: #c5c8c6 } -.terminal-1767175273-r2 { fill: #98a84b } -.terminal-1767175273-r3 { fill: #9a9b99 } -.terminal-1767175273-r4 { fill: #608ab1 } -.terminal-1767175273-r5 { fill: #d0b344 } + .terminal-3677110743-r1 { fill: #c5c8c6 } +.terminal-3677110743-r2 { fill: #98a84b } +.terminal-3677110743-r3 { fill: #9a9b99 } +.terminal-3677110743-r4 { fill: #608ab1 } +.terminal-3677110743-r5 { fill: #d0b344 } +.terminal-3677110743-r6 { fill: #292929;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -88,22 +89,22 @@ - - - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO     Generating nf-test snapshot                                                                 + + + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index 3398899b7e..e76364a259 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - + - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - - -INFO    'subworkflows/nf-core/bam_rseqc' is already up to date                                      -INFO    'subworkflows/nf-core/utils_nextflow_pipeline' is already up to date                        -INFO    'subworkflows/nf-core/utils_nfcore_pipeline' is already up to date                          -INFO    'subworkflows/nf-core/utils_nfvalidation_plugin' is already up to date                      -INFO     Updates complete ✨                                                                         + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' diff --git a/docs/images/nf-core-sync.svg b/docs/images/nf-core-sync.svg index b2c6d734fc..3a198f5713 100644 --- a/docs/images/nf-core-sync.svg +++ b/docs/images/nf-core-sync.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core sync - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.13.1 - https://nf-co.re - - -WARNING  Could not find GitHub authentication token. Some API requests may fail.                     -INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -INFO     Original pipeline repository branch is 'master' -INFO     Deleting all files in 'TEMPLATE' branch                                                     -INFO     Making a new template pipeline using pipeline variables                                     -INFO     Committed changes to 'TEMPLATE' branch                                                      -INFO     Checking out original branch: 'master' -INFO     Now try to merge the updates in to your pipeline:                                           -           cd /home/runner/work/tools/tools/tmp/nf-core-nextbigthing -           git merge TEMPLATE                                                                        + + $ nf-core sync + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +WARNING  Could not find GitHub authentication token. Some API requests may fail.                     +INFO     Pipeline directory: /home/runner/work/tools/tools/tmp/nf-core-nextbigthing +INFO     Original pipeline repository branch is 'master' From 4d0012e15c37236b75df9f885e643fabfb27975b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 7 May 2024 15:46:55 +0000 Subject: [PATCH 265/270] Generate new screengrabs with rich-codex --- docs/images/nf-core-licences.svg | 135 ++++++++------------------- docs/images/nf-core-list-stars.svg | 104 ++++++++++----------- docs/images/nf-core-list.svg | 108 ++++++++++----------- docs/images/nf-core-modules-lint.svg | 115 ++++++++--------------- 4 files changed, 183 insertions(+), 279 deletions(-) diff --git a/docs/images/nf-core-licences.svg b/docs/images/nf-core-licences.svg index 4cbed577c1..d8aa9b03a3 100644 --- a/docs/images/nf-core-licences.svg +++ b/docs/images/nf-core-licences.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core licences deepvariant - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - -INFO     Fetching licence information for 8 tools                                                    -INFO     Warning: This tool only prints licence information for the software tools packaged using    -         conda.                                                                                      -INFO     The pipeline may use other software and dependencies not described here.                    -┏━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┓ -┃Package Name┃Version┃Licence┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━┩ -│ lbzip2       │ 2.5     │ GPL v3  │ -│ deepvariant  │ 0.7.0   │ MIT     │ -│ htslib       │ 1.9     │ MIT     │ -│ picard       │ 2.18.7  │ MIT     │ -│ pip          │ 10.0.1  │ MIT     │ -│ samtools     │ 1.9     │ MIT     │ -│ python       │ 2.7.15  │ PSF     │ -│ bzip2        │ 1.0.6   │ bzip2   │ -└──────────────┴─────────┴─────────┘ + + $ nf-core licences deepvariant + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +INFO     Fetching licence information for 8 tools                                                    diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index f033a14092..49c10f7cd7 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-3593730463-matrix { + .terminal-3664247200-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3593730463-title { + .terminal-3664247200-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3593730463-r1 { fill: #c5c8c6 } -.terminal-3593730463-r2 { fill: #98a84b } -.terminal-3593730463-r3 { fill: #9a9b99 } -.terminal-3593730463-r4 { fill: #608ab1 } -.terminal-3593730463-r5 { fill: #d0b344 } -.terminal-3593730463-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3593730463-r7 { fill: #868887 } -.terminal-3593730463-r8 { fill: #868887;font-style: italic; } + .terminal-3664247200-r1 { fill: #c5c8c6 } +.terminal-3664247200-r2 { fill: #98a84b } +.terminal-3664247200-r3 { fill: #9a9b99 } +.terminal-3664247200-r4 { fill: #608ab1 } +.terminal-3664247200-r5 { fill: #d0b344 } +.terminal-3664247200-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3664247200-r7 { fill: #868887 } +.terminal-3664247200-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ -│ sarek               │   340 │          3.4.2 │   3 hours ago │           - │ -                   │ -│ mag                 │   182 │          2.5.4 │  3 months ago │           - │ -                   │ -│ chipseq             │   172 │          2.0.0 │   2 years ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ +│ sarek               │   340 │          3.4.2 │   4 hours ago │           - │ -                   │ +│ mag                 │   182 │          2.5.4 │  3 months ago │           - │ -                   │ +│ chipseq             │   172 │          2.0.0 │   2 years ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index 79e73ed0e7..ede53daf6f 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-1180468483-matrix { + .terminal-2737538270-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1180468483-title { + .terminal-2737538270-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1180468483-r1 { fill: #c5c8c6 } -.terminal-1180468483-r2 { fill: #98a84b } -.terminal-1180468483-r3 { fill: #9a9b99 } -.terminal-1180468483-r4 { fill: #608ab1 } -.terminal-1180468483-r5 { fill: #d0b344 } -.terminal-1180468483-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1180468483-r7 { fill: #868887 } -.terminal-1180468483-r8 { fill: #868887;font-style: italic; } + .terminal-2737538270-r1 { fill: #c5c8c6 } +.terminal-2737538270-r2 { fill: #98a84b } +.terminal-2737538270-r3 { fill: #9a9b99 } +.terminal-2737538270-r4 { fill: #608ab1 } +.terminal-2737538270-r5 { fill: #d0b344 } +.terminal-2737538270-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-2737538270-r7 { fill: #868887 } +.terminal-2737538270-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ -│ sarek               │   340 │          3.4.2 │   3 hours ago │           - │ -                   │ -│ variantbenchmarking │     4 │            dev │   an hour ago │           - │ -                   │ -│ meerpipe            │     0 │            dev │   8 hours ago │           - │ -                   │ -│ mcmicro             │     4 │            dev │  20 hours ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ +│ sarek               │   340 │          3.4.2 │   4 hours ago │           - │ -                   │ +│ variantbenchmarking │     4 │            dev │   3 hours ago │           - │ -                   │ +│ meerpipe            │     0 │            dev │   9 hours ago │           - │ -                   │ +│ mcmicro             │     4 │            dev │  22 hours ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-modules-lint.svg b/docs/images/nf-core-modules-lint.svg index 11c1c4b091..355ca8d32d 100644 --- a/docs/images/nf-core-modules-lint.svg +++ b/docs/images/nf-core-modules-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - $ nf-core modules lint multiqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - - -INFO     Linting modules repo: '.' -INFO     Linting module: 'multiqc' - -╭───────────────────────╮ -│LINT RESULTS SUMMARY│ -├───────────────────────┤ -│[✔]  60 Tests Passed │ -│[!]   0 Test Warnings│ -│[✗]   0 Tests Failed │ -╰───────────────────────╯ + + $ nf-core modules lint multiqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting module: 'multiqc' From b56906e0d01bc729d1d7886ae6dedb8aa060acc0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 8 May 2024 07:56:30 +0000 Subject: [PATCH 266/270] Generate new screengrabs with rich-codex --- docs/images/nf-core-lint.svg | 888 ++++++++++++++++++-- docs/images/nf-core-list-rna.svg | 138 +-- docs/images/nf-core-list-stars.svg | 104 +-- docs/images/nf-core-list.svg | 108 +-- docs/images/nf-core-subworkflows-create.svg | 110 ++- docs/images/nf-core-subworkflows-lint.svg | 314 ++++++- docs/images/nf-core-subworkflows-remove.svg | 103 ++- docs/images/nf-core-subworkflows-test.svg | 73 +- docs/images/nf-core-subworkflows-update.svg | 72 +- 9 files changed, 1452 insertions(+), 458 deletions(-) diff --git a/docs/images/nf-core-lint.svg b/docs/images/nf-core-lint.svg index d5a6fb8488..9276638e99 100644 --- a/docs/images/nf-core-lint.svg +++ b/docs/images/nf-core-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core lint - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - - -INFO     Testing pipeline: . - - -╭─[?] 1 Pipeline Test Ignored────────────────────────────────────────────────────────────────────╮ -││ -│pipeline_todos: pipeline_todos                                                                  │ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─[!] 1 Pipeline Test Warning────────────────────────────────────────────────────────────────────╮ -││ -│readme: README contains the placeholder zenodo.XXXXXXX. This should be replaced with the zenodo │ -│doi (after the first release).                                                                  │ -││ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ - - -╭──────────────────────╮ -│LINT RESULTS SUMMARY│ -├──────────────────────┤ -│[✔] 194 Tests Passed│ -│[?]   1 Test Ignored│ -│[!]   1 Test Warning│ -│[✗]   0 Tests Failed│ -╰──────────────────────╯ + + + + $ nf-core lint + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Testing pipeline: . + +╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/requests/models.py:971 in    │ +│json│ +││ +│ 968 │   │   │   │   │   raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)                    │ +│ 969 │   │   │ +│ 970 │   │   try:                                                                              │ +│❱  971 │   │   │   return complexjson.loads(self.text, **kwargs)                                 │ +│ 972 │   │   except JSONDecodeError as e:                                                      │ +│ 973 │   │   │   # Catch JSON-related errors and raise as requests.JSONDecodeError│ +│ 974 │   │   │   # This aliases json.JSONDecodeError and simplejson.JSONDecodeError│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/json/__init__.py:346 in loads│ +││ +│343 │   if (clsisNoneand object_hook isNoneand│ +│344 │   │   │   parse_int isNoneand parse_float isNoneand│ +│345 │   │   │   parse_constant isNoneand object_pairs_hook isNoneandnot kw):              │ +│❱ 346 │   │   return _default_decoder.decode(s)                                                  │ +│347 │   ifclsisNone:                                                                        │ +│348 │   │   cls = JSONDecoder                                                                  │ +│349 │   if object_hook isnotNone:                                                            │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/json/decoder.py:337 in decode│ +││ +│334 │   │   containing a JSON document).│ +│335 │   │   │ +│336 │   │   """│ +│❱ 337 │   │   obj, end = self.raw_decode(s, idx=_w(s, 0).end())                                  │ +│338 │   │   end = _w(s, end).end()                                                             │ +│339 │   │   if end != len(s):                                                                  │ +│340 │   │   │   raise JSONDecodeError("Extra data", s, end)                                    │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/json/decoder.py:355 in raw_decode│ +││ +│352 │   │   try:                                                                               │ +│353 │   │   │   obj, end = self.scan_once(s, idx)                                              │ +│354 │   │   exceptStopIterationas err:                                                       │ +│❱ 355 │   │   │   raise JSONDecodeError("Expecting value", s, err.value) fromNone│ +│356 │   │   return obj, end                                                                    │ +│357 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +JSONDecodeError: Expecting value: line 1 column 1(char 0) + +During handling of the above exception, another exception occurred: + +╭───────────────────────────────Traceback (most recent call last)────────────────────────────────╮ +│/opt/hostedtoolcache/Python/3.12.3/x64/bin/nf-core:8 in <module>│ +││ +│5 fromnf_core.__main__import run_nf_core                                                     │ +│6 if__name__ == '__main__':                                                                   │ +│7 │   sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])                         │ +│❱ 8 │   sys.exit(run_nf_core())                                                                  │ +│9 │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/__main__.py:150 in   │ +│run_nf_core│ +││ +│ 147 │   │   │   log.debug(f"Could not check latest version: {e}")                             │ +│ 148 │   │   stderr.print("\n")                                                                │ +│ 149 │   # Launch the click cli│ +│❱  150 │   nf_core_cli(auto_envvar_prefix="NFCORE")                                              │ +│ 151 │ +│ 152 │ +│ 153 @tui()                                                                                    │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/rich_click/rich_command.py:3│ +│67 in __call__│ +││ +│364 │   │   # Include this here because I run into a false warning│ +│365 │   │   # in the PyCharm IDE otherwise; for some reason PyCharm doesn't│ +│366 │   │   # seem to think RichGroups are callable. (No issues with Mypy, though.)│ +│❱ 367 │   │   returnsuper().__call__(*args, **kwargs)                                           │ +│368 │ +│369 │ +│370 classRichCommandCollection(CommandCollection, RichGroup):                                 │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/core.py:1157 in        │ +│__call__│ +││ +│1154 │   │ +│1155 │   def__call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any:                           │ +│1156 │   │   """Alias for :meth:`main`."""│ +│❱ 1157 │   │   returnself.main(*args, **kwargs)                                                 │ +│1158 │ +│1159 │ +│1160 classCommand(BaseCommand):                                                               │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/rich_click/rich_command.py:1│ +│52 in main│ +││ +│149 │   │   try:                                                                               │ +│150 │   │   │   try:                                                                           │ +│151 │   │   │   │   withself.make_context(prog_name, args, **extra) as ctx:                   │ +│❱ 152 │   │   │   │   │   rv = self.invoke(ctx)                                                  │ +│153 │   │   │   │   │   ifnot standalone_mode:                                                │ +│154 │   │   │   │   │   │   return rv                                                          │ +│155 │   │   │   │   │   # it's not safe to `ctx.exit(rv)` here!│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/core.py:1688 in invoke│ +││ +│1685 │   │   │   │   super().invoke(ctx)                                                       │ +│1686 │   │   │   │   sub_ctx = cmd.make_context(cmd_name, args, parent=ctx)                    │ +│1687 │   │   │   │   with sub_ctx:                                                             │ +│❱ 1688 │   │   │   │   │   return _process_result(sub_ctx.command.invoke(sub_ctx))               │ +│1689 │   │   │ +│1690 │   │   # In chain mode we create the contexts step by step, but after the│ +│1691 │   │   # base command has been invoked.  Because at that point we do not│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/core.py:1434 in invoke│ +││ +│1431 │   │   │   echo(style(message, fg="red"), err=True)                                      │ +│1432 │   │   │ +│1433 │   │   ifself.callback isnotNone:                                                     │ +│❱ 1434 │   │   │   return ctx.invoke(self.callback, **ctx.params)                                │ +│1435 │   │ +│1436 │   defshell_complete(self, ctx: Context, incomplete: str) -> t.List["CompletionItem"]:  │ +│1437 │   │   """Return a list of completions for the incomplete value. Looks│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/core.py:783 in invoke│ +││ +│ 780 │   │   │ +│ 781 │   │   with augment_usage_errors(__self):                                                │ +│ 782 │   │   │   with ctx:                                                                     │ +│❱  783 │   │   │   │   return __callback(*args, **kwargs)                                        │ +│ 784 │   │ +│ 785 │   defforward(                                                                          │ +│ 786 │   │   __self, __cmd: "Command", *args: t.Any, **kwargs: t.Any  # noqa: B902│ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/click/decorators.py:33 in    │ +│new_func│ +││ +│ 30 │   """│ +│ 31 │   │ +│ 32 │   defnew_func(*args: "P.args", **kwargs: "P.kwargs") -> "R":                            │ +│❱  33 │   │   return f(get_current_context(), *args, **kwargs)                                   │ +│ 34 │   │ +│ 35 │   return update_wrapper(new_func, f)                                                     │ +│ 36 │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/__main__.py:635 in   │ +│lint│ +││ +│ 632 │   │ +│ 633 │   # Run the lint tests!│ +│ 634 │   try:                                                                                  │ +│❱  635 │   │   lint_obj, module_lint_obj, subworkflow_lint_obj = run_linting(                    │ +│ 636 │   │   │   dir,                                                                          │ +│ 637 │   │   │   release,                                                                      │ +│ 638 │   │   │   fix,                                                                          │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/lint/__init__.py:603│ +│ in run_linting│ +││ +│600 │   │ +│601 │   # Run the pipeline linting tests│ +│602 │   try:                                                                                   │ +│❱ 603 │   │   lint_obj._lint_pipeline()                                                          │ +│604 │   exceptAssertionErroras e:                                                            │ +│605 │   │   log.critical(f"Critical error: {e}")                                               │ +│606 │   │   log.info("Stopping tests...")                                                      │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/lint/__init__.py:228│ +│ in _lint_pipeline│ +││ +│225 │   │   │   │   │   continue│ +│226 │   │   │   │   self.progress_bar.update(lint_progress, advance=1, test_name=test_name)    │ +│227 │   │   │   │   log.debug(f"Running lint test: {test_name}")                               │ +│❱ 228 │   │   │   │   test_results = getattr(self, test_name)()                                  │ +│229 │   │   │   │   for test in test_results.get("passed", []):                                │ +│230 │   │   │   │   │   self.passed.append((test_name, test))                                  │ +│231 │   │   │   │   for test in test_results.get("ignored", []):                               │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/nf_core/lint/actions_schema_│ +│validation.py:31 in actions_schema_validation│ +││ +│28 │   │ +│29 │   # Load the GitHub workflow schema│ +│30 │   r = requests.get("https://json.schemastore.org/github-workflow", allow_redirects=Tru│ +│❱ 31 │   schema = r.json()                                                                       │ +│32 │   │ +│33 │   # Validate all workflows against the schema│ +│34 │   for wf_path in action_workflows:                                                        │ +││ +│/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/requests/models.py:975 in    │ +│json│ +││ +│ 972 │   │   except JSONDecodeError as e:                                                      │ +│ 973 │   │   │   # Catch JSON-related errors and raise as requests.JSONDecodeError│ +│ 974 │   │   │   # This aliases json.JSONDecodeError and simplejson.JSONDecodeError│ +│❱  975 │   │   │   raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)                            │ +│ 976 │   │ +│ 977 │   @property│ +│ 978 │   deflinks(self):                                                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +JSONDecodeError: Expecting value: line 1 column 1(char 0) diff --git a/docs/images/nf-core-list-rna.svg b/docs/images/nf-core-list-rna.svg index d9df6fe641..26d224ddc2 100644 --- a/docs/images/nf-core-list-rna.svg +++ b/docs/images/nf-core-list-rna.svg @@ -19,114 +19,114 @@ font-weight: 700; } - .terminal-1811696893-matrix { + .terminal-3988802752-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1811696893-title { + .terminal-3988802752-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1811696893-r1 { fill: #c5c8c6 } -.terminal-1811696893-r2 { fill: #98a84b } -.terminal-1811696893-r3 { fill: #9a9b99 } -.terminal-1811696893-r4 { fill: #608ab1 } -.terminal-1811696893-r5 { fill: #d0b344 } -.terminal-1811696893-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-1811696893-r7 { fill: #868887 } + .terminal-3988802752-r1 { fill: #c5c8c6 } +.terminal-3988802752-r2 { fill: #98a84b } +.terminal-3988802752-r3 { fill: #9a9b99 } +.terminal-3988802752-r4 { fill: #608ab1 } +.terminal-3988802752-r5 { fill: #d0b344 } +.terminal-3988802752-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-3988802752-r7 { fill: #868887 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -138,38 +138,38 @@ - + - - $ nf-core list rna rna-seq - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ -│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ -│ spatialvi           │    39 │            dev │     yesterday │           - │ -                   │ -│ scnanoseq           │     5 │            dev │    6 days ago │           - │ -                   │ -│ scrnaseq            │   169 │          2.5.1 │  4 months ago │           - │ -                   │ -│ differentialabunda… │    45 │          1.4.0 │  5 months ago │           - │ -                   │ -│ circrna             │    40 │            dev │   2 weeks ago │           - │ -                   │ -│ rnafusion           │   129 │          3.0.2 │   4 weeks ago │           - │ -                   │ -│ rnasplice           │    33 │          1.0.3 │  2 months ago │           - │ -                   │ -│ dualrnaseq          │    16 │          1.0.0 │   3 years ago │           - │ -                   │ -│ marsseq             │     5 │          1.0.3 │ 10 months ago │           - │ -                   │ -│ lncpipe             │    30 │            dev │   2 years ago │           - │ -                   │ -│ scflow              │    24 │            dev │   3 years ago │           - │ -                   │ -└─────────────────────┴───────┴────────────────┴───────────────┴─────────────┴─────────────────────┘ + + $ nf-core list rna rna-seq + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ differentialabunda… │    45 │          1.4.0 │  5 months ago │           - │ -                   │ +│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ +│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ +│ spatialvi           │    39 │            dev │     yesterday │           - │ -                   │ +│ scnanoseq           │     5 │            dev │    1 week ago │           - │ -                   │ +│ scrnaseq            │   169 │          2.6.0 │  18 hours ago │           - │ -                   │ +│ circrna             │    40 │            dev │   2 weeks ago │           - │ -                   │ +│ rnafusion           │   129 │          3.0.2 │   4 weeks ago │           - │ -                   │ +│ rnasplice           │    33 │          1.0.3 │  2 months ago │           - │ -                   │ +│ dualrnaseq          │    16 │          1.0.0 │   3 years ago │           - │ -                   │ +│ marsseq             │     5 │          1.0.3 │ 10 months ago │           - │ -                   │ +│ lncpipe             │    30 │            dev │   2 years ago │           - │ -                   │ +│ scflow              │    24 │            dev │   3 years ago │           - │ -                   │ +└─────────────────────┴───────┴────────────────┴───────────────┴─────────────┴─────────────────────┘ diff --git a/docs/images/nf-core-list-stars.svg b/docs/images/nf-core-list-stars.svg index 49c10f7cd7..be2b84287b 100644 --- a/docs/images/nf-core-list-stars.svg +++ b/docs/images/nf-core-list-stars.svg @@ -19,88 +19,88 @@ font-weight: 700; } - .terminal-3664247200-matrix { + .terminal-358676910-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3664247200-title { + .terminal-358676910-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3664247200-r1 { fill: #c5c8c6 } -.terminal-3664247200-r2 { fill: #98a84b } -.terminal-3664247200-r3 { fill: #9a9b99 } -.terminal-3664247200-r4 { fill: #608ab1 } -.terminal-3664247200-r5 { fill: #d0b344 } -.terminal-3664247200-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-3664247200-r7 { fill: #868887 } -.terminal-3664247200-r8 { fill: #868887;font-style: italic; } + .terminal-358676910-r1 { fill: #c5c8c6 } +.terminal-358676910-r2 { fill: #98a84b } +.terminal-358676910-r3 { fill: #9a9b99 } +.terminal-358676910-r4 { fill: #608ab1 } +.terminal-358676910-r5 { fill: #d0b344 } +.terminal-358676910-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-358676910-r7 { fill: #868887 } +.terminal-358676910-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -112,29 +112,29 @@ - + - - $ nf-core list -s stars - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ -│ sarek               │   340 │          3.4.2 │   4 hours ago │           - │ -                   │ -│ mag                 │   182 │          2.5.4 │  3 months ago │           - │ -                   │ -│ chipseq             │   172 │          2.0.0 │   2 years ago │           - │ -                   │ -[..truncated..] + + $ nf-core list -s stars + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ rnaseq              │   784 │         3.14.0 │  4 months ago │           - │ -                   │ +│ sarek               │   340 │          3.4.2 │  20 hours ago │           - │ -                   │ +│ mag                 │   182 │          2.5.4 │  3 months ago │           - │ -                   │ +│ chipseq             │   172 │          2.0.0 │   2 years ago │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-list.svg b/docs/images/nf-core-list.svg index ede53daf6f..6857db79f9 100644 --- a/docs/images/nf-core-list.svg +++ b/docs/images/nf-core-list.svg @@ -19,91 +19,91 @@ font-weight: 700; } - .terminal-2737538270-matrix { + .terminal-4024274838-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2737538270-title { + .terminal-4024274838-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2737538270-r1 { fill: #c5c8c6 } -.terminal-2737538270-r2 { fill: #98a84b } -.terminal-2737538270-r3 { fill: #9a9b99 } -.terminal-2737538270-r4 { fill: #608ab1 } -.terminal-2737538270-r5 { fill: #d0b344 } -.terminal-2737538270-r6 { fill: #c5c8c6;font-weight: bold } -.terminal-2737538270-r7 { fill: #868887 } -.terminal-2737538270-r8 { fill: #868887;font-style: italic; } + .terminal-4024274838-r1 { fill: #c5c8c6 } +.terminal-4024274838-r2 { fill: #98a84b } +.terminal-4024274838-r3 { fill: #9a9b99 } +.terminal-4024274838-r4 { fill: #608ab1 } +.terminal-4024274838-r5 { fill: #d0b344 } +.terminal-4024274838-r6 { fill: #c5c8c6;font-weight: bold } +.terminal-4024274838-r7 { fill: #868887 } +.terminal-4024274838-r8 { fill: #868887;font-style: italic; } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -115,30 +115,30 @@ - + - - $ nf-core list - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - -┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ -┃┃┃┃┃┃Have latest        ┃ -┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ -┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ -│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ -│ sarek               │   340 │          3.4.2 │   4 hours ago │           - │ -                   │ -│ variantbenchmarking │     4 │            dev │   3 hours ago │           - │ -                   │ -│ meerpipe            │     0 │            dev │   9 hours ago │           - │ -                   │ -│ mcmicro             │     4 │            dev │  22 hours ago │           - │ -                   │ -[..truncated..] + + $ nf-core list + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + +┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓ +┃┃┃┃┃┃Have latest        ┃ +┃Pipeline Name      ┃Stars┃Latest Release┃     Released┃Last Pulled┃release?           ┃ +┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩ +│ differentialabunda… │    45 │          1.4.0 │  5 months ago │           - │ -                   │ +│ smrnaseq            │    68 │          2.3.1 │   3 weeks ago │           - │ -                   │ +│ sarek               │   340 │          3.4.2 │  20 hours ago │           - │ -                   │ +│ variantbenchmarking │     4 │            dev │  19 hours ago │           - │ -                   │ +│ meerpipe            │     0 │            dev │     yesterday │           - │ -                   │ +[..truncated..] diff --git a/docs/images/nf-core-subworkflows-create.svg b/docs/images/nf-core-subworkflows-create.svg index f5b3212fa7..52d5e7c39f 100644 --- a/docs/images/nf-core-subworkflows-create.svg +++ b/docs/images/nf-core-subworkflows-create.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - - -ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + $ nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Repository type: modules +INFO    Press enter to use default values (shown in brackets)or type your own responses.  +ctrl+click underlined text to open links. +INFO     Created component template: 'bam_stats_samtools' +INFO     Created following files:                                                                    +           subworkflows/nf-core/bam_stats_samtools/main.nf                                           +           subworkflows/nf-core/bam_stats_samtools/meta.yml                                          +           subworkflows/nf-core/bam_stats_samtools/tests/tags.yml                                    +           subworkflows/nf-core/bam_stats_samtools/tests/main.nf.test                                diff --git a/docs/images/nf-core-subworkflows-lint.svg b/docs/images/nf-core-subworkflows-lint.svg index 8250d22683..a4497c6cd2 100644 --- a/docs/images/nf-core-subworkflows-lint.svg +++ b/docs/images/nf-core-subworkflows-lint.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core subworkflows lint bam_stats_samtools - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - - -CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + + + $ nf-core subworkflows lint bam_stats_samtools + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Linting modules repo: '.' +INFO     Linting subworkflow: 'bam_stats_samtools' + +╭─[!] 15 Subworkflow Test Warnings───────────────────────────────────────────────────────────────╮ +│                     ╷                                     ╷                                      │ +│Subworkflow name   │File path                          │Test message                        │ +│╶────────────────────┼─────────────────────────────────────┼─────────────────────────────────────╴│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: //Add │ +│││all required assertions to verify │ +│││the test output.│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Add │ +│││tags for all modules used within │ +│││this subworkflow. Example:│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Change │ +│││the test name preferably indicating │ +│││the test-data and file-format used│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: Once │ +│││you have added the required tests, │ +│││please run the following command to │ +│││build this file:│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf.test: define │ +│││inputs of the workflow here. │ +│││Example:│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: A │ +│││subworkflow SHOULD import at least │ +│││two modules│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: If in doubt │ +│││look at other nf-core/subworkflows │ +│││to see how we are doing things! :)│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit emitted│ +│││channels│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: edit input │ +│││(take) channels│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in main.nf: substitute │ +│││modules here for the modules of your│ +│││subworkflow│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a │ +│││description of the subworkflow and │ +│││list keywords│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #Add a list│ +│││of the modules and/or subworkflows │ +│││used in the subworkflow│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ +│││of the channels used as input with a│ +│││description and their structure│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│TODO string in meta.yml: #List all │ +│││of the channels used as output with │ +│││a descriptions and their structure│ +│bam_stats_samtools │subworkflows/nf-core/bam_stats_sam…│versions not found in snapshot file │ +│                     ╵                                     ╵                                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭───────────────────────╮ +│LINT RESULTS SUMMARY│ +├───────────────────────┤ +│[✔]  42 Tests Passed │ +│[!]  15 Test Warnings│ +│[✗]   0 Tests Failed │ +╰───────────────────────╯ diff --git a/docs/images/nf-core-subworkflows-remove.svg b/docs/images/nf-core-subworkflows-remove.svg index 61e782532d..0fa3b40f89 100644 --- a/docs/images/nf-core-subworkflows-remove.svg +++ b/docs/images/nf-core-subworkflows-remove.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + - + - - - - $ nf-core subworkflows remove bam_rseqc - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - - -CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + + + $ nf-core subworkflows remove bam_rseqc + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Removed files for 'rseqc/bamstat' and its dependencies 'rseqc/bamstat'.                     +INFO     Removed files for 'rseqc/inferexperiment' and its dependencies 'rseqc/inferexperiment'.     +INFO     Removed files for 'rseqc/innerdistance' and its dependencies 'rseqc/innerdistance'.         +INFO     Removed files for 'rseqc/junctionannotation' and its dependencies                           +'rseqc/junctionannotation'.                                                                 +INFO     Removed files for 'bam_rseqc' and its dependencies 'bam_rseqc, rseqc_bamstat,  +rseqc_inferexperiment, rseqc_innerdistance, rseqc_junctionannotation'.                      diff --git a/docs/images/nf-core-subworkflows-test.svg b/docs/images/nf-core-subworkflows-test.svg index 447fb6e5ec..a497ee900d 100644 --- a/docs/images/nf-core-subworkflows-test.svg +++ b/docs/images/nf-core-subworkflows-test.svg @@ -19,65 +19,64 @@ font-weight: 700; } - .terminal-3677110743-matrix { + .terminal-1767306345-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3677110743-title { + .terminal-1767306345-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3677110743-r1 { fill: #c5c8c6 } -.terminal-3677110743-r2 { fill: #98a84b } -.terminal-3677110743-r3 { fill: #9a9b99 } -.terminal-3677110743-r4 { fill: #608ab1 } -.terminal-3677110743-r5 { fill: #d0b344 } -.terminal-3677110743-r6 { fill: #292929;font-weight: bold } + .terminal-1767306345-r1 { fill: #c5c8c6 } +.terminal-1767306345-r2 { fill: #98a84b } +.terminal-1767306345-r3 { fill: #9a9b99 } +.terminal-1767306345-r4 { fill: #608ab1 } +.terminal-1767306345-r5 { fill: #d0b344 } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +88,22 @@ - - - - $ nf-core subworkflows test bam_rseqc --no-prompts - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - - -CRITICAL Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + + + $ nf-core subworkflows test bam_rseqc --no-prompts + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +INFO     Generating nf-test snapshot                                                                 diff --git a/docs/images/nf-core-subworkflows-update.svg b/docs/images/nf-core-subworkflows-update.svg index e76364a259..bff01d365b 100644 --- a/docs/images/nf-core-subworkflows-update.svg +++ b/docs/images/nf-core-subworkflows-update.svg @@ -19,65 +19,65 @@ font-weight: 700; } - .terminal-2341026535-matrix { + .terminal-795743372-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2341026535-title { + .terminal-795743372-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2341026535-r1 { fill: #c5c8c6 } -.terminal-2341026535-r2 { fill: #98a84b } -.terminal-2341026535-r3 { fill: #9a9b99 } -.terminal-2341026535-r4 { fill: #608ab1 } -.terminal-2341026535-r5 { fill: #d0b344 } -.terminal-2341026535-r6 { fill: #cc555a;font-weight: bold } + .terminal-795743372-r1 { fill: #c5c8c6 } +.terminal-795743372-r2 { fill: #98a84b } +.terminal-795743372-r3 { fill: #9a9b99 } +.terminal-795743372-r4 { fill: #608ab1 } +.terminal-795743372-r5 { fill: #d0b344 } +.terminal-795743372-r6 { fill: #cc555a;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -89,22 +89,22 @@ - + - - $ nf-core subworkflows update --all --no-preview - -                                          ,--./,-. -          ___     __   __   __   ___     /,-._.--~\ -    |\ | |__  __ /  ` /  \ |__) |__         }  { -    | \| |       \__, \__/ |  \ |___     \`-._,-`-, -                                          `._,._,' - -    nf-core/tools version 2.14.0 - https://nf-co.re - - - -ERROR    Branch 'master' not found in 'https://github.com/nf-core/modules.git' + + $ nf-core subworkflows update --all --no-preview + +                                          ,--./,-. +          ___     __   __   __   ___     /,-._.--~\ +    |\ | |__  __ /  ` /  \ |__) |__         }  { +    | \| |       \__, \__/ |  \ |___     \`-._,-`-, +                                          `._,._,' + +    nf-core/tools version 2.14.0 - https://nf-co.re + + + +ERROR   'rseqc/junctionsaturation' From 8616411691c0309bc6cf0b9d65680ee34412a049 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 8 May 2024 12:12:37 +0200 Subject: [PATCH 267/270] handle request errors more gracefully --- nf_core/lint/actions_schema_validation.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nf_core/lint/actions_schema_validation.py b/nf_core/lint/actions_schema_validation.py index fa4471d98c..386b63eccf 100644 --- a/nf_core/lint/actions_schema_validation.py +++ b/nf_core/lint/actions_schema_validation.py @@ -19,6 +19,7 @@ def actions_schema_validation(self): """ passed = [] failed = [] + warned = [] # Only show error messages from schema logging.getLogger("nf_core.schema").setLevel(logging.ERROR) @@ -28,6 +29,12 @@ def actions_schema_validation(self): # Load the GitHub workflow schema r = requests.get("https://json.schemastore.org/github-workflow", allow_redirects=True) + # handle "Service Unavailable" error + if r.status_code not in [200, 301]: + warned.append( + f"Failed to fetch schema: Response code for `https://json.schemastore.org/github-workflow` was {r.status_code}" + ) + return {"passed": passed, "failed": failed, "warned": warned} schema = r.json() # Validate all workflows against the schema @@ -55,4 +62,4 @@ def actions_schema_validation(self): except Exception as e: failed.append(f"Workflow validation failed for {wf}: {e}") - return {"passed": passed, "failed": failed} + return {"passed": passed, "failed": failed, "warned": warned} From dcfcd4ddad5db856e41282dcf3e904001215141b Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 8 May 2024 12:17:45 +0200 Subject: [PATCH 268/270] add type hints --- nf_core/lint/actions_schema_validation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nf_core/lint/actions_schema_validation.py b/nf_core/lint/actions_schema_validation.py index 386b63eccf..7e878a1af8 100644 --- a/nf_core/lint/actions_schema_validation.py +++ b/nf_core/lint/actions_schema_validation.py @@ -1,13 +1,14 @@ import glob import logging import os +from typing import Any, Dict, List import jsonschema import requests import yaml -def actions_schema_validation(self): +def actions_schema_validation(self) -> Dict[str, List[str]]: """Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. @@ -17,9 +18,9 @@ def actions_schema_validation(self): To pass this test, make sure that all your workflows contain the required properties ``on`` and ``jobs`` and that all other properties are of the correct type, as specified in the schema (link above). """ - passed = [] - failed = [] - warned = [] + passed: List[str] = [] + failed: List[str] = [] + warned: List[str] = [] # Only show error messages from schema logging.getLogger("nf_core.schema").setLevel(logging.ERROR) @@ -35,7 +36,7 @@ def actions_schema_validation(self): f"Failed to fetch schema: Response code for `https://json.schemastore.org/github-workflow` was {r.status_code}" ) return {"passed": passed, "failed": failed, "warned": warned} - schema = r.json() + schema: Dict[str, Any] = r.json() # Validate all workflows against the schema for wf_path in action_workflows: @@ -53,7 +54,7 @@ def actions_schema_validation(self): try: wf_json["on"] = wf_json.pop(True) except Exception: - failed.append("Missing 'on' keyword in {}.format(wf)") + failed.append(f"Missing 'on' keyword in {wf}") # Validate the workflow try: From bd8e46e680f71a6e6ea3eec08eed9b402cc0855f Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 8 May 2024 12:21:05 +0200 Subject: [PATCH 269/270] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba5d53f3c..a7eea54dd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Include test for presence of versions in snapshot ([#2888](https://github.com/nf-core/tools/pull/2888)) - Components: set correct sha before running component lint tests ([#2952](https://github.com/nf-core/tools/pull/2952)) - Less strict logo comparison ([#2956](https://github.com/nf-core/tools/pull/2956)) +- Handle request errors more gracefully for actions validation ([#2959](https://github.com/nf-core/tools/pull/2959)) ### Download From bda8d42627618d566f731e685f8d0e9f08512cd5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 8 May 2024 13:49:35 +0200 Subject: [PATCH 270/270] fix linting test --- tests/lint/actions_schema_validation.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/lint/actions_schema_validation.py b/tests/lint/actions_schema_validation.py index ad65d90018..e202b3b1ce 100644 --- a/tests/lint/actions_schema_validation.py +++ b/tests/lint/actions_schema_validation.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path import yaml @@ -9,10 +9,11 @@ def test_actions_schema_validation_missing_jobs(self): """Missing 'jobs' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: + awstest_yml_path = Path(new_pipeline) / ".github" / "workflows" / "awstest.yml" + with open(awstest_yml_path) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml.pop("jobs") - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + with open(awstest_yml_path, "w") as fh: yaml.dump(awstest_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -27,10 +28,11 @@ def test_actions_schema_validation_missing_on(self): """Missing 'on' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: + awstest_yml_path = Path(new_pipeline) / ".github" / "workflows" / "awstest.yml" + with open(awstest_yml_path) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml.pop(True) - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + with open(awstest_yml_path, "w") as fh: yaml.dump(awstest_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -38,7 +40,7 @@ def test_actions_schema_validation_missing_on(self): results = lint_obj.actions_schema_validation() - assert results["failed"][0] == "Missing 'on' keyword in {}.format(wf)" + assert results["failed"][0] == "Missing 'on' keyword in awstest.yml" assert "Workflow validation failed for awstest.yml: 'on' is a required property" in results["failed"][1] @@ -46,10 +48,11 @@ def test_actions_schema_validation_fails_for_additional_property(self): """Missing 'jobs' field should result in failure""" new_pipeline = self._make_pipeline_copy() - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml")) as fh: + awstest_yml_path = Path(new_pipeline) / ".github" / "workflows" / "awstest.yml" + with open(awstest_yml_path) as fh: awstest_yml = yaml.safe_load(fh) awstest_yml["not_jobs"] = awstest_yml["jobs"] - with open(os.path.join(new_pipeline, ".github", "workflows", "awstest.yml"), "w") as fh: + with open(awstest_yml_path, "w") as fh: yaml.dump(awstest_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline)