From a1e64127b5dfc87bbc5a6195667a81e1728ea8a0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 5 Dec 2022 11:46:26 +0100 Subject: [PATCH 01/13] add conda channel order to nextflow.config template --- nf_core/pipeline-template/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 8ccb08d409..e30cbcd6cb 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -91,6 +91,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + channels = ['conda-forge','bioconda','defaults'] } mamba { params.enable_conda = true From a023d6e572d4f1f150ebc6cab85f08ac585b8e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Mon, 5 Dec 2022 12:46:41 +0100 Subject: [PATCH 02/13] Update nf_core/pipeline-template/nextflow.config Co-authored-by: Maxime U. Garcia --- nf_core/pipeline-template/nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index e30cbcd6cb..e7faeb41c1 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -91,7 +91,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false - channels = ['conda-forge','bioconda','defaults'] + channels = ['conda-forge', 'bioconda', 'defaults'] } mamba { params.enable_conda = true From 6ea4d62ed918919d187bfbbf0988e12679a0cff3 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 20 Dec 2023 16:47:55 +0100 Subject: [PATCH 03/13] Add a Github Action Workflow to the pipeline template that tests a successful download. --- CHANGELOG.md | 2 + .../.github/workflows/download_pipeline.yml | 70 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 nf_core/pipeline-template/.github/workflows/download_pipeline.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a252cbf67..c6fd827245 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Add a Github Action Workflow to the pipeline template that tests a successful download with 'nf-core download' ([#2618](https://github.com/nf-core/tools/pull/2618)) + ### Download ### Linting diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml new file mode 100644 index 0000000000..7653ac8af4 --- /dev/null +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -0,0 +1,70 @@ +name: Test successful pipeline download with 'nf-core download' + +# Run the workflow when: +# - dispatched manually +# - when a PR is opened or reopened to dev or master branches +# - the head branch of the pull request is updated. (Do we need this?) + +on: + workflow_dispatch: + pull_request: + types: + - opened + branches: + - master + - dev + #pull_request_target: + # branches: + # - master + # - dev + +env: + NXF_ANSI_LOG: false + +jobs: + download: + runs-on: ubuntu-latest + steps: + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + architecture: "x64" + - uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.8.3 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install git+https://github.com/nf-core/tools.git@dev + + - name: Get the repository name and current branch set as environment variable + run: | + echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} + echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} + echo "REPO_BRANCH=${GITHUB_REF#refs/heads/}" >> ${GITHUB_ENV} + + - name: Download the pipeline + env: + NXF_SINGULARITY_CACHEDIR: ./ + run: | + nf-core download ${{ env.REPO_LOWERCASE }} \ + --revision ${{ env.REPO_BRANCH }} \ + --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ + --compress "none" \ + --container-system 'singularity' \ + --container-library "quay.io" -l "docker.io" -l "ghcr.io" \ + --container-cache-utilisation 'amend' \ + --download-configuration + + - name: Inspect download + run: tree ./${{ env.REPOTITLE_LOWERCASE }} + + - name: Run the downloaded pipeline + env: + NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_HOME_MOUNT: true + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results From d4f3dc2c7941f5389c1f365d8f3b60aece10e93e Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 4 Jan 2024 14:40:20 +0100 Subject: [PATCH 04/13] Only run on self-hosted runners due to space constraints in GitHub hosted runners (large container downloads). --- .../pipeline-template/.github/workflows/download_pipeline.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 7653ac8af4..1577304e7b 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -12,18 +12,16 @@ on: - opened branches: - master - - dev #pull_request_target: # branches: # - master - # - dev env: NXF_ANSI_LOG: false jobs: download: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - name: Install Nextflow uses: nf-core/setup-nextflow@v1 From f93b9244adfdbf85105c54360c2e9967e51df063 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Thu, 4 Jan 2024 14:46:56 +0100 Subject: [PATCH 05/13] Since we now limit it to PRs to master, updating a PR branch should be a trigger as well. This only applies to last-minute fixes prior to a pipeline release. --- .../.github/workflows/download_pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 1577304e7b..820c02bec4 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -3,7 +3,7 @@ name: Test successful pipeline download with 'nf-core download' # Run the workflow when: # - dispatched manually # - when a PR is opened or reopened to dev or master branches -# - the head branch of the pull request is updated. (Do we need this?) +# - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. on: workflow_dispatch: @@ -12,9 +12,9 @@ on: - opened branches: - master - #pull_request_target: - # branches: - # - master + pull_request_target: + branches: + - master env: NXF_ANSI_LOG: false From f0d154342d8d2ec68b02acfd88be73098b10c11d Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 5 Jan 2024 12:22:57 +0100 Subject: [PATCH 06/13] add more verbose fix-linting bot --- .github/workflows/fix-linting.yml | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 5e857e8dbb..5f157b6b5a 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -17,6 +17,13 @@ jobs: with: token: ${{ secrets.nf_core_bot_auth_token }} + # indication that the linting is being fixed + - name: React on comment + uses: peter-evans/create-or-update-comment@v3 + with: + comment-id: ${{ github.event.comment.id }} + reactions: eyes + # Action runs on the issue comment, so we don't get the PR by default # Use the gh cli to check out the PR - name: Checkout Pull Request @@ -34,9 +41,21 @@ jobs: run: pip install pre-commit - name: Run pre-commit + id: pre-commit run: pre-commit run --all-files + continue-on-error: true + + # indication that the linting has finished + - name: react if linting finished succesfully + if: ${{ steps.pre-commit.outcome }} == 'success' + uses: peter-evans/create-or-update-comment@v3 + with: + comment-id: ${{ github.event.comment.id }} + reactions: green_check_mark - name: Commit & push changes + id: commit-and-push + if: ${{ steps.pre-commit.outcome }} == 'failure' run: | git config user.email "core@nf-co.re" git config user.name "nf-core-bot" @@ -45,3 +64,19 @@ jobs: git status git commit -m "[automated] Fix code linting" git push + + - name: react if linting errors were fixed + if: ${{ steps.commit-and-push.outcome }} == 'success' + uses: peter-evans/create-or-update-comment@v3 + with: + comment-id: ${{ github.event.comment.id }} + reactions: pencil2 + + - name: react if linting errors were not fixed + if: ${{ steps.commit-and-push.outcome }} == 'failure' + uses: peter-evans/create-or-update-comment@v3 + with: + comment-id: ${{ github.event.comment.id }} + reactions: x + body: | + @${{ github.actor }} I tried to fix the linting errors, but it didn't work. Please fix them manually. From ce26f6407ecb1eb59f67c9cd80ba7f8a386e4c1b Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 5 Jan 2024 12:25:49 +0100 Subject: [PATCH 07/13] introduce linting error --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 3a9b31187a..05620b4ac1 100644 --- a/README.md +++ b/README.md @@ -291,7 +291,6 @@ Do you want to run this command now? [y/n]: ``` ### Launch tool options - - `-r`, `--revision` - Specify a pipeline release (or branch / git commit sha) of the project to run - `-i`, `--id` From 444aa84c29d6c3d0ec837cfa96055e32323eedf1 Mon Sep 17 00:00:00 2001 From: mashehu Date: Fri, 5 Jan 2024 12:29:39 +0100 Subject: [PATCH 08/13] Revert "introduce linting error" This reverts commit ce26f6407ecb1eb59f67c9cd80ba7f8a386e4c1b. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 05620b4ac1..3a9b31187a 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,7 @@ Do you want to run this command now? [y/n]: ``` ### Launch tool options + - `-r`, `--revision` - Specify a pipeline release (or branch / git commit sha) of the project to run - `-i`, `--id` From 2f3e38fc65a2a79baeeb6851a494ba6f1ffcaf48 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 8 Jan 2024 13:23:39 +0100 Subject: [PATCH 09/13] set pdiff is nf-test differ in docker image for gitpod --- .gitpod.yml | 5 ----- nf_core/gitpod/gitpod.Dockerfile | 6 +++++- nf_core/pipeline-template/.gitpod.yml | 5 +---- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index cadccac4ba..39fb530da2 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -10,11 +10,6 @@ tasks: command: | unset JAVA_TOOL_OPTIONS - - name: Use pdiff for nf-test diff viewing - command: | - export NFT_DIFF="pdiff" - export NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" - vscode: extensions: # based on nf-core.nf-core-extensionpack - codezombiech.gitignore # Language support for .gitignore files diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index de1086410b..ad4bed5052 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -62,4 +62,8 @@ RUN conda config --add channels defaults && \ RUN nextflow self-update # Install nf-core -RUN python -m pip install . +RUN python -m pip install . --no-cache-dir + +# Setup pdiff for nf-test diffs +RUN export NFT_DIFF="pdiff" && \ + export NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index 78d68ffcb0..363d5b1d42 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -7,10 +7,7 @@ tasks: - name: unset JAVA_TOOL_OPTIONS command: | unset JAVA_TOOL_OPTIONS - - name: Use pdiff for nf-test diff viewing - command: | - export NFT_DIFF="pdiff" - export NFT_DIFF_ARGS="--line-numbers --expand-tabs=2" + vscode: extensions: # based on nf-core.nf-core-extensionpack - codezombiech.gitignore # Language support for .gitignore files From f54f1f051d2dafd3c0e44e08618b8872f669a8f9 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 8 Jan 2024 12:26:44 +0000 Subject: [PATCH 10/13] [automated] Update CHANGELOG.md [no ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4bcbce645..fdf2555e49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Update pre-commit hook pre-commit/mirrors-mypy to v1.8.0 ([#2630](https://github.com/nf-core/tools/pull/2630)) - Update mshick/add-pr-comment action to v2 ([#2632](https://github.com/nf-core/tools/pull/2632)) - update python image version in docker file ([#2636](https://github.com/nf-core/tools/pull/2636)) +- Set pdiff as nf-test differ in Docker image for Gitpod ([#2642](https://github.com/nf-core/tools/pull/2642)) # [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20] From 22613c50c1a5fa47b79e672d5fddd9c611862947 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 8 Jan 2024 13:28:25 +0100 Subject: [PATCH 11/13] update changelog [no ci] --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdf2555e49..001948b776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ ### Template - Use `pre-commit` to lint files in GitHub CI ([#2635](https://github.com/nf-core/tools/pull/2635)) -- Use pdiff also on gitpod for nf-test ([#2640](https://github.com/nf-core/tools/pull/2640)) ### Download From 9fb5498b7fcc442b2803b8d9462cf9a713990602 Mon Sep 17 00:00:00 2001 From: Matthias Zepper <6963520+MatthiasZepper@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:10:52 +0100 Subject: [PATCH 12/13] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks @mirpedrol ! Co-authored-by: Júlia Mir Pedrol --- .../pipeline-template/.github/workflows/download_pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 820c02bec4..d5d0307262 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -2,7 +2,7 @@ name: Test successful pipeline download with 'nf-core download' # Run the workflow when: # - dispatched manually -# - when a PR is opened or reopened to dev or master branches +# - when a PR is opened or reopened to master branch # - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. on: @@ -28,7 +28,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: "3.11" architecture: "x64" - uses: eWaterCycle/setup-singularity@v7 with: From df48d77be32c890c83e80031537fe4c2b8ed65ef Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 9 Jan 2024 09:38:04 +0100 Subject: [PATCH 13/13] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc96df408f..ec96e24d54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Add a Github Action Workflow to the pipeline template that tests a successful download with 'nf-core download' ([#2618](https://github.com/nf-core/tools/pull/2618)) - Use `pre-commit` to lint files in GitHub CI ([#2635](https://github.com/nf-core/tools/pull/2635)) +- Add conda channel order to nextflow.config ([#2094](https://github.com/nf-core/tools/pull/2094)) ### Download